@theokit/sdk-memory 0.4.0 → 0.5.0

package/dist/index.js

@@ -1,9 +1,9 @@
 import { mkdir, readdir, readFile, stat, access, unlink } from 'fs/promises';
-import { join, resolve, relative } from 'path';
+import { join, resolve } from 'path';
 import { Security } from '@theokit/sdk';
-import { replaceFileAtomic, atomicWriteJson, withCwdMutex, openSqliteResilient, sanitizeFts5Query } from '@theokit/sdk/persistence';
-import { notesDir, memoryDir, readFactsFromMarkdown, memoryMdPath, appendFactToMarkdown } from '@theokit/sdk/internal/memory-store';
-export { appendFact, appendFactToMarkdown, claudeProjectMemoryDir, memoryDir, memoryMdPath, memoryWriteDir, notesDir, readFacts, readFactsFromMarkdown } from '@theokit/sdk/internal/memory-store';
+import { replaceFileAtomic, withCwdMutex, openSqliteResilient, sanitizeFts5Query } from '@theokit/sdk/persistence';
+import { notesDir, resolveMemoryRoot, persistActiveMemoryTranscript, readFactsFromMarkdown, appendDiaryEntry, lanceStoragePath, defaultIndexPath, projectMemoryDir, collectMarkdownFiles, readAllSqliteFacts, memoryMdPath, appendFactToMarkdown } from '@theokit/sdk/internal/memory-store';
+export { MEMORY_INDEX_MAX_BYTES, MEMORY_INDEX_MAX_LINES, appendDiaryEntry, appendFact, appendFactToMarkdown, asMemoryRoot, claudeProjectMemoryDir, collectMarkdownFiles, defaultIndexPath, diaryPath, discoverSessionFiles, discoverWikiFiles, entryHash, indexBudgetWarning, lanceStoragePath, memoryMdPath, memoryReadRoots, notesDir, persistActiveMemoryTranscript, projectMemoryDir, readAllSqliteFacts, readFacts, readFactsFromMarkdown, renderDiaryEntry, resolveMemoryRoot, sessionSummaryPath, sessionsDir, wikiDir, writeSessionSummary } from '@theokit/sdk/internal/memory-store';
 import { createHash } from 'crypto';
 import { createOpenAiCompatibleRuntime } from '@theokit/sdk/internal/memory-adapters';
 export { createOpenAiCompatibleRuntime } from '@theokit/sdk/internal/memory-adapters';
@@ -243,24 +243,14 @@ function createInMemoryMarkdownProvider() {
 async function listNotes(cwd) {
   let entries = [];
   try {
-    entries = await readdir(notesDir(cwd));
+    entries = await readdir(notesDir(resolveMemoryRoot(cwd)));
   } catch {
     return [];
   }
-  return entries.filter((name) => name.endsWith(".md")).map((name) => ({ slug: name.replace(/\.md$/, ""), path: join(notesDir(cwd), name) }));
-}
-
-// src/internal/store/transcript-store.ts
-async function persistActiveMemoryTranscript(cwd, transcript) {
-  try {
-    const dir = join(memoryDir(cwd), "transcripts", "active-memory");
-    const file = join(dir, `${transcript.runId}.json`);
-    await atomicWriteJson(file, transcript);
-  } catch (cause) {
-    const message = cause instanceof Error ? cause.message : String(cause);
-    process.stderr.write(`[theokit-sdk] active-memory transcript persist failed: ${message}
-`);
-  }
+  return entries.filter((name) => name.endsWith(".md")).map((name) => ({
+    slug: name.replace(/\.md$/, ""),
+    path: join(notesDir(resolveMemoryRoot(cwd)), name)
+  }));
 }
 
 // src/internal/active-memory/active-memory.ts
@@ -403,7 +393,7 @@ async function finalize(args, queryMode, result) {
   };
   args.cache?.set(args.userText, queryMode, result, tenantCtx);
   if (args.persistTranscripts === true && args.cwd !== void 0) {
-    await persistActiveMemoryTranscript(args.cwd, {
+    await persistActiveMemoryTranscript(resolveMemoryRoot(args.cwd), {
       runId: args.runId ?? `run-${Date.now()}`,
       startedAtMs: Date.now() - result.durationMs,
       userText: args.userText,
@@ -1085,47 +1075,6 @@ ${bullet}
   return `${raw.slice(0, insertAt)}
 ${bullet}${raw.slice(insertAt)}`;
 }
-function diaryPath(cwd) {
-  return join(memoryDir(cwd), "dream-diary.md");
-}
-function renderDiaryEntry(entry) {
-  const stamp = new Date(entry.timestampMs).toISOString();
-  const hash = entryHash(entry).slice(0, 8);
-  return [
-    `## ${stamp}`,
-    "",
-    `- entry-hash: ${hash}`,
-    `- facts before: ${entry.factsBefore}`,
-    `- facts after: ${entry.factsAfter}`,
-    `- duplicates removed: ${entry.duplicatesRemoved}`,
-    `- clusters created: ${entry.clustersCreated}`,
-    `- notes written: ${entry.notesWritten}`,
-    ""
-  ].join("\n");
-}
-async function appendDiaryEntry(cwd, entry) {
-  const path = diaryPath(cwd);
-  let raw = "";
-  try {
-    raw = await readFile(path, "utf8");
-  } catch {
-    raw = "# Dream Diary\n\n";
-  }
-  const next = `${raw.endsWith("\n") ? raw : `${raw}
-`}${renderDiaryEntry(entry)}`;
-  await replaceFileAtomic(path, next);
-}
-function entryHash(entry) {
-  return createHash("sha256").update(
-    [
-      entry.factsBefore,
-      entry.factsAfter,
-      entry.duplicatesRemoved,
-      entry.clustersCreated,
-      entry.notesWritten
-    ].join("|")
-  ).digest("hex");
-}
 
 // src/internal/dreaming/dreaming-phases.ts
 var DEFAULT_DEDUP_THRESHOLD = 0.95;
@@ -1247,7 +1196,7 @@ async function runInner(options) {
       notesWritten,
       diaryEntryHash: void 0
     };
-    await appendDiaryEntry(options.cwd, {
+    await appendDiaryEntry(resolveMemoryRoot(options.cwd), {
       timestampMs,
       factsBefore: result.factsBefore,
       factsAfter: result.factsAfter,
@@ -1265,7 +1214,7 @@ async function runInner(options) {
 }
 async function writeConsolidatedNotes(cwd, clusters, timestampMs) {
   if (clusters.length === 0) return 0;
-  const notesDir3 = join(memoryDir(cwd), "notes");
+  const notesDir3 = join(resolveMemoryRoot(cwd), "notes");
   await mkdir(notesDir3, { recursive: true });
   const isoSlug = new Date(timestampMs).toISOString().replace(/[^\dT]/g, "-");
   const file = join(notesDir3, `dreamed-${isoSlug}.md`);
@@ -1338,9 +1287,6 @@ async function openMemoryDb(opts) {
     }
   });
 }
-function defaultIndexPath(cwd) {
-  return join(cwd, ".theokit", "memory", ".index", "memory.sqlite");
-}
 var HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
 function chunkMarkdown(text, options = {}) {
   const maxChars = options.maxChars ?? 800;
@@ -1416,91 +1362,6 @@ function findWordBoundarySplit(text, maxChars) {
   }
   return maxChars;
 }
-var MAX_TURN_CHARS = 2e3;
-function sessionsDir(cwd) {
-  return join(memoryDir(cwd), "sessions");
-}
-function sessionSummaryPath(cwd, runId) {
-  return join(sessionsDir(cwd), `${sanitizeRunId2(runId)}.md`);
-}
-function sanitizeRunId2(runId) {
-  return runId.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 128);
-}
-function truncate2(text) {
-  if (text.length <= MAX_TURN_CHARS) return text;
-  return `${text.slice(0, MAX_TURN_CHARS)}\u2026`;
-}
-async function writeSessionSummary(input) {
-  if (input.status !== "finished") return;
-  const path = sessionSummaryPath(input.cwd, input.runId);
-  await mkdir(sessionsDir(input.cwd), { recursive: true });
-  const safeUser = redactSecrets(truncate2(input.userText));
-  const safeAssistant = redactSecrets(truncate2(input.assistantText));
-  const iso = new Date(input.at).toISOString();
-  const body = [
-    "---",
-    `runId: ${input.runId}`,
-    `agentId: ${input.agentId}`,
-    `at: ${iso}`,
-    `status: ${input.status}`,
-    "---",
-    "",
-    "## User",
-    "",
-    safeUser,
-    "",
-    "## Assistant",
-    "",
-    safeAssistant,
-    ""
-  ].join("\n");
-  await replaceFileAtomic(path, body);
-}
-
-// src/internal/store/session-loader.ts
-async function discoverSessionFiles(cwd) {
-  let entries;
-  try {
-    entries = await readdir(sessionsDir(cwd));
-  } catch {
-    return [];
-  }
-  const root = memoryDir(cwd);
-  return entries.filter((entry) => entry.endsWith(".md")).map((entry) => {
-    const absolutePath = join(sessionsDir(cwd), entry);
-    return {
-      absolutePath,
-      relPath: relativeToRoot(root, absolutePath)
-    };
-  });
-}
-function relativeToRoot(root, absolutePath) {
-  if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
-  return absolutePath;
-}
-function wikiDir(cwd) {
-  return join(memoryDir(cwd), "wiki");
-}
-async function discoverWikiFiles(cwd) {
-  let entries;
-  try {
-    entries = await readdir(wikiDir(cwd));
-  } catch {
-    return [];
-  }
-  const root = memoryDir(cwd);
-  return entries.filter((entry) => entry.endsWith(".md")).map((entry) => ({
-    absolutePath: join(wikiDir(cwd), entry),
-    relPath: join("wiki", entry)
-  })).map((file) => ({
-    absolutePath: file.absolutePath,
-    relPath: relativeToRoot2(root, file.absolutePath)
-  }));
-}
-function relativeToRoot2(root, absolutePath) {
-  if (absolutePath.startsWith(`${root}/`)) return absolutePath.slice(root.length + 1);
-  return absolutePath;
-}
 function requireLance() {
   try {
     const r = createRequire(import.meta.url);
@@ -1523,7 +1384,7 @@ var LanceIndex = class _LanceIndex {
   embeddingDim;
   static async open(opts) {
     const lance = requireLance();
-    const storagePath = opts.storagePath ?? join(opts.cwd, ".theokit", "memory", "lance");
+    const storagePath = opts.storagePath ?? lanceStoragePath(opts.memoryRoot ?? resolveMemoryRoot(opts.cwd));
     mkdirSync(storagePath, { recursive: true });
     const conn = await lance.connect(storagePath);
     const dim = opts.embedding.dimension;
@@ -1625,9 +1486,6 @@ function isLanceAvailable() {
     return false;
   }
 }
-function lanceStoragePath(cwd) {
-  return join(cwd, ".theokit", "memory", "lance");
-}
 
 // src/internal/index/memory-index.ts
 function parseSearchOptions(options = {}) {
@@ -1827,12 +1685,12 @@ async function embedMissingChunks(args) {
 
 // src/internal/index/index-manager.ts
 var IndexManager = class _IndexManager {
-  constructor(cwd, db, embedding) {
-    this.cwd = cwd;
+  constructor(memoryRoot, db, embedding) {
+    this.memoryRoot = memoryRoot;
     this.db = db;
     this.embedding = embedding;
   }
-  cwd;
+  memoryRoot;
   db;
   embedding;
   lastSyncMs;
@@ -1845,9 +1703,10 @@ var IndexManager = class _IndexManager {
   }
   /** Internal SQLite-path open. Renamed from previous public `open`. */
   static async openSqliteInternal(opts) {
-    const filePath = opts.filePath ?? defaultIndexPath(opts.cwd);
+    const memoryRoot = opts.memoryRoot ?? resolveMemoryRoot(opts.cwd);
+    const filePath = opts.filePath ?? defaultIndexPath(projectMemoryDir(opts.cwd));
     const db = await openMemoryDb({ filePath });
-    const manager = new _IndexManager(opts.cwd, db, opts.embedding);
+    const manager = new _IndexManager(memoryRoot, db, opts.embedding);
     if (opts.embedding !== void 0) await manager.initVectorBackend(opts.embedding);
     return manager;
   }
@@ -1868,7 +1727,7 @@ var IndexManager = class _IndexManager {
   }
   /** Walk the memory corpus + (re)index changed files. */
   async sync() {
-    const files = await collectMarkdownFiles(this.cwd);
+    const files = await collectMarkdownFiles(this.memoryRoot);
     let filesUpdated = 0;
     let chunksWritten = 0;
     const existingByPath = this.loadFilesIndex();
@@ -2065,45 +1924,6 @@ function blendScores(hit, vectorScore, weights) {
     ...vectorScore > 0 ? { vectorScore } : {}
   };
 }
-async function collectMarkdownFiles(cwd) {
-  const root = memoryDir(cwd);
-  const results = [];
-  try {
-    await stat(memoryMdPath(cwd));
-    results.push({
-      absolutePath: memoryMdPath(cwd),
-      relPath: relative(root, memoryMdPath(cwd)),
-      source: "memory"
-    });
-  } catch {
-  }
-  try {
-    const entries = await readdir(notesDir(cwd));
-    for (const entry of entries) {
-      if (!entry.endsWith(".md")) continue;
-      const abs = join(notesDir(cwd), entry);
-      results.push({ absolutePath: abs, relPath: relative(root, abs), source: "memory" });
-    }
-  } catch {
-  }
-  const wikiFiles = await discoverWikiFiles(cwd);
-  for (const wiki of wikiFiles) {
-    results.push({
-      absolutePath: wiki.absolutePath,
-      relPath: wiki.relPath,
-      source: "wiki"
-    });
-  }
-  const sessionFiles = await discoverSessionFiles(cwd);
-  for (const session of sessionFiles) {
-    results.push({
-      absolutePath: session.absolutePath,
-      relPath: session.relPath,
-      source: "sessions"
-    });
-  }
-  return results;
-}
 function sha256(text) {
   return createHash("sha256").update(text).digest("hex");
 }
@@ -2111,30 +1931,14 @@ function truncateSnippet(text) {
   const max = 500;
   return text.length <= max ? text : `${text.slice(0, max)}\u2026`;
 }
-async function readAllSqliteFacts(cwd) {
-  const dbPath = defaultIndexPath(cwd);
-  if (!existsSync(dbPath)) return [];
-  const db = await openMemoryDb({ filePath: dbPath });
-  try {
-    const stmt = db.prepare("SELECT id, path, source, start_line, end_line, text FROM chunks");
-    const rows = stmt.all();
-    return rows.map((r) => ({
-      ...r,
-      namespace: "default",
-      scope: "agent",
-      user_id: "default"
-    }));
-  } finally {
-    db.close();
-  }
-}
 function nfcEqual(a, b) {
   return a.normalize("NFC") === b.normalize("NFC");
 }
 async function migrateSqliteToLance(opts) {
   const cwd = opts.cwd;
-  const finalPath = lanceStoragePath(cwd);
-  const newPath = join(cwd, ".theokit", "memory", "lance-new");
+  const memoryRoot = resolveMemoryRoot(cwd, { directory: opts.directory });
+  const finalPath = lanceStoragePath(memoryRoot);
+  const newPath = join(memoryRoot, "lance-new");
   const rawLog = opts.logger ?? ((m) => console.log(m));
   const log = (m) => rawLog(redactSecrets(m));
   if (existsSync(finalPath)) {
@@ -2148,7 +1952,7 @@ async function migrateSqliteToLance(opts) {
     rmSync(newPath, { recursive: true, force: true });
   }
   log(`Reading SQLite facts from ${cwd}/.theokit/memory/index.sqlite ...`);
-  const sqliteFacts = await readAllSqliteFacts(cwd);
+  const sqliteFacts = await readAllSqliteFacts(memoryRoot);
   log(`SQLite has ${sqliteFacts.length} facts.`);
   if (sqliteFacts.length === 0) {
     return {
@@ -2292,7 +2096,7 @@ async function migrateLegacyJson(cwd, config) {
   if (!await fileExists(jsonPath)) {
     return { migrated: false, factCount: 0, reason: "no-legacy-json" };
   }
-  if (await fileExists(memoryMdPath(cwd))) {
+  if (await fileExists(memoryMdPath(resolveMemoryRoot(cwd)))) {
     process.stderr.write(
       `[theokit-sdk] memory migration skipped: both MEMORY.md and legacy JSON exist at ${jsonPath}; leaving both intact
 `
@@ -2378,7 +2182,7 @@ function createMemorySearchTool(opts) {
   };
 }
 function createMemoryGetTool(opts) {
-  const memoryRoot = resolve(memoryDir(opts.cwd));
+  const memoryRoot = resolve(opts.root);
   return {
     name: "memory_get",
     description: GET_DESCRIPTION,
@@ -2447,6 +2251,6 @@ function isPathInside(root, candidate) {
   return candidate === root || candidate.startsWith(normalizedRoot);
 }
 
-export { ActiveMemoryCache, CircuitBreaker, DEFAULT_AZURE_OPENAI_EMBEDDING_MODEL, DEFAULT_COHERE_EMBEDDING_MODEL, DEFAULT_DEEPINFRA_EMBEDDING_MODEL, DEFAULT_GEMINI_EMBEDDING_MODEL, DEFAULT_JINA_EMBEDDING_MODEL, DEFAULT_MEMORY_READ_LINES, DEFAULT_MISTRAL_EMBEDDING_MODEL, DEFAULT_OLLAMA_EMBEDDING_MODEL, DEFAULT_OPENAI_EMBEDDING_MODEL, DEFAULT_OPENROUTER_EMBEDDING_MODEL, DEFAULT_VOYAGE_EMBEDDING_MODEL, IndexManager, LanceIndex, LanceMemoryAdapter, MEMORY_EMBEDDING_ADAPTERS, META_KEY_DIMENSION, META_KEY_MODEL, META_KEY_PROVIDER_ID, PRAGMA_STATEMENTS, SCHEMA_STATEMENTS, VALID_BACKENDS, appendDiaryEntry, assertValidBackend, azureOpenAiMemoryEmbeddingProviderAdapter, buildErrorMetadata, chunkMarkdown, cohereMemoryEmbeddingProviderAdapter, createCategorizedMemory, createInMemoryMarkdownProvider, createMemoryGetTool, createMemorySearchTool, createVectorIndex, deepPhase, deepinfraMemoryEmbeddingProviderAdapter, defaultIndexPath, diaryPath, discoverSessionFiles, discoverWikiFiles, dropVectorIndex, embedMissingChunks, entryHash, geminiMemoryEmbeddingProviderAdapter, identityMatches, isLanceAvailable, isSqliteVecLoaded, jinaMemoryEmbeddingProviderAdapter, lanceStoragePath, legacyMemoryJsonPath, lightPhase, listNotes, loadSqliteVecExtension, mapOpenAICompatibleError, migrateLegacyJson, migrateSqliteToLance, mistralMemoryEmbeddingProviderAdapter, ollamaMemoryEmbeddingProviderAdapter, openAiMemoryEmbeddingProviderAdapter, openLanceIndex, openMemoryDb, openRouterMemoryEmbeddingProviderAdapter, packVector, parseRetryAfter, parseSearchOptions, persistActiveMemoryTranscript, readEmbeddingIdentity, readMemoryFileBounded, redactSecrets, remPhase, renderDiaryEntry, resetMigrationStateForTests, runActiveMemory, runDreamingSweep, sessionSummaryPath, sessionsDir, truncateRaw, upsertEmbedding, vectorSearch, voyageMemoryEmbeddingProviderAdapter, wikiDir, writeEmbeddingIdentity, writeSessionSummary };
+export { ActiveMemoryCache, CircuitBreaker, DEFAULT_AZURE_OPENAI_EMBEDDING_MODEL, DEFAULT_COHERE_EMBEDDING_MODEL, DEFAULT_DEEPINFRA_EMBEDDING_MODEL, DEFAULT_GEMINI_EMBEDDING_MODEL, DEFAULT_JINA_EMBEDDING_MODEL, DEFAULT_MEMORY_READ_LINES, DEFAULT_MISTRAL_EMBEDDING_MODEL, DEFAULT_OLLAMA_EMBEDDING_MODEL, DEFAULT_OPENAI_EMBEDDING_MODEL, DEFAULT_OPENROUTER_EMBEDDING_MODEL, DEFAULT_VOYAGE_EMBEDDING_MODEL, IndexManager, LanceIndex, LanceMemoryAdapter, MEMORY_EMBEDDING_ADAPTERS, META_KEY_DIMENSION, META_KEY_MODEL, META_KEY_PROVIDER_ID, PRAGMA_STATEMENTS, SCHEMA_STATEMENTS, VALID_BACKENDS, assertValidBackend, azureOpenAiMemoryEmbeddingProviderAdapter, buildErrorMetadata, chunkMarkdown, cohereMemoryEmbeddingProviderAdapter, createCategorizedMemory, createInMemoryMarkdownProvider, createMemoryGetTool, createMemorySearchTool, createVectorIndex, deepPhase, deepinfraMemoryEmbeddingProviderAdapter, dropVectorIndex, embedMissingChunks, geminiMemoryEmbeddingProviderAdapter, identityMatches, isLanceAvailable, isSqliteVecLoaded, jinaMemoryEmbeddingProviderAdapter, legacyMemoryJsonPath, lightPhase, listNotes, loadSqliteVecExtension, mapOpenAICompatibleError, migrateLegacyJson, migrateSqliteToLance, mistralMemoryEmbeddingProviderAdapter, ollamaMemoryEmbeddingProviderAdapter, openAiMemoryEmbeddingProviderAdapter, openLanceIndex, openMemoryDb, openRouterMemoryEmbeddingProviderAdapter, packVector, parseRetryAfter, parseSearchOptions, readEmbeddingIdentity, readMemoryFileBounded, redactSecrets, remPhase, resetMigrationStateForTests, runActiveMemory, runDreamingSweep, truncateRaw, upsertEmbedding, vectorSearch, voyageMemoryEmbeddingProviderAdapter, writeEmbeddingIdentity };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map