@theokit/sdk 4.63.4 → 5.0.0-next.2

package/dist/{chunk-L3MTL4NA.js → chunk-2ZLVT7H3.js}

@@ -1,10 +1,114 @@
 import { LruEmbeddingCache } from './chunk-H73MEMQB.js';
-import { resolveMemoryRoot, defaultIndexPath, projectMemoryDir, openMemoryDb, collectMarkdownFiles, sha256, truncateSnippet, resolveWeights, blendScores, LanceIndex } from './chunk-L4BDVXIM.js';
-import { sanitizeFts5Query } from './chunk-WE22OXQA.js';
-import { ConfigurationError } from './chunk-ZZMZWBR6.js';
+import { resolveMemoryRoot, defaultIndexPath, projectMemoryDir, openMemoryDb, truncateSnippet, resolveWeights, blendScores, LanceIndex, collectMarkdownFiles, sha256 } from './chunk-UNK4HNJH.js';
+import { sanitizeFts5Query, containsCjk } from './chunk-WE22OXQA.js';
+import { ConfigurationError } from './chunk-ALUN2B4W.js';
+import { diag } from './chunk-CZJ6Q7CW.js';
 import { createHash } from 'crypto';
 import { readFile, stat } from 'fs/promises';
 
+var HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
+function chunkMarkdown(text, options = {}) {
+  const maxChars = options.maxChars ?? 800;
+  if (text.length === 0) return [];
+  const lines = text.split("\n");
+  const acc = { chunks: [], lines: [], start: 1, heading: void 0 };
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i] ?? "";
+    const lineNumber = i + 1;
+    processLine(acc, line, lineNumber, maxChars);
+  }
+  flushAccumulator(acc, lines.length, maxChars);
+  return acc.chunks;
+}
+function processLine(acc, line, lineNumber, maxChars) {
+  const headingMatch = HEADING_RE.exec(line);
+  const isBlank = line.trim().length === 0;
+  if (headingMatch !== null && acc.lines.length > 0) {
+    flushAccumulator(acc, lineNumber - 1, maxChars);
+  }
+  if (headingMatch !== null) acc.heading = headingMatch[2];
+  acc.lines.push(line);
+  if (isBlank && acc.lines.length > 1) {
+    const joined = acc.lines.join("\n");
+    if (joined.trim().length > 0) flushAccumulator(acc, lineNumber, maxChars);
+  }
+}
+function flushAccumulator(acc, endLineExclusive, maxChars) {
+  if (acc.lines.length === 0) return;
+  const chunkText = acc.lines.join("\n");
+  if (chunkText.trim().length === 0) {
+    acc.lines = [];
+    acc.start = endLineExclusive + 1;
+    return;
+  }
+  if (chunkText.length > maxChars) {
+    pushOversizedSlices(acc.chunks, chunkText, acc.start, acc.heading, maxChars);
+  } else {
+    acc.chunks.push(buildChunk(chunkText, acc.start, endLineExclusive, acc.heading));
+  }
+  acc.lines = [];
+  acc.start = endLineExclusive + 1;
+}
+function buildChunk(text, startLine, endLine, heading) {
+  return {
+    startLine,
+    endLine,
+    text,
+    hash: createHash("sha256").update(text).digest("hex"),
+    ...heading !== void 0 ? { heading } : {}
+  };
+}
+function pushOversizedSlices(chunks, text, startLine, heading, maxChars) {
+  let remaining = text;
+  let lineCursor = startLine;
+  while (remaining.length > maxChars) {
+    const splitAt = findWordBoundarySplit(remaining, maxChars);
+    const slice = remaining.slice(0, splitAt);
+    const sliceLines = slice.split("\n").length;
+    chunks.push(buildChunk(slice, lineCursor, lineCursor + sliceLines - 1, heading));
+    remaining = remaining.slice(splitAt).replace(/^\s+/, "");
+    lineCursor += sliceLines - 1;
+  }
+  if (remaining.length > 0) {
+    const sliceLines = remaining.split("\n").length;
+    chunks.push(buildChunk(remaining, lineCursor, lineCursor + sliceLines - 1, heading));
+  }
+}
+function findWordBoundarySplit(text, maxChars) {
+  if (text.length <= maxChars) return text.length;
+  for (let i = maxChars; i > Math.max(0, maxChars - 200); i--) {
+    if (/\s/.test(text[i] ?? "x")) return i;
+  }
+  return maxChars;
+}
+
+// src/internal/memory/corpus-sync.ts
+async function syncCorpus(repo, memoryRoot) {
+  const files = await collectMarkdownFiles(memoryRoot);
+  const existingByPath = repo.filesByPath();
+  let filesUpdated = 0;
+  let chunksWritten = 0;
+  for (const entry of files) {
+    const raw = await readFile(entry.absolutePath, "utf8");
+    const hash = sha256(raw);
+    if (existingByPath.get(entry.absolutePath)?.hash === hash) continue;
+    const stats = await stat(entry.absolutePath);
+    const fileId = repo.upsertFile({
+      absPath: entry.absolutePath,
+      relPath: entry.relPath,
+      hash,
+      mtimeMs: stats.mtimeMs,
+      source: entry.source
+    });
+    repo.deleteChunksForFile(fileId);
+    const chunks = chunkMarkdown(raw);
+    for (const chunk of chunks) repo.insertChunk({ fileId, ...chunk });
+    filesUpdated += 1;
+    chunksWritten += chunks.length;
+  }
+  return { filesScanned: files.length, filesUpdated, chunksWritten };
+}
+
 // src/internal/memory/escape-like-pattern.ts
 function escapeLikePattern(query) {
   return `%${query.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_")}%`;
@@ -20,11 +124,12 @@ function parseSearchOptions(options = {}) {
 
 // src/internal/memory/lance-memory-adapter.ts
 var DEFAULT_NAMESPACE = "default";
-var EMPTY_SYNC_RESULT = Object.freeze({
+var UNSUPPORTED_SYNC_RESULT = Object.freeze({
   filesScanned: 0,
   filesUpdated: 0,
   chunksWritten: 0,
-  chunksEmbedded: 0
+  chunksEmbedded: 0,
+  supported: false
 });
 var LanceMemoryAdapter = class {
   constructor(inner) {
@@ -32,11 +137,11 @@ var LanceMemoryAdapter = class {
   }
   inner;
   /**
-   * No-op for Lance — see file header. Returns zero counts so callers'
-   * existing logging (`filesScanned: X`) does not break.
+   * Not applicable to Lance — see file header. Returns zero counts with `supported: false` so a
+   * caller can distinguish this from a real sync that found nothing.
    */
   async sync() {
-    return EMPTY_SYNC_RESULT;
+    return UNSUPPORTED_SYNC_RESULT;
   }
   // jscpd:ignore-start — search signature + early-out is idiomatic
   // overlap with IndexManager.search; further factoring would hurt
@@ -49,30 +154,24 @@ var LanceMemoryAdapter = class {
       limit: maxResults * 2,
       ...options.sources !== void 0 ? { sources: options.sources } : {}
     });
-    return lanceHits.filter((h) => h.score >= minScore).slice(0, maxResults).map((h) => translateLanceHit(h, query));
+    return lanceHits.filter((h) => h.score >= minScore).slice(0, maxResults).map((h) => translateLanceHit(h, query, resolveWeights(options)));
   }
   status() {
     return {
       backend: "hybrid",
       filesIndexed: 0,
-      chunksIndexed: 0
+      chunksIndexed: 0,
+      countsExact: false
     };
   }
   async close() {
     await this.inner.close();
   }
-  /**
-   * Escape hatch for advanced callers (migration tool, benchmark script)
-   * that need direct access to addFacts/countFacts/removeFacts.
-   */
-  unwrap() {
-    return this.inner;
-  }
 };
-function translateLanceHit(hit, query) {
+function translateLanceHit(hit, query, weights) {
   const textScore = query !== void 0 ? computeTermOverlapScore(query, hit.text) : 0;
   const vectorScore = hit.score;
-  const combined = 0.7 * vectorScore + 0.3 * textScore;
+  const combined = (vectorScore * weights.vectorWeight + textScore * weights.textWeight) / weights.total;
   return {
     path: hit.id,
     startLine: 0,
@@ -118,6 +217,85 @@ async function openLanceIndex(opts) {
   return new LanceMemoryAdapter(lance);
 }
 
+// src/internal/memory/index-repository.ts
+var MemoryIndexRepository = class {
+  constructor(db) {
+    this.db = db;
+  }
+  db;
+  counts() {
+    const files = this.db.prepare("SELECT COUNT(*) as n FROM files").get() ?? { n: 0 };
+    const chunks = this.db.prepare("SELECT COUNT(*) as n FROM chunks").get() ?? { n: 0 };
+    return { files: Number(files.n ?? 0), chunks: Number(chunks.n ?? 0) };
+  }
+  /**
+   * FTS5 match, ordered by bm25. THROWS on an FTS5 error rather than swallowing it — whether that
+   * error means "CJK input the default tokenizer cannot segment" or "the table is missing" is a
+   * judgement the caller makes, and it makes it differently for each.
+   */
+  ftsRows(sanitizedQuery, limit) {
+    return this.db.prepare(
+      `SELECT chunks.id as id, files.rel_path as rel_path, files.source as source,
+                chunks.start_line as start_line, chunks.end_line as end_line,
+                chunks.text as text, bm25(chunks_fts) as bm25_score
+         FROM chunks_fts
+         JOIN chunks ON chunks_fts.rowid = chunks.id
+         JOIN files  ON chunks.file_id = files.id
+         WHERE chunks_fts MATCH ?
+         ORDER BY bm25_score
+         LIMIT ?`
+    ).all(sanitizedQuery, limit);
+  }
+  /** Substring scan — the CJK fallback. No relevance score exists here; the caller supplies one. */
+  likeRows(escapedPattern, limit) {
+    return this.db.prepare(
+      `SELECT chunks.id as id, files.rel_path as rel_path, files.source as source,
+                chunks.start_line as start_line, chunks.end_line as end_line,
+                chunks.text as text
+         FROM chunks
+         JOIN files ON chunks.file_id = files.id
+         WHERE chunks.text LIKE ? ESCAPE '\\'
+         LIMIT ?`
+    ).all(escapedPattern, limit);
+  }
+  /** Hydrate the chunks a vector search returned by id. Empty in, empty out — no query runs. */
+  chunkRowsByIds(ids) {
+    if (ids.length === 0) return [];
+    const placeholders = ids.map(() => "?").join(",");
+    return this.db.prepare(
+      `SELECT chunks.id as id, files.rel_path as rel_path, files.source as source,
+                chunks.start_line as start_line, chunks.end_line as end_line,
+                chunks.text as text
+         FROM chunks JOIN files ON chunks.file_id = files.id
+         WHERE chunks.id IN (${placeholders})`
+    ).all(...ids);
+  }
+  /** Absolute path → its row, for the corpus walk to compare hashes against disk. */
+  filesByPath() {
+    const rows = this.db.prepare("SELECT id, path, hash FROM files").all();
+    return new Map(rows.map((row) => [row.path, { id: row.id, hash: row.hash }]));
+  }
+  upsertFile(file) {
+    const row = this.db.prepare(
+      `INSERT INTO files (path, rel_path, mtime, hash, source) VALUES (?, ?, ?, ?, ?)
+         ON CONFLICT(path) DO UPDATE SET hash = excluded.hash, mtime = excluded.mtime, source = excluded.source
+         RETURNING id`
+    ).get(file.absPath, file.relPath, Math.floor(file.mtimeMs), file.hash, file.source);
+    return row.id;
+  }
+  deleteChunksForFile(fileId) {
+    this.db.prepare("DELETE FROM chunks WHERE file_id = ?").run(fileId);
+  }
+  insertChunk(chunk) {
+    this.db.prepare(
+      "INSERT INTO chunks (file_id, start_line, end_line, text, hash) VALUES (?, ?, ?, ?, ?)"
+    ).run(chunk.fileId, chunk.startLine, chunk.endLine, chunk.text, chunk.hash);
+  }
+  close() {
+    this.db.close();
+  }
+};
+
 // src/internal/memory/sqlite-vec-loader.ts
 async function loadSqliteVecExtension(db) {
   try {
@@ -135,81 +313,6 @@ async function loadSqliteVecExtension(db) {
     );
   }
 }
-var HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
-function chunkMarkdown(text, options = {}) {
-  const maxChars = options.maxChars ?? 800;
-  if (text.length === 0) return [];
-  const lines = text.split("\n");
-  const acc = { chunks: [], lines: [], start: 1, heading: void 0 };
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i] ?? "";
-    const lineNumber = i + 1;
-    processLine(acc, line, lineNumber, maxChars);
-  }
-  flushAccumulator(acc, lines.length, maxChars);
-  return acc.chunks;
-}
-function processLine(acc, line, lineNumber, maxChars) {
-  const headingMatch = HEADING_RE.exec(line);
-  const isBlank = line.trim().length === 0;
-  if (headingMatch !== null && acc.lines.length > 0) {
-    flushAccumulator(acc, lineNumber - 1, maxChars);
-  }
-  if (headingMatch !== null) acc.heading = headingMatch[2];
-  acc.lines.push(line);
-  if (isBlank && acc.lines.length > 1) {
-    const joined = acc.lines.join("\n");
-    if (joined.trim().length > 0) flushAccumulator(acc, lineNumber, maxChars);
-  }
-}
-function flushAccumulator(acc, endLineExclusive, maxChars) {
-  if (acc.lines.length === 0) return;
-  const chunkText = acc.lines.join("\n");
-  if (chunkText.trim().length === 0) {
-    acc.lines = [];
-    acc.start = endLineExclusive + 1;
-    return;
-  }
-  if (chunkText.length > maxChars) {
-    pushOversizedSlices(acc.chunks, chunkText, acc.start, acc.heading, maxChars);
-  } else {
-    acc.chunks.push(buildChunk(chunkText, acc.start, endLineExclusive, acc.heading));
-  }
-  acc.lines = [];
-  acc.start = endLineExclusive + 1;
-}
-function buildChunk(text, startLine, endLine, heading) {
-  return {
-    startLine,
-    endLine,
-    text,
-    hash: createHash("sha256").update(text).digest("hex"),
-    ...heading !== void 0 ? { heading } : {}
-  };
-}
-function pushOversizedSlices(chunks, text, startLine, heading, maxChars) {
-  let remaining = text;
-  let lineCursor = startLine;
-  while (remaining.length > maxChars) {
-    const splitAt = findWordBoundarySplit(remaining, maxChars);
-    const slice = remaining.slice(0, splitAt);
-    const sliceLines = slice.split("\n").length;
-    chunks.push(buildChunk(slice, lineCursor, lineCursor + sliceLines - 1, heading));
-    remaining = remaining.slice(splitAt).replace(/^\s+/, "");
-    lineCursor += sliceLines - 1;
-  }
-  if (remaining.length > 0) {
-    const sliceLines = remaining.split("\n").length;
-    chunks.push(buildChunk(remaining, lineCursor, lineCursor + sliceLines - 1, heading));
-  }
-}
-function findWordBoundarySplit(text, maxChars) {
-  if (text.length <= maxChars) return text.length;
-  for (let i = maxChars; i > Math.max(0, maxChars - 200); i--) {
-    if (/\s/.test(text[i] ?? "x")) return i;
-  }
-  return maxChars;
-}
 
 // src/internal/memory/vec-index.ts
 var META_KEY_PROVIDER_ID = "embedding.providerId";
@@ -292,12 +395,15 @@ var IndexManager = class _IndexManager {
     this.memoryRoot = memoryRoot;
     this.db = db;
     this.embedding = embedding;
+    this.repo = new MemoryIndexRepository(db);
   }
   memoryRoot;
   db;
   embedding;
   lastSyncMs;
   vectorReady = false;
+  /** Every SQL string this class used to hold. See `index-repository.ts` for why it moved. */
+  repo;
   static async open(opts) {
     const backend = opts.backend ?? "sqlite-vec";
     assertValidBackend(backend);
@@ -329,38 +435,18 @@ var IndexManager = class _IndexManager {
     this.vectorReady = true;
   }
   /** Walk the memory corpus + (re)index changed files. */
+  // Declares `Promise<SyncResult>` rather than repeating the shape inline. The inline version was a
+  // structural copy of the interface it implements, so widening `SyncResult` with `supported` left
+  // this method silently no longer assignable to `MemoryIndex` — the compiler caught it, which is
+  // the argument for naming the type instead of restating it.
   async sync() {
-    const files = await collectMarkdownFiles(this.memoryRoot);
-    let filesUpdated = 0;
-    let chunksWritten = 0;
-    const existingByPath = this.loadFilesIndex();
-    for (const entry of files) {
-      const raw = await readFile(entry.absolutePath, "utf8");
-      const hash = sha256(raw);
-      const existing = existingByPath.get(entry.absolutePath);
-      if (existing !== void 0 && existing.hash === hash) continue;
-      const stats = await stat(entry.absolutePath);
-      const fileId = this.upsertFile(
-        entry.absolutePath,
-        entry.relPath,
-        hash,
-        stats.mtimeMs,
-        entry.source
-      );
-      this.deleteChunksForFile(fileId);
-      const chunks = chunkMarkdown(raw);
-      for (const chunk of chunks) {
-        this.insertChunk(fileId, chunk.startLine, chunk.endLine, chunk.text, chunk.hash);
-      }
-      filesUpdated += 1;
-      chunksWritten += chunks.length;
-    }
+    const counts = await syncCorpus(this.repo, this.memoryRoot);
     let chunksEmbedded = 0;
     if (this.vectorReady && this.embedding !== void 0) {
       chunksEmbedded = await embedMissingChunks({ db: this.db, runtime: this.embedding });
     }
     this.lastSyncMs = Date.now();
-    return { filesScanned: files.length, filesUpdated, chunksWritten, chunksEmbedded };
+    return { ...counts, chunksEmbedded, supported: true };
   }
   async search(query, options = {}) {
     if (query.trim().length === 0) return [];
@@ -371,12 +457,13 @@ var IndexManager = class _IndexManager {
     return combined.filter((h) => h.score >= minScore).filter((h) => options.sources === void 0 || options.sources.includes(h.source)).slice(0, maxResults);
   }
   status() {
-    const files = this.db.prepare("SELECT COUNT(*) as n FROM files").get() ?? { n: 0 };
-    const chunks = this.db.prepare("SELECT COUNT(*) as n FROM chunks").get() ?? { n: 0 };
+    const { files, chunks } = this.repo.counts();
     const status = {
       backend: this.vectorReady ? "hybrid" : "fts-only",
-      filesIndexed: Number(files.n ?? 0),
-      chunksIndexed: Number(chunks.n ?? 0)
+      filesIndexed: files,
+      chunksIndexed: chunks,
+      // Counted with SELECT COUNT(*) just above — these are measurements, not placeholders.
+      countsExact: true
     };
     if (this.lastSyncMs !== void 0) status.lastSyncMs = this.lastSyncMs;
     return status;
@@ -385,21 +472,15 @@ var IndexManager = class _IndexManager {
   ftsSearch(query, limit) {
     const sanitized = sanitizeFts5Query(query);
     if (sanitized.length === 0) return [];
-    const stmt = this.db.prepare(
-      `SELECT chunks.id as id, files.rel_path as rel_path, files.source as source,
-              chunks.start_line as start_line, chunks.end_line as end_line,
-              chunks.text as text, bm25(chunks_fts) as bm25_score
-       FROM chunks_fts
-       JOIN chunks ON chunks_fts.rowid = chunks.id
-       JOIN files  ON chunks.file_id = files.id
-       WHERE chunks_fts MATCH ?
-       ORDER BY bm25_score
-       LIMIT ?`
-    );
     let rows = [];
     try {
-      rows = stmt.all(sanitized, limit);
-    } catch {
+      rows = this.repo.ftsRows(sanitized, limit);
+    } catch (err) {
+      if (!containsCjk(query)) {
+        diag(
+          `memory FTS query failed for a non-CJK query and fell back to a LIKE scan \u2014 relevance is degraded and the index may be missing or corrupt: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
       return this.likeSearchFallback(query, limit);
     }
     return rows.map((row) => {
@@ -430,17 +511,7 @@ var IndexManager = class _IndexManager {
    * `ftsSearch` so callers are unaware of the fallback.
    */
   likeSearchFallback(query, limit) {
-    const pattern = escapeLikePattern(query);
-    const stmt = this.db.prepare(
-      `SELECT chunks.id as id, files.rel_path as rel_path, files.source as source,
-              chunks.start_line as start_line, chunks.end_line as end_line,
-              chunks.text as text
-       FROM chunks
-       JOIN files ON chunks.file_id = files.id
-       WHERE chunks.text LIKE ? ESCAPE '\\'
-       LIMIT ?`
-    );
-    const rows = stmt.all(pattern, limit);
+    const rows = this.repo.likeRows(escapeLikePattern(query), limit);
     return rows.map((row) => {
       const path = String(row.rel_path ?? "");
       const startLine = Number(row.start_line ?? 0);
@@ -494,15 +565,7 @@ var IndexManager = class _IndexManager {
     return merged;
   }
   fetchChunksByIds(ids) {
-    if (ids.length === 0) return [];
-    const placeholders = ids.map(() => "?").join(",");
-    const rows = this.db.prepare(
-      `SELECT chunks.id as id, files.rel_path as rel_path, files.source as source,
-                chunks.start_line as start_line, chunks.end_line as end_line,
-                chunks.text as text
-         FROM chunks JOIN files ON chunks.file_id = files.id
-         WHERE chunks.id IN (${placeholders})`
-    ).all(...ids);
+    const rows = this.repo.chunkRowsByIds(ids);
     return rows.map((row) => {
       const startLine = Number(row.start_line ?? 0);
       const endLine = Number(row.end_line ?? 0);
@@ -520,33 +583,11 @@ var IndexManager = class _IndexManager {
       };
     });
   }
-  // ───── persistence helpers ─────────────────────────────────────────
-  loadFilesIndex() {
-    const rows = this.db.prepare("SELECT id, path, hash FROM files").all();
-    return new Map(rows.map((row) => [row.path, { id: row.id, hash: row.hash }]));
-  }
-  upsertFile(absPath, relPath, hash, mtimeMs, source = "memory") {
-    const stmt = this.db.prepare(
-      `INSERT INTO files (path, rel_path, mtime, hash, source) VALUES (?, ?, ?, ?, ?)
-       ON CONFLICT(path) DO UPDATE SET hash = excluded.hash, mtime = excluded.mtime, source = excluded.source
-       RETURNING id`
-    );
-    const row = stmt.get(absPath, relPath, Math.floor(mtimeMs), hash, source);
-    return row.id;
-  }
-  deleteChunksForFile(fileId) {
-    this.db.prepare("DELETE FROM chunks WHERE file_id = ?").run(fileId);
-  }
-  insertChunk(fileId, startLine, endLine, text, hash) {
-    this.db.prepare(
-      "INSERT INTO chunks (file_id, start_line, end_line, text, hash) VALUES (?, ?, ?, ?, ?)"
-    ).run(fileId, startLine, endLine, text, hash);
-  }
   close() {
-    this.db.close();
+    this.repo.close();
   }
 };
 
 export { IndexManager };
-//# sourceMappingURL=chunk-L3MTL4NA.js.map
-//# sourceMappingURL=chunk-L3MTL4NA.js.map
+//# sourceMappingURL=chunk-2ZLVT7H3.js.map
+//# sourceMappingURL=chunk-2ZLVT7H3.js.map