kontext-engine 0.1.5 → 0.1.6

package/dist/cli/index.js

@@ -1158,6 +1158,8 @@ function searchVectors(db, query, limit) {
 // src/storage/db.ts
 var DEFAULT_DIMENSIONS = 384;
 var VECTOR_DIMENSIONS_META_KEY = "vector_dimensions";
+var INDEX_EMBEDDER_PROVIDER_META_KEY = "index_embedder_provider";
+var INDEX_EMBEDDER_MODEL_META_KEY = "index_embedder_model";
 function createDatabase(dbPath, dimensions) {
   const dir = path3.dirname(dbPath);
   if (!fs4.existsSync(dir)) {
@@ -1190,6 +1192,16 @@ function createDatabase(dbPath, dimensions) {
   const stmtGetChunksByFile = db.prepare(
     "SELECT id, file_id as fileId, line_start as lineStart, line_end as lineEnd, type, name, parent, text, imports, exports, hash FROM chunks WHERE file_id = ? ORDER BY line_start"
   );
+  const stmtGetChunksMissingVectors = db.prepare(
+    `SELECT c.id, c.file_id as fileId, f.path as filePath, f.language,
+            c.line_start as lineStart, c.line_end as lineEnd,
+            c.type, c.name, c.parent, c.text, c.exports as exports
+     FROM chunks c
+     JOIN files f ON f.id = c.file_id
+     LEFT JOIN chunk_vectors v ON v.rowid = c.id
+     WHERE v.rowid IS NULL
+     ORDER BY c.id`
+  );
   const stmtGetChunkIdsByFile = db.prepare(
     "SELECT id FROM chunks WHERE file_id = ?"
   );
@@ -1219,6 +1231,10 @@ function createDatabase(dbPath, dimensions) {
   const stmtLastIndexed = db.prepare(
     "SELECT MAX(last_indexed) as lastIndexed FROM files"
   );
+  const stmtGetMeta = db.prepare("SELECT value FROM meta WHERE key = ?");
+  const stmtSetMeta = db.prepare(
+    "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)"
+  );
   return {
     upsertFile(file) {
       const row = stmtUpsertFile.get({
@@ -1332,6 +1348,13 @@ function createDatabase(dbPath, dimensions) {
         exports: r.exports === 1
       }));
     },
+    getChunksMissingVectors() {
+      const rows = stmtGetChunksMissingVectors.all();
+      return rows.map((r) => ({
+        ...r,
+        exports: r.exports === 1
+      }));
+    },
     searchChunks(filters, limit) {
       const conditions = [];
       const params = [];
@@ -1429,6 +1452,66 @@ function createDatabase(dbPath, dimensions) {
         return Object.values(result[0])[0];
       }
       return String(result);
+    },
+    getVectorDimensions() {
+      const row = stmtGetMeta.get(VECTOR_DIMENSIONS_META_KEY);
+      if (!row) return null;
+      const dimensions2 = Number.parseInt(row.value, 10);
+      if (!Number.isInteger(dimensions2) || dimensions2 <= 0) {
+        throw new DatabaseError(
+          `Invalid stored vector dimensions metadata: ${row.value}`,
+          ErrorCode.DB_CORRUPTED
+        );
+      }
+      return dimensions2;
+    },
+    getIndexEmbedder() {
+      const providerRow = stmtGetMeta.get(INDEX_EMBEDDER_PROVIDER_META_KEY);
+      const modelRow = stmtGetMeta.get(INDEX_EMBEDDER_MODEL_META_KEY);
+      if (!providerRow && !modelRow) return null;
+      if (!providerRow || !modelRow) {
+        throw new DatabaseError(
+          "Corrupted index embedder metadata: provider/model keys are incomplete.",
+          ErrorCode.DB_CORRUPTED
+        );
+      }
+      const dimensions2 = this.getVectorDimensions();
+      if (dimensions2 === null) {
+        throw new DatabaseError(
+          "Corrupted index embedder metadata: vector dimensions are missing.",
+          ErrorCode.DB_CORRUPTED
+        );
+      }
+      return {
+        provider: providerRow.value,
+        model: modelRow.value,
+        dimensions: dimensions2
+      };
+    },
+    setIndexEmbedder(metadata) {
+      if (!metadata.provider || !metadata.model) {
+        throw new DatabaseError(
+          "Invalid index embedder metadata: provider and model are required.",
+          ErrorCode.DB_WRITE_FAILED
+        );
+      }
+      if (!Number.isInteger(metadata.dimensions) || metadata.dimensions <= 0) {
+        throw new DatabaseError(
+          `Invalid index embedder metadata dimensions: ${String(metadata.dimensions)}`,
+          ErrorCode.DB_WRITE_FAILED
+        );
+      }
+      const vectorDimensions = this.getVectorDimensions();
+      if (vectorDimensions !== null && vectorDimensions !== metadata.dimensions) {
+        throw new DatabaseError(
+          `Index embedder metadata dimensions (${metadata.dimensions}) do not match vector table dimensions (${vectorDimensions}).`,
+          ErrorCode.DB_WRITE_FAILED
+        );
+      }
+      db.transaction(() => {
+        stmtSetMeta.run(INDEX_EMBEDDER_PROVIDER_META_KEY, metadata.provider);
+        stmtSetMeta.run(INDEX_EMBEDDER_MODEL_META_KEY, metadata.model);
+      })();
     }
   };
 }
@@ -1789,10 +1872,14 @@ function validateProjectEmbedderConfig(config) {
 }
 
 // src/cli/commands/init.ts
+function isSameEmbedderConfig(a, b) {
+  return a.provider === b.provider && a.model === b.model && a.dimensions === b.dimensions;
+}
 var CTX_DIR2 = ".ctx";
 var DB_FILENAME = "index.db";
 var CONFIG_FILENAME2 = "config.json";
 var GITIGNORE_ENTRY = ".ctx/";
+var EMBEDDING_SAVE_BATCH_SIZE = 128;
 function ensureGitignore(projectRoot) {
   const gitignorePath = path5.join(projectRoot, ".gitignore");
   if (fs6.existsSync(gitignorePath)) {
@@ -1827,6 +1914,25 @@ function formatLanguageSummary(counts) {
   const entries = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([lang, count]) => `${lang}: ${count}`);
   return entries.join(", ");
 }
+async function embedAndPersistInBatches(db, embedder, chunks, log) {
+  let vectorsCreated = 0;
+  const total = chunks.length;
+  for (let i = 0; i < chunks.length; i += EMBEDDING_SAVE_BATCH_SIZE) {
+    const batch = chunks.slice(i, i + EMBEDDING_SAVE_BATCH_SIZE);
+    const texts = batch.map(
+      (chunk) => prepareChunkText(chunk.filePath, chunk.parent, chunk.text)
+    );
+    const vectors = await embedder.embed(texts);
+    db.transaction(() => {
+      for (let j = 0; j < batch.length; j++) {
+        db.insertVector(batch[j].id, vectors[j]);
+      }
+    });
+    vectorsCreated += vectors.length;
+    log(`  Embedding... ${vectorsCreated}/${total}`);
+  }
+  return vectorsCreated;
+}
 async function runInit(projectPath, options = {}) {
   const log = options.log ?? console.log;
   const absoluteRoot = path5.resolve(projectPath);
@@ -1840,6 +1946,24 @@ async function runInit(projectPath, options = {}) {
   const dbPath = path5.join(ctxDir, DB_FILENAME);
   const db = createDatabase(dbPath, embedderConfig.dimensions);
   try {
+    const existingEmbedder = db.getIndexEmbedder();
+    if (existingEmbedder) {
+      if (!isSameEmbedderConfig(existingEmbedder, embedderConfig)) {
+        throw new IndexError(
+          `Index embedder mismatch: index uses ${existingEmbedder.provider} (${existingEmbedder.model}, ${existingEmbedder.dimensions} dims) but config requests ${embedderConfig.provider} (${embedderConfig.model}, ${embedderConfig.dimensions} dims). Rebuild the index.`,
+          ErrorCode.CONFIG_INVALID
+        );
+      }
+    } else {
+      const isEmptyIndex = db.getFileCount() === 0 && db.getChunkCount() === 0 && db.getVectorCount() === 0;
+      if (isEmptyIndex) {
+        db.setIndexEmbedder({
+          provider: embedderConfig.provider,
+          model: embedderConfig.model,
+          dimensions: embedderConfig.dimensions
+        });
+      }
+    }
     const discovered = await discoverFiles({
       root: absoluteRoot,
       extraIgnore: [".ctx/"]
@@ -1925,21 +2049,34 @@ async function runInit(projectPath, options = {}) {
     }
     log(`  ${allChunksWithMeta.length} chunks created`);
     let vectorsCreated = 0;
-    if (!options.skipEmbedding && allChunksWithMeta.length > 0) {
-      const embedder = await createEmbedder(absoluteRoot);
-      const texts = allChunksWithMeta.map(
-        (cm) => prepareChunkText(cm.fileRelPath, cm.chunk.parent, cm.chunk.text)
-      );
-      const vectors = await embedder.embed(texts, (done, total) => {
-        log(`  Embedding... ${done}/${total}`);
-      });
-      db.transaction(() => {
-        for (let i = 0; i < allChunksWithMeta.length; i++) {
-          const chunkDbId = parseInt(allChunksWithMeta[i].chunk.id, 10);
-          db.insertVector(chunkDbId, vectors[i]);
+    if (!options.skipEmbedding) {
+      const chunksMissingVectors = db.getChunksMissingVectors().map((chunk) => ({
+        id: chunk.id,
+        filePath: chunk.filePath,
+        parent: chunk.parent,
+        text: chunk.text
+      }));
+      if (chunksMissingVectors.length > 0) {
+        log(`  ${chunksMissingVectors.length} chunks need embeddings`);
+      }
+      if (chunksMissingVectors.length > 0) {
+        const embedder = await createEmbedder(absoluteRoot);
+        try {
+          vectorsCreated = await embedAndPersistInBatches(
+            db,
+            embedder,
+            chunksMissingVectors,
+            log
+          );
+        } catch (err) {
+          const total = chunksMissingVectors.length;
+          throw new IndexError(
+            `Embedding failed after saving ${vectorsCreated}/${total} vectors. Run "ctx init" again to resume. ${err instanceof Error ? err.message : String(err)}`,
+            ErrorCode.EMBEDDER_FAILED,
+            err instanceof Error ? err : void 0
+          );
         }
-      });
-      vectorsCreated = vectors.length;
+      }
     }
     const durationMs = performance.now() - start;
     const dbSize = fs6.existsSync(dbPath) ? fs6.statSync(dbPath).size : 0;
@@ -2284,6 +2421,7 @@ var PATH_BOOST_PARTIAL = 1.2;
 var IMPORT_PENALTY = 0.5;
 var TEST_FILE_PENALTY = 0.65;
 var SMALL_SNIPPET_PENALTY = 0.75;
+var DATA_LITERAL_PENALTY = 0.7;
 var PUBLIC_API_BOOST = 1.12;
 var TEST_FILE_DIRECTORY_PATTERN = /(?:^|\/)(?:tests|__tests__)(?:\/|$)/;
 var TEST_FILE_NAME_PATTERN = /(?:^|\/)[^/]*\.(?:test|spec)\.[cm]?[jt]sx?$/;
@@ -2298,7 +2436,8 @@ function fusionMergeWithPathBoost(strategyResults, limit, pathBoostTerms) {
   const importAdjusted = applyImportDeprioritization(boosted);
   const testAdjusted = applyTestFileDeprioritization(importAdjusted);
   const snippetAdjusted = applySmallSnippetDeprioritization(testAdjusted);
-  const boostedApi = applyPublicApiBoost(snippetAdjusted);
+  const dataLiteralAdjusted = applyDataLiteralDeprioritization(snippetAdjusted);
+  const boostedApi = applyPublicApiBoost(dataLiteralAdjusted);
   const adjusted = applyFileDiversityDiminishingReturns(boostedApi);
   adjusted.sort((a, b) => b.score - a.score);
   const sliced = adjusted.slice(0, limit);
@@ -2379,6 +2518,21 @@ function applySmallSnippetDeprioritization(results) {
     return r;
   });
 }
+function applyDataLiteralDeprioritization(results) {
+  const hasNonDataLiteral = results.some((r) => !isDataLiteralChunk(r));
+  if (!hasNonDataLiteral) return results;
+  const maxNonDataScore = Math.max(
+    ...results.filter((r) => !isDataLiteralChunk(r)).map((r) => r.score),
+    0
+  );
+  if (maxNonDataScore === 0) return results;
+  return results.map((r) => {
+    if (isDataLiteralChunk(r)) {
+      return { ...r, score: r.score * DATA_LITERAL_PENALTY };
+    }
+    return r;
+  });
+}
 function applyPublicApiBoost(results) {
   return results.map((r) => {
     if (isPublicApiSymbol(r)) {
@@ -2413,6 +2567,23 @@ function isPublicApiSymbol(result) {
   const textStart = result.text.trimStart().toLowerCase();
   return textStart.startsWith("export ");
 }
+function isDataLiteralChunk(result) {
+  if (result.type !== "constant") return false;
+  const text = result.text;
+  const lines = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
+  if (lines.length === 0) return false;
+  const keyValueLineCount = lines.filter(
+    (line) => /['"`]?[A-Za-z0-9_-]+['"`]?\s*:\s*/.test(line)
+  ).length;
+  const quoteCount = (text.match(/["'`]/g) ?? []).length;
+  const colonCount = (text.match(/:/g) ?? []).length;
+  const structuralCount = quoteCount + colonCount;
+  const nonWhitespaceLength = text.replace(/\s+/g, "").length;
+  const structuralDensity = structuralCount / Math.max(nonWhitespaceLength, 1);
+  const hasLiteralShape = text.includes("{") && text.includes("}") || text.includes("[") && text.includes("]");
+  const kvLineRatio = keyValueLineCount / lines.length;
+  return hasLiteralShape && keyValueLineCount >= 2 && kvLineRatio >= 0.35 && structuralDensity >= 0.04;
+}
 function getFileDiversityFactor(fileOccurrence) {
   if (fileOccurrence <= 1) return 1;
   if (fileOccurrence === 2) return 0.9;
@@ -3679,6 +3850,7 @@ function createWatcher(options, events) {
 // src/cli/commands/watch.ts
 var CTX_DIR5 = ".ctx";
 var DB_FILENAME4 = "index.db";
+var EMBEDDING_SAVE_BATCH_SIZE2 = 128;
 function timestamp() {
   return (/* @__PURE__ */ new Date()).toLocaleTimeString("en-GB", { hour12: false });
 }
@@ -3695,6 +3867,9 @@ async function hashFile(absolutePath) {
   const content = fs9.readFileSync(absolutePath);
   return createHash3("sha256").update(content).digest("hex");
 }
+function isSameEmbedderConfig2(a, b) {
+  return a.provider === b.provider && a.model === b.model && a.dimensions === b.dimensions;
+}
 async function reindexChanges(db, changes, projectPath, options) {
   const start = performance.now();
   const log = options.log;
@@ -3763,16 +3938,30 @@ async function reindexChanges(db, changes, projectPath, options) {
   }
   if (!options.skipEmbedding && allChunksWithMeta.length > 0) {
     const embedder = await loadEmbedder3(projectPath);
-    const texts = allChunksWithMeta.map(
-      (cm) => prepareChunkText(cm.fileRelPath, cm.chunk.parent, cm.chunk.text)
-    );
-    const vectors = await embedder.embed(texts);
-    db.transaction(() => {
-      for (let i = 0; i < allChunksWithMeta.length; i++) {
-        const chunkDbId = parseInt(allChunksWithMeta[i].chunk.id, 10);
-        db.insertVector(chunkDbId, vectors[i]);
+    let vectorsCreated = 0;
+    const total = allChunksWithMeta.length;
+    try {
+      for (let i = 0; i < allChunksWithMeta.length; i += EMBEDDING_SAVE_BATCH_SIZE2) {
+        const batch = allChunksWithMeta.slice(i, i + EMBEDDING_SAVE_BATCH_SIZE2);
+        const texts = batch.map(
+          (cm) => prepareChunkText(cm.fileRelPath, cm.chunk.parent, cm.chunk.text)
+        );
+        const vectors = await embedder.embed(texts);
+        db.transaction(() => {
+          for (let j = 0; j < batch.length; j++) {
+            const chunkDbId = parseInt(batch[j].chunk.id, 10);
+            db.insertVector(chunkDbId, vectors[j]);
+          }
+        });
+        vectorsCreated += vectors.length;
       }
-    });
+    } catch (err) {
+      throw new IndexError(
+        `Embedding failed after saving ${vectorsCreated}/${total} vectors. Continue watching, then run "ctx init" to backfill missing vectors. ${err instanceof Error ? err.message : String(err)}`,
+        ErrorCode.EMBEDDER_FAILED,
+        err instanceof Error ? err : void 0
+      );
+    }
   }
   const durationMs = performance.now() - start;
   return { filesProcessed, chunksUpdated, durationMs };
@@ -3806,6 +3995,25 @@ async function runWatch(projectPath, options = {}) {
   await initParser();
   const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
   const db = createDatabase(dbPath, embedderConfig.dimensions);
+  const existingEmbedder = db.getIndexEmbedder();
+  if (existingEmbedder) {
+    if (!isSameEmbedderConfig2(existingEmbedder, embedderConfig)) {
+      db.close();
+      throw new IndexError(
+        `Index embedder mismatch: index uses ${existingEmbedder.provider} (${existingEmbedder.model}, ${existingEmbedder.dimensions} dims) but config requests ${embedderConfig.provider} (${embedderConfig.model}, ${embedderConfig.dimensions} dims). Rebuild the index.`,
+        ErrorCode.CONFIG_INVALID
+      );
+    }
+  } else {
+    const isEmptyIndex = db.getFileCount() === 0 && db.getChunkCount() === 0 && db.getVectorCount() === 0;
+    if (isEmptyIndex) {
+      db.setIndexEmbedder({
+        provider: embedderConfig.provider,
+        model: embedderConfig.model,
+        dimensions: embedderConfig.dimensions
+      });
+    }
+  }
   let watcherHandle = null;
   let reindexQueue = Promise.resolve();
   const watcher = createWatcher(
@@ -3951,15 +4159,54 @@ function formatStatus(projectPath, output) {
       lines.push(`    ${label}${count} file${count !== 1 ? "s" : ""}`);
     }
   }
-  if (output.config) {
-    lines.push("");
-    lines.push(
-      `  Embedder: ${output.config.provider} (${output.config.model}, ${output.config.dimensions} dims)`
-    );
+  const hasConfig = output.config !== null;
+  const hasIndexEmbedder = output.indexEmbedder !== null;
+  if (hasConfig || hasIndexEmbedder) lines.push("");
+  if (hasConfig && hasIndexEmbedder) {
+    const config = output.config;
+    const indexEmbedder = output.indexEmbedder;
+    if (!config || !indexEmbedder) {
+      lines.push("  Embedder: unknown");
+    } else if (isSameEmbedder(config, indexEmbedder)) {
+      lines.push(
+        `  Embedder: ${indexEmbedder.provider} (${indexEmbedder.model}, ${indexEmbedder.dimensions} dims)`
+      );
+    } else {
+      lines.push(
+        `  Index embedder:  ${indexEmbedder.provider} (${indexEmbedder.model}, ${indexEmbedder.dimensions} dims)`
+      );
+      lines.push(
+        `  Config embedder: ${config.provider} (${config.model}, ${config.dimensions} dims)`
+      );
+    }
+  } else if (hasIndexEmbedder) {
+    const indexEmbedder = output.indexEmbedder;
+    if (!indexEmbedder) {
+      lines.push("  Index embedder: unknown");
+    } else {
+      lines.push(
+        `  Index embedder: ${indexEmbedder.provider} (${indexEmbedder.model}, ${indexEmbedder.dimensions} dims)`
+      );
+    }
+  } else if (hasConfig) {
+    const config = output.config;
+    if (!config) {
+      lines.push("  Config embedder: unknown");
+    } else {
+      lines.push(
+        `  Config embedder: ${config.provider} (${config.model}, ${config.dimensions} dims)`
+      );
+    }
+  }
+  if (output.embedderWarning) {
+    lines.push(`  Warning: ${output.embedderWarning}`);
   }
   lines.push("");
   return lines.join("\n");
 }
+function isSameEmbedder(a, b) {
+  return a.provider === b.provider && a.model === b.model && a.dimensions === b.dimensions;
+}
 async function runStatus(projectPath) {
   const absoluteRoot = path10.resolve(projectPath);
   const ctxDir = path10.join(absoluteRoot, CTX_DIR6);
@@ -3974,6 +4221,8 @@ async function runStatus(projectPath) {
       lastIndexed: null,
       languages: /* @__PURE__ */ new Map(),
       config: null,
+      indexEmbedder: null,
+      embedderWarning: null,
       text: formatNotInitialized(absoluteRoot)
     };
     return output;
@@ -3986,7 +4235,9 @@ async function runStatus(projectPath) {
     const languages = db.getLanguageBreakdown();
     const lastIndexed = db.getLastIndexed();
     const config = readConfig2(ctxDir);
+    const indexEmbedder = db.getIndexEmbedder();
     const dbSizeBytes = fs10.statSync(dbPath).size;
+    const embedderWarning = config && indexEmbedder && !isSameEmbedder(config, indexEmbedder) ? `Index built with ${indexEmbedder.provider} (${indexEmbedder.dimensions} dims), config requests ${config.provider} (${config.dimensions} dims) \u2014 rebuild needed.` : null;
     const output = {
       initialized: true,
       fileCount,
@@ -3996,6 +4247,8 @@ async function runStatus(projectPath) {
       lastIndexed,
       languages,
       config,
+      indexEmbedder,
+      embedderWarning,
       text: ""
     };
     output.text = formatStatus(absoluteRoot, output);