kontext-engine 0.1.4 → 0.1.6

package/dist/cli/index.js

@@ -779,6 +779,25 @@ function prepareChunkText(filePath, parent, text) {
   parts.push(text);
   return parts.join("\n");
 }
+var MAX_RETRIES = 3;
+var BASE_DELAY_MS = 500;
+async function fetchWithRetry(url, init) {
+  let lastError = null;
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    const response = await fetch(url, init);
+    if (response.ok) return response;
+    if (response.status === 429 && attempt < MAX_RETRIES) {
+      const delay = BASE_DELAY_MS * Math.pow(2, attempt);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+      lastError = new Error(`HTTP ${response.status}: ${response.statusText}`);
+      continue;
+    }
+    throw new Error(
+      `Embedding API error: HTTP ${response.status} ${response.statusText}`
+    );
+  }
+  throw lastError ?? new Error("Embedding API request failed after retries");
+}
 var LOCAL_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
 var LOCAL_DIMENSIONS = 384;
 var LOCAL_BATCH_SIZE = 32;
@@ -836,6 +855,87 @@ async function createLocalEmbedder() {
     }
   };
 }
+var VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings";
+var VOYAGE_MODEL = "voyage-code-3";
+var VOYAGE_DEFAULT_DIMENSIONS = 1024;
+var VOYAGE_BATCH_SIZE = 128;
+function createVoyageEmbedder(apiKey, dimensions = VOYAGE_DEFAULT_DIMENSIONS) {
+  return {
+    name: VOYAGE_MODEL,
+    dimensions,
+    async embed(texts, onProgress) {
+      const results = [];
+      for (let i = 0; i < texts.length; i += VOYAGE_BATCH_SIZE) {
+        const batch = texts.slice(i, i + VOYAGE_BATCH_SIZE);
+        const vectors = await callVoyageAPI(apiKey, batch, "document", dimensions);
+        results.push(...vectors);
+        onProgress?.(Math.min(i + batch.length, texts.length), texts.length);
+      }
+      return results;
+    },
+    async embedSingle(text) {
+      const vectors = await callVoyageAPI(apiKey, [text], "query", dimensions);
+      return vectors[0];
+    }
+  };
+}
+async function callVoyageAPI(apiKey, texts, inputType, dimensions) {
+  const response = await fetchWithRetry(VOYAGE_API_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`
+    },
+    body: JSON.stringify({
+      model: VOYAGE_MODEL,
+      input: texts,
+      input_type: inputType,
+      output_dimension: dimensions
+    })
+  });
+  const json = await response.json();
+  return json.data.map((d) => normalizeVector(new Float32Array(d.embedding)));
+}
+var OPENAI_API_URL = "https://api.openai.com/v1/embeddings";
+var OPENAI_MODEL = "text-embedding-3-large";
+var OPENAI_DEFAULT_DIMENSIONS = 1024;
+var OPENAI_BATCH_SIZE = 128;
+function createOpenAIEmbedder(apiKey, dimensions = OPENAI_DEFAULT_DIMENSIONS) {
+  return {
+    name: OPENAI_MODEL,
+    dimensions,
+    async embed(texts, onProgress) {
+      const results = [];
+      for (let i = 0; i < texts.length; i += OPENAI_BATCH_SIZE) {
+        const batch = texts.slice(i, i + OPENAI_BATCH_SIZE);
+        const vectors = await callOpenAIAPI(apiKey, batch, dimensions);
+        results.push(...vectors);
+        onProgress?.(Math.min(i + batch.length, texts.length), texts.length);
+      }
+      return results;
+    },
+    async embedSingle(text) {
+      const vectors = await callOpenAIAPI(apiKey, [text], dimensions);
+      return vectors[0];
+    }
+  };
+}
+async function callOpenAIAPI(apiKey, texts, dimensions) {
+  const response = await fetchWithRetry(OPENAI_API_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`
+    },
+    body: JSON.stringify({
+      model: OPENAI_MODEL,
+      input: texts,
+      dimensions
+    })
+  });
+  const json = await response.json();
+  return json.data.map((d) => normalizeVector(new Float32Array(d.embedding)));
+}
 
 // src/utils/errors.ts
 var ErrorCode = {
@@ -877,6 +977,12 @@ var ConfigError = class extends KontextError {
     this.name = "ConfigError";
   }
 };
+var DatabaseError = class extends KontextError {
+  constructor(message, code, cause) {
+    super(message, code, cause);
+    this.name = "DatabaseError";
+  }
+};
 
 // src/utils/error-boundary.ts
 function handleCommandError(err, logger, verbose) {
@@ -1051,7 +1157,10 @@ function searchVectors(db, query, limit) {
 
 // src/storage/db.ts
 var DEFAULT_DIMENSIONS = 384;
-function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
+var VECTOR_DIMENSIONS_META_KEY = "vector_dimensions";
+var INDEX_EMBEDDER_PROVIDER_META_KEY = "index_embedder_provider";
+var INDEX_EMBEDDER_MODEL_META_KEY = "index_embedder_model";
+function createDatabase(dbPath, dimensions) {
   const dir = path3.dirname(dbPath);
   if (!fs4.existsSync(dir)) {
     fs4.mkdirSync(dir, { recursive: true });
@@ -1060,7 +1169,8 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
   db.pragma("journal_mode = WAL");
   db.pragma("foreign_keys = ON");
   sqliteVec.load(db);
-  initializeSchema(db, dimensions);
+  initializeSchema(db, dimensions ?? DEFAULT_DIMENSIONS);
+  ensureVectorDimensions(db, dimensions);
   const stmtUpsertFile = db.prepare(`
     INSERT INTO files (path, language, hash, last_indexed, size)
     VALUES (@path, @language, @hash, @lastIndexed, @size)
@@ -1069,6 +1179,7 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
       hash = excluded.hash,
       last_indexed = excluded.last_indexed,
       size = excluded.size
+    RETURNING id
   `);
   const stmtGetFile = db.prepare(
     "SELECT id, path, language, hash, last_indexed as lastIndexed, size FROM files WHERE path = ?"
@@ -1081,6 +1192,16 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
   const stmtGetChunksByFile = db.prepare(
     "SELECT id, file_id as fileId, line_start as lineStart, line_end as lineEnd, type, name, parent, text, imports, exports, hash FROM chunks WHERE file_id = ? ORDER BY line_start"
   );
+  const stmtGetChunksMissingVectors = db.prepare(
+    `SELECT c.id, c.file_id as fileId, f.path as filePath, f.language,
+            c.line_start as lineStart, c.line_end as lineEnd,
+            c.type, c.name, c.parent, c.text, c.exports as exports
+     FROM chunks c
+     JOIN files f ON f.id = c.file_id
+     LEFT JOIN chunk_vectors v ON v.rowid = c.id
+     WHERE v.rowid IS NULL
+     ORDER BY c.id`
+  );
   const stmtGetChunkIdsByFile = db.prepare(
     "SELECT id FROM chunks WHERE file_id = ?"
   );
@@ -1110,20 +1231,26 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
   const stmtLastIndexed = db.prepare(
     "SELECT MAX(last_indexed) as lastIndexed FROM files"
   );
+  const stmtGetMeta = db.prepare("SELECT value FROM meta WHERE key = ?");
+  const stmtSetMeta = db.prepare(
+    "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)"
+  );
   return {
     upsertFile(file) {
-      const result = stmtUpsertFile.run({
+      const row = stmtUpsertFile.get({
         path: file.path,
         language: file.language,
         hash: file.hash,
         lastIndexed: Date.now(),
         size: file.size
       });
-      if (result.changes > 0 && result.lastInsertRowid) {
-        return Number(result.lastInsertRowid);
+      if (!row?.id) {
+        throw new DatabaseError(
+          `Failed to upsert file: ${file.path}`,
+          ErrorCode.DB_WRITE_FAILED
+        );
       }
-      const existing = stmtGetFile.get(file.path);
-      return existing?.id ?? 0;
+      return row.id;
     },
     getFile(filePath) {
       const row = stmtGetFile.get(filePath);
@@ -1166,15 +1293,17 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
       return row.lastIndexed;
     },
     deleteFile(filePath) {
-      const file = stmtGetFile.get(filePath);
-      if (file) {
-        const chunkRows = stmtGetChunkIdsByFile.all(file.id);
-        const chunkIds = chunkRows.map((r) => r.id);
-        if (chunkIds.length > 0) {
-          deleteVectorsByChunkIds(db, chunkIds);
+      db.transaction(() => {
+        const file = stmtGetFile.get(filePath);
+        if (file) {
+          const chunkRows = stmtGetChunkIdsByFile.all(file.id);
+          const chunkIds = chunkRows.map((r) => r.id);
+          if (chunkIds.length > 0) {
+            deleteVectorsByChunkIds(db, chunkIds);
+          }
         }
-      }
-      stmtDeleteFile.run(filePath);
+        stmtDeleteFile.run(filePath);
+      })();
     },
     insertChunks(fileId, chunks) {
       const ids = [];
@@ -1219,6 +1348,13 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
         exports: r.exports === 1
       }));
     },
+    getChunksMissingVectors() {
+      const rows = stmtGetChunksMissingVectors.all();
+      return rows.map((r) => ({
+        ...r,
+        exports: r.exports === 1
+      }));
+    },
     searchChunks(filters, limit) {
       const conditions = [];
       const params = [];
@@ -1269,12 +1405,14 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
       }));
     },
     deleteChunksByFile(fileId) {
-      const chunkRows = stmtGetChunkIdsByFile.all(fileId);
-      const chunkIds = chunkRows.map((r) => r.id);
-      if (chunkIds.length > 0) {
-        deleteVectorsByChunkIds(db, chunkIds);
-      }
-      stmtDeleteChunksByFile.run(fileId);
+      db.transaction(() => {
+        const chunkRows = stmtGetChunkIdsByFile.all(fileId);
+        const chunkIds = chunkRows.map((r) => r.id);
+        if (chunkIds.length > 0) {
+          deleteVectorsByChunkIds(db, chunkIds);
+        }
+        stmtDeleteChunksByFile.run(fileId);
+      })();
     },
     insertDependency(sourceChunkId, targetChunkId, type) {
       stmtInsertDep.run(sourceChunkId, targetChunkId, type);
@@ -1314,6 +1452,66 @@ function createDatabase(dbPath, dimensions = DEFAULT_DIMENSIONS) {
         return Object.values(result[0])[0];
       }
       return String(result);
+    },
+    getVectorDimensions() {
+      const row = stmtGetMeta.get(VECTOR_DIMENSIONS_META_KEY);
+      if (!row) return null;
+      const dimensions2 = Number.parseInt(row.value, 10);
+      if (!Number.isInteger(dimensions2) || dimensions2 <= 0) {
+        throw new DatabaseError(
+          `Invalid stored vector dimensions metadata: ${row.value}`,
+          ErrorCode.DB_CORRUPTED
+        );
+      }
+      return dimensions2;
+    },
+    getIndexEmbedder() {
+      const providerRow = stmtGetMeta.get(INDEX_EMBEDDER_PROVIDER_META_KEY);
+      const modelRow = stmtGetMeta.get(INDEX_EMBEDDER_MODEL_META_KEY);
+      if (!providerRow && !modelRow) return null;
+      if (!providerRow || !modelRow) {
+        throw new DatabaseError(
+          "Corrupted index embedder metadata: provider/model keys are incomplete.",
+          ErrorCode.DB_CORRUPTED
+        );
+      }
+      const dimensions2 = this.getVectorDimensions();
+      if (dimensions2 === null) {
+        throw new DatabaseError(
+          "Corrupted index embedder metadata: vector dimensions are missing.",
+          ErrorCode.DB_CORRUPTED
+        );
+      }
+      return {
+        provider: providerRow.value,
+        model: modelRow.value,
+        dimensions: dimensions2
+      };
+    },
+    setIndexEmbedder(metadata) {
+      if (!metadata.provider || !metadata.model) {
+        throw new DatabaseError(
+          "Invalid index embedder metadata: provider and model are required.",
+          ErrorCode.DB_WRITE_FAILED
+        );
+      }
+      if (!Number.isInteger(metadata.dimensions) || metadata.dimensions <= 0) {
+        throw new DatabaseError(
+          `Invalid index embedder metadata dimensions: ${String(metadata.dimensions)}`,
+          ErrorCode.DB_WRITE_FAILED
+        );
+      }
+      const vectorDimensions = this.getVectorDimensions();
+      if (vectorDimensions !== null && vectorDimensions !== metadata.dimensions) {
+        throw new DatabaseError(
+          `Index embedder metadata dimensions (${metadata.dimensions}) do not match vector table dimensions (${vectorDimensions}).`,
+          ErrorCode.DB_WRITE_FAILED
+        );
+      }
+      db.transaction(() => {
+        stmtSetMeta.run(INDEX_EMBEDDER_PROVIDER_META_KEY, metadata.provider);
+        stmtSetMeta.run(INDEX_EMBEDDER_MODEL_META_KEY, metadata.model);
+      })();
     }
   };
 }
@@ -1336,6 +1534,59 @@ function getMetaVersion(db) {
     return 0;
   }
 }
+function ensureVectorDimensions(db, expectedDimensions) {
+  const actual = getExistingVectorDimensions(db);
+  const stored = db.prepare("SELECT value FROM meta WHERE key = ?").get(VECTOR_DIMENSIONS_META_KEY);
+  const storedValue = stored?.value;
+  const storedDimensions = storedValue ? Number.parseInt(storedValue, 10) : void 0;
+  if (storedDimensions !== void 0 && (!Number.isInteger(storedDimensions) || storedDimensions <= 0)) {
+    throw new DatabaseError(
+      `Invalid stored vector dimensions metadata: ${storedValue ?? "unknown"}`,
+      ErrorCode.DB_CORRUPTED
+    );
+  }
+  if (actual !== null && storedDimensions !== void 0 && storedDimensions !== actual) {
+    throw new DatabaseError(
+      `Vector dimensions metadata mismatch: meta=${storedDimensions}, table=${actual}.`,
+      ErrorCode.DB_CORRUPTED
+    );
+  }
+  if (expectedDimensions === void 0) {
+    if (storedDimensions !== void 0) return;
+    const dimensions = actual ?? DEFAULT_DIMENSIONS;
+    db.prepare(
+      "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)"
+    ).run(VECTOR_DIMENSIONS_META_KEY, String(dimensions));
+    return;
+  }
+  if (!stored) {
+    const dimensions = actual ?? expectedDimensions;
+    db.prepare(
+      "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)"
+    ).run(VECTOR_DIMENSIONS_META_KEY, String(dimensions));
+    if (actual !== null && actual !== expectedDimensions) {
+      throw new DatabaseError(
+        `Vector dimension mismatch: index uses ${actual} dims, but config requests ${expectedDimensions} dims. Rebuild the index.`,
+        ErrorCode.CONFIG_INVALID
+      );
+    }
+    return;
+  }
+  if (storedDimensions !== expectedDimensions) {
+    throw new DatabaseError(
+      `Vector dimension mismatch: index uses ${storedDimensions} dims, but config requests ${expectedDimensions} dims. Rebuild the index.`,
+      ErrorCode.CONFIG_INVALID
+    );
+  }
+}
+function getExistingVectorDimensions(db) {
+  const row = db.prepare("SELECT sql FROM sqlite_master WHERE name = 'chunk_vectors'").get();
+  const sql = row?.sql;
+  if (!sql) return null;
+  const match = sql.match(/embedding\s+float\[(\d+)\]/i);
+  if (!match) return null;
+  return Number.parseInt(match[1], 10);
+}
 
 // src/cli/commands/config.ts
 import fs5 from "fs";
@@ -1451,6 +1702,20 @@ function setNestedValue(obj, key, value) {
   }
   current[parts[parts.length - 1]] = value;
 }
+function hasNestedKey(obj, key) {
+  const parts = key.split(".");
+  let current = obj;
+  for (const part of parts) {
+    if (current === null || current === void 0 || typeof current !== "object") {
+      return false;
+    }
+    if (!(part in current)) {
+      return false;
+    }
+    current = current[part];
+  }
+  return true;
+}
 function parseValue(rawValue) {
   if (rawValue === "null") return null;
   if (rawValue === "true") return true;
@@ -1489,9 +1754,26 @@ function runConfigSet(projectPath, key, rawValue) {
   setNestedValue(config, key, value);
   writeConfig(ctxDir, config);
 }
-function runConfigReset(projectPath) {
+function runConfigReset(projectPath, key) {
   const ctxDir = resolveCtxDir(projectPath);
-  writeConfig(ctxDir, structuredClone(DEFAULT_CONFIG));
+  if (!key) {
+    writeConfig(ctxDir, structuredClone(DEFAULT_CONFIG));
+    return;
+  }
+  if (!hasNestedKey(DEFAULT_CONFIG, key)) {
+    throw new ConfigError(`Invalid config key: ${key}`, ErrorCode.CONFIG_INVALID);
+  }
+  const config = readConfig(ctxDir);
+  const defaultValue = getNestedValue(
+    DEFAULT_CONFIG,
+    key
+  );
+  setNestedValue(
+    config,
+    key,
+    structuredClone(defaultValue)
+  );
+  writeConfig(ctxDir, config);
 }
 function registerConfigCommand(program2) {
   const cmd = program2.command("config").description("Show or modify configuration");
@@ -1526,21 +1808,78 @@ function registerConfigCommand(program2) {
       configErrorHandler(err);
     }
   });
-  cmd.command("reset").description("Reset configuration to defaults").action(() => {
+  cmd.command("reset [key]").description("Reset configuration to defaults or reset a specific key").action((key) => {
     try {
-      runConfigReset(process.cwd());
-      console.log("Configuration reset to defaults.");
+      runConfigReset(process.cwd(), key);
+      if (key) {
+        console.log(`Reset ${key} to default.`);
+      } else {
+        console.log("Configuration reset to defaults.");
+      }
     } catch (err) {
       configErrorHandler(err);
     }
   });
 }
 
+// src/cli/embedder.ts
+function getProjectEmbedderConfig(projectPath) {
+  const { config } = runConfigShow(projectPath);
+  return config.embedder;
+}
+async function createProjectEmbedder(projectPath) {
+  const config = getProjectEmbedderConfig(projectPath);
+  validateProjectEmbedderConfig(config);
+  switch (config.provider) {
+    case "local":
+      return await createLocalEmbedder();
+    case "voyage": {
+      const apiKey = requireApiKey("CTX_VOYAGE_KEY", "voyage");
+      return createVoyageEmbedder(apiKey, config.dimensions);
+    }
+    case "openai": {
+      const apiKey = requireApiKey("CTX_OPENAI_KEY", "openai");
+      return createOpenAIEmbedder(apiKey, config.dimensions);
+    }
+    default:
+      throw new ConfigError(
+        `Unsupported embedder provider "${config.provider}". Use local, voyage, or openai.`,
+        ErrorCode.CONFIG_INVALID
+      );
+  }
+}
+function requireApiKey(envVar, provider) {
+  const value = process.env[envVar];
+  if (typeof value === "string" && value.length > 0) return value;
+  throw new ConfigError(
+    `Embedder provider "${provider}" requires ${envVar}. Export ${envVar} before running this command.`,
+    ErrorCode.CONFIG_INVALID
+  );
+}
+function validateProjectEmbedderConfig(config) {
+  if (!Number.isInteger(config.dimensions) || config.dimensions <= 0) {
+    throw new ConfigError(
+      `Invalid embedder.dimensions (${String(config.dimensions)}). Must be a positive integer.`,
+      ErrorCode.CONFIG_INVALID
+    );
+  }
+  if (config.provider === "local" && config.dimensions !== 384) {
+    throw new ConfigError(
+      'Local embedder requires "embedder.dimensions" = 384. Update config or switch provider.',
+      ErrorCode.CONFIG_INVALID
+    );
+  }
+}
+
 // src/cli/commands/init.ts
+function isSameEmbedderConfig(a, b) {
+  return a.provider === b.provider && a.model === b.model && a.dimensions === b.dimensions;
+}
 var CTX_DIR2 = ".ctx";
 var DB_FILENAME = "index.db";
 var CONFIG_FILENAME2 = "config.json";
 var GITIGNORE_ENTRY = ".ctx/";
+var EMBEDDING_SAVE_BATCH_SIZE = 128;
 function ensureGitignore(projectRoot) {
   const gitignorePath = path5.join(projectRoot, ".gitignore");
   if (fs6.existsSync(gitignorePath)) {
@@ -1575,6 +1914,25 @@ function formatLanguageSummary(counts) {
   const entries = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([lang, count]) => `${lang}: ${count}`);
   return entries.join(", ");
 }
+async function embedAndPersistInBatches(db, embedder, chunks, log) {
+  let vectorsCreated = 0;
+  const total = chunks.length;
+  for (let i = 0; i < chunks.length; i += EMBEDDING_SAVE_BATCH_SIZE) {
+    const batch = chunks.slice(i, i + EMBEDDING_SAVE_BATCH_SIZE);
+    const texts = batch.map(
+      (chunk) => prepareChunkText(chunk.filePath, chunk.parent, chunk.text)
+    );
+    const vectors = await embedder.embed(texts);
+    db.transaction(() => {
+      for (let j = 0; j < batch.length; j++) {
+        db.insertVector(batch[j].id, vectors[j]);
+      }
+    });
+    vectorsCreated += vectors.length;
+    log(`  Embedding... ${vectorsCreated}/${total}`);
+  }
+  return vectorsCreated;
+}
 async function runInit(projectPath, options = {}) {
   const log = options.log ?? console.log;
   const absoluteRoot = path5.resolve(projectPath);
@@ -1584,9 +1942,28 @@ async function runInit(projectPath, options = {}) {
   if (!fs6.existsSync(ctxDir)) fs6.mkdirSync(ctxDir, { recursive: true });
   ensureGitignore(absoluteRoot);
   ensureConfig(ctxDir);
+  const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
   const dbPath = path5.join(ctxDir, DB_FILENAME);
-  const db = createDatabase(dbPath);
+  const db = createDatabase(dbPath, embedderConfig.dimensions);
   try {
+    const existingEmbedder = db.getIndexEmbedder();
+    if (existingEmbedder) {
+      if (!isSameEmbedderConfig(existingEmbedder, embedderConfig)) {
+        throw new IndexError(
+          `Index embedder mismatch: index uses ${existingEmbedder.provider} (${existingEmbedder.model}, ${existingEmbedder.dimensions} dims) but config requests ${embedderConfig.provider} (${embedderConfig.model}, ${embedderConfig.dimensions} dims). Rebuild the index.`,
+          ErrorCode.CONFIG_INVALID
+        );
+      }
+    } else {
+      const isEmptyIndex = db.getFileCount() === 0 && db.getChunkCount() === 0 && db.getVectorCount() === 0;
+      if (isEmptyIndex) {
+        db.setIndexEmbedder({
+          provider: embedderConfig.provider,
+          model: embedderConfig.model,
+          dimensions: embedderConfig.dimensions
+        });
+      }
+    }
     const discovered = await discoverFiles({
       root: absoluteRoot,
       extraIgnore: [".ctx/"]
@@ -1672,21 +2049,34 @@ async function runInit(projectPath, options = {}) {
     }
     log(`  ${allChunksWithMeta.length} chunks created`);
     let vectorsCreated = 0;
-    if (!options.skipEmbedding && allChunksWithMeta.length > 0) {
-      const embedder = await createEmbedder();
-      const texts = allChunksWithMeta.map(
-        (cm) => prepareChunkText(cm.fileRelPath, cm.chunk.parent, cm.chunk.text)
-      );
-      const vectors = await embedder.embed(texts, (done, total) => {
-        log(`  Embedding... ${done}/${total}`);
-      });
-      db.transaction(() => {
-        for (let i = 0; i < allChunksWithMeta.length; i++) {
-          const chunkDbId = parseInt(allChunksWithMeta[i].chunk.id, 10);
-          db.insertVector(chunkDbId, vectors[i]);
+    if (!options.skipEmbedding) {
+      const chunksMissingVectors = db.getChunksMissingVectors().map((chunk) => ({
+        id: chunk.id,
+        filePath: chunk.filePath,
+        parent: chunk.parent,
+        text: chunk.text
+      }));
+      if (chunksMissingVectors.length > 0) {
+        log(`  ${chunksMissingVectors.length} chunks need embeddings`);
+      }
+      if (chunksMissingVectors.length > 0) {
+        const embedder = await createEmbedder(absoluteRoot);
+        try {
+          vectorsCreated = await embedAndPersistInBatches(
+            db,
+            embedder,
+            chunksMissingVectors,
+            log
+          );
+        } catch (err) {
+          const total = chunksMissingVectors.length;
+          throw new IndexError(
+            `Embedding failed after saving ${vectorsCreated}/${total} vectors. Run "ctx init" again to resume. ${err instanceof Error ? err.message : String(err)}`,
+            ErrorCode.EMBEDDER_FAILED,
+            err instanceof Error ? err : void 0
+          );
         }
-      });
-      vectorsCreated = vectors.length;
+      }
     }
     const durationMs = performance.now() - start;
     const dbSize = fs6.existsSync(dbPath) ? fs6.statSync(dbPath).size : 0;
@@ -1711,8 +2101,8 @@ async function runInit(projectPath, options = {}) {
     db.close();
   }
 }
-async function createEmbedder() {
-  return createLocalEmbedder();
+async function createEmbedder(projectPath) {
+  return createProjectEmbedder(projectPath);
 }
 function registerInitCommand(program2) {
   program2.command("init [path]").description("Index current directory or specified path").action(async (inputPath) => {
@@ -2031,6 +2421,7 @@ var PATH_BOOST_PARTIAL = 1.2;
 var IMPORT_PENALTY = 0.5;
 var TEST_FILE_PENALTY = 0.65;
 var SMALL_SNIPPET_PENALTY = 0.75;
+var DATA_LITERAL_PENALTY = 0.7;
 var PUBLIC_API_BOOST = 1.12;
 var TEST_FILE_DIRECTORY_PATTERN = /(?:^|\/)(?:tests|__tests__)(?:\/|$)/;
 var TEST_FILE_NAME_PATTERN = /(?:^|\/)[^/]*\.(?:test|spec)\.[cm]?[jt]sx?$/;
@@ -2045,7 +2436,8 @@ function fusionMergeWithPathBoost(strategyResults, limit, pathBoostTerms) {
   const importAdjusted = applyImportDeprioritization(boosted);
   const testAdjusted = applyTestFileDeprioritization(importAdjusted);
   const snippetAdjusted = applySmallSnippetDeprioritization(testAdjusted);
-  const boostedApi = applyPublicApiBoost(snippetAdjusted);
+  const dataLiteralAdjusted = applyDataLiteralDeprioritization(snippetAdjusted);
+  const boostedApi = applyPublicApiBoost(dataLiteralAdjusted);
   const adjusted = applyFileDiversityDiminishingReturns(boostedApi);
   adjusted.sort((a, b) => b.score - a.score);
   const sliced = adjusted.slice(0, limit);
@@ -2126,6 +2518,21 @@ function applySmallSnippetDeprioritization(results) {
     return r;
   });
 }
+function applyDataLiteralDeprioritization(results) {
+  const hasNonDataLiteral = results.some((r) => !isDataLiteralChunk(r));
+  if (!hasNonDataLiteral) return results;
+  const maxNonDataScore = Math.max(
+    ...results.filter((r) => !isDataLiteralChunk(r)).map((r) => r.score),
+    0
+  );
+  if (maxNonDataScore === 0) return results;
+  return results.map((r) => {
+    if (isDataLiteralChunk(r)) {
+      return { ...r, score: r.score * DATA_LITERAL_PENALTY };
+    }
+    return r;
+  });
+}
 function applyPublicApiBoost(results) {
   return results.map((r) => {
     if (isPublicApiSymbol(r)) {
@@ -2160,6 +2567,23 @@ function isPublicApiSymbol(result) {
   const textStart = result.text.trimStart().toLowerCase();
   return textStart.startsWith("export ");
 }
+function isDataLiteralChunk(result) {
+  if (result.type !== "constant") return false;
+  const text = result.text;
+  const lines = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
+  if (lines.length === 0) return false;
+  const keyValueLineCount = lines.filter(
+    (line) => /['"`]?[A-Za-z0-9_-]+['"`]?\s*:\s*/.test(line)
+  ).length;
+  const quoteCount = (text.match(/["'`]/g) ?? []).length;
+  const colonCount = (text.match(/:/g) ?? []).length;
+  const structuralCount = quoteCount + colonCount;
+  const nonWhitespaceLength = text.replace(/\s+/g, "").length;
+  const structuralDensity = structuralCount / Math.max(nonWhitespaceLength, 1);
+  const hasLiteralShape = text.includes("{") && text.includes("}") || text.includes("[") && text.includes("]");
+  const kvLineRatio = keyValueLineCount / lines.length;
+  return hasLiteralShape && keyValueLineCount >= 2 && kvLineRatio >= 0.35 && structuralDensity >= 0.04;
+}
 function getFileDiversityFactor(fileOccurrence) {
   if (fileOccurrence <= 1) return 1;
   if (fileOccurrence === 2) return 0.9;
@@ -2340,9 +2764,13 @@ async function runQuery(projectPath, query, options) {
     );
   }
   const start = performance.now();
-  const db = createDatabase(dbPath);
+  const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
+  const db = createDatabase(dbPath, embedderConfig.dimensions);
   try {
-    const strategyResults = await runStrategies(db, query, { ...options, limit });
+    const strategyResults = await runStrategies(db, absoluteRoot, query, {
+      ...options,
+      limit
+    });
     const pathBoostTerms = extractPathBoostTerms(query);
     const fused = fusionMergeWithPathBoost(strategyResults, limit, pathBoostTerms);
     const outputResults = fused.map(toOutputResult);
@@ -2362,7 +2790,7 @@ async function runQuery(projectPath, query, options) {
     db.close();
   }
 }
-async function runStrategies(db, query, options) {
+async function runStrategies(db, projectPath, query, options) {
   const results = [];
   const filters = options.language ? { language: options.language } : void 0;
   const limit = options.limit * 3;
@@ -2371,6 +2799,7 @@ async function runStrategies(db, query, options) {
     const weight = effectiveWeights[strategy];
     const searchResults = await executeStrategy(
       db,
+      projectPath,
       strategy,
       query,
       limit,
@@ -2382,10 +2811,10 @@ async function runStrategies(db, query, options) {
   }
   return results;
 }
-async function executeStrategy(db, strategy, query, limit, filters) {
+async function executeStrategy(db, projectPath, strategy, query, limit, filters) {
   switch (strategy) {
     case "vector": {
-      const embedder = await loadEmbedder();
+      const embedder = await loadEmbedder(projectPath);
       return vectorSearch(db, embedder, query, limit, filters);
     }
     case "fts":
@@ -2418,9 +2847,16 @@ async function executeStrategy(db, strategy, query, limit, filters) {
   }
 }
 var embedderInstance = null;
-async function loadEmbedder() {
-  if (embedderInstance) return embedderInstance;
-  embedderInstance = await createLocalEmbedder();
+var embedderKey = null;
+function getCacheKey(projectPath) {
+  const config = getProjectEmbedderConfig(projectPath);
+  return `${projectPath}:${config.provider}:${config.model}:${config.dimensions}`;
+}
+async function loadEmbedder(projectPath) {
+  const cacheKey = getCacheKey(projectPath);
+  if (embedderInstance && embedderKey === cacheKey) return embedderInstance;
+  embedderInstance = await createProjectEmbedder(projectPath);
+  embedderKey = cacheKey;
   return embedderInstance;
 }
 function registerQueryCommand(program2) {
@@ -2838,7 +3274,9 @@ var COMMON_STEMS = {
   transformer: "transform",
   transformation: "transform",
   connection: "connect",
+  connecting: "connect",
   connector: "connect",
+  migrating: "migrate",
   migration: "migrate",
   scheduling: "schedule",
   scheduler: "schedule",
@@ -2847,7 +3285,8 @@ var COMMON_STEMS = {
   routing: "route",
   router: "route",
   indexing: "index",
-  indexer: "index"
+  indexer: "index",
+  subscribing: "subscribe"
 };
 var STEM_SUFFIXES = [
   "tion",
@@ -3127,13 +3566,13 @@ function formatTextOutput2(output) {
   );
   return lines.join("\n");
 }
-function createSearchExecutor(db, query) {
+function createSearchExecutor(db, projectPath, query) {
   const pathBoostTerms = extractPathBoostTerms(query);
   return async (strategies, limit) => {
     const strategyResults = [];
     const fetchLimit = limit * 3;
     for (const plan of strategies) {
-      const results = await executeStrategy2(db, plan, fetchLimit);
+      const results = await executeStrategy2(db, projectPath, plan, fetchLimit);
       if (results.length > 0) {
         strategyResults.push({
           strategy: plan.strategy,
@@ -3152,10 +3591,10 @@ function extractSymbolNames2(query) {
 function isPathLike2(query) {
   return query.includes("/") || query.includes("*") || query.includes(".");
 }
-async function executeStrategy2(db, plan, limit) {
+async function executeStrategy2(db, projectPath, plan, limit) {
   switch (plan.strategy) {
     case "vector": {
-      const embedder = await loadEmbedder2();
+      const embedder = await loadEmbedder2(projectPath);
       return vectorSearch(db, embedder, plan.query, limit);
     }
     case "fts":
@@ -3184,13 +3623,20 @@ async function executeStrategy2(db, plan, limit) {
   }
 }
 var embedderInstance2 = null;
-async function loadEmbedder2() {
-  if (embedderInstance2) return embedderInstance2;
-  embedderInstance2 = await createLocalEmbedder();
+var embedderKey2 = null;
+function getCacheKey2(projectPath) {
+  const config = getProjectEmbedderConfig(projectPath);
+  return `${projectPath}:${config.provider}:${config.model}:${config.dimensions}`;
+}
+async function loadEmbedder2(projectPath) {
+  const cacheKey = getCacheKey2(projectPath);
+  if (embedderInstance2 && embedderKey2 === cacheKey) return embedderInstance2;
+  embedderInstance2 = await createProjectEmbedder(projectPath);
+  embedderKey2 = cacheKey;
   return embedderInstance2;
 }
-async function fallbackSearch(db, query, limit) {
-  const executor = createSearchExecutor(db, query);
+async function fallbackSearch(db, projectPath, query, limit) {
+  const executor = createSearchExecutor(db, projectPath, query);
   const fallbackStrategies = buildFallbackStrategies(query);
   const results = await executor(fallbackStrategies, limit);
   return {
@@ -3217,18 +3663,19 @@ async function runAsk(projectPath, query, options) {
       ErrorCode.NOT_INITIALIZED
     );
   }
-  const db = createDatabase(dbPath);
+  const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
+  const db = createDatabase(dbPath, embedderConfig.dimensions);
   try {
     const provider = options.provider ?? null;
     if (!provider) {
-      const output = await fallbackSearch(db, query, limit);
+      const output = await fallbackSearch(db, absoluteRoot, query, limit);
       output.warning = FALLBACK_NOTICE;
       if (options.format === "text") {
         output.text = formatTextOutput2(output);
       }
       return output;
     }
-    const executor = createSearchExecutor(db, query);
+    const executor = createSearchExecutor(db, absoluteRoot, query);
     if (options.noExplain) {
       return await runNoExplain(provider, query, limit, options, executor);
     }
@@ -3403,6 +3850,7 @@ function createWatcher(options, events) {
 // src/cli/commands/watch.ts
 var CTX_DIR5 = ".ctx";
 var DB_FILENAME4 = "index.db";
+var EMBEDDING_SAVE_BATCH_SIZE2 = 128;
 function timestamp() {
   return (/* @__PURE__ */ new Date()).toLocaleTimeString("en-GB", { hour12: false });
 }
@@ -3419,6 +3867,9 @@ async function hashFile(absolutePath) {
   const content = fs9.readFileSync(absolutePath);
   return createHash3("sha256").update(content).digest("hex");
 }
+function isSameEmbedderConfig2(a, b) {
+  return a.provider === b.provider && a.model === b.model && a.dimensions === b.dimensions;
+}
 async function reindexChanges(db, changes, projectPath, options) {
   const start = performance.now();
   const log = options.log;
@@ -3430,8 +3881,8 @@ async function reindexChanges(db, changes, projectPath, options) {
     const language = detectLanguage(change.path);
     if (change.type === "unlink") {
       log(`[${timestamp()}] Deleted: ${change.path}`);
-      const existingFile2 = db.getFile(change.path);
-      if (existingFile2) {
+      const existingFile = db.getFile(change.path);
+      if (existingFile) {
         db.deleteFile(change.path);
       }
       filesProcessed++;
@@ -3441,10 +3892,6 @@ async function reindexChanges(db, changes, projectPath, options) {
     if (!fs9.existsSync(absolutePath)) continue;
     const label = change.type === "add" ? "Added" : "Changed";
     log(`[${timestamp()}] ${label}: ${change.path}`);
-    const existingFile = db.getFile(change.path);
-    if (existingFile) {
-      db.deleteChunksByFile(existingFile.id);
-    }
     let nodes;
     try {
       nodes = await parseFile(absolutePath, language);
@@ -3455,26 +3902,31 @@ async function reindexChanges(db, changes, projectPath, options) {
     const chunks = chunkFile(nodes, change.path);
     const hash = await hashFile(absolutePath);
     const size = fs9.statSync(absolutePath).size;
-    const fileId = db.upsertFile({
-      path: change.path,
-      language,
-      hash,
-      size
+    const chunkRows = chunks.map((c) => ({
+      lineStart: c.lineStart,
+      lineEnd: c.lineEnd,
+      type: c.type,
+      name: c.name,
+      parent: c.parent,
+      text: c.text,
+      imports: c.imports,
+      exports: c.exports,
+      hash: c.hash
+    }));
+    let chunkIds = [];
+    db.transaction(() => {
+      const existingFile = db.getFile(change.path);
+      if (existingFile) {
+        db.deleteChunksByFile(existingFile.id);
+      }
+      const fileId = db.upsertFile({
+        path: change.path,
+        language,
+        hash,
+        size
+      });
+      chunkIds = db.insertChunks(fileId, chunkRows);
     });
-    const chunkIds = db.insertChunks(
-      fileId,
-      chunks.map((c) => ({
-        lineStart: c.lineStart,
-        lineEnd: c.lineEnd,
-        type: c.type,
-        name: c.name,
-        parent: c.parent,
-        text: c.text,
-        imports: c.imports,
-        exports: c.exports,
-        hash: c.hash
-      }))
-    );
     for (let i = 0; i < chunks.length; i++) {
       allChunksWithMeta.push({
         fileRelPath: change.path,
@@ -3485,25 +3937,46 @@ async function reindexChanges(db, changes, projectPath, options) {
     filesProcessed++;
   }
   if (!options.skipEmbedding && allChunksWithMeta.length > 0) {
-    const embedder = await loadEmbedder3();
-    const texts = allChunksWithMeta.map(
-      (cm) => prepareChunkText(cm.fileRelPath, cm.chunk.parent, cm.chunk.text)
-    );
-    const vectors = await embedder.embed(texts);
-    db.transaction(() => {
-      for (let i = 0; i < allChunksWithMeta.length; i++) {
-        const chunkDbId = parseInt(allChunksWithMeta[i].chunk.id, 10);
-        db.insertVector(chunkDbId, vectors[i]);
+    const embedder = await loadEmbedder3(projectPath);
+    let vectorsCreated = 0;
+    const total = allChunksWithMeta.length;
+    try {
+      for (let i = 0; i < allChunksWithMeta.length; i += EMBEDDING_SAVE_BATCH_SIZE2) {
+        const batch = allChunksWithMeta.slice(i, i + EMBEDDING_SAVE_BATCH_SIZE2);
+        const texts = batch.map(
+          (cm) => prepareChunkText(cm.fileRelPath, cm.chunk.parent, cm.chunk.text)
+        );
+        const vectors = await embedder.embed(texts);
+        db.transaction(() => {
+          for (let j = 0; j < batch.length; j++) {
+            const chunkDbId = parseInt(batch[j].chunk.id, 10);
+            db.insertVector(chunkDbId, vectors[j]);
+          }
+        });
+        vectorsCreated += vectors.length;
       }
-    });
+    } catch (err) {
+      throw new IndexError(
+        `Embedding failed after saving ${vectorsCreated}/${total} vectors. Continue watching, then run "ctx init" to backfill missing vectors. ${err instanceof Error ? err.message : String(err)}`,
+        ErrorCode.EMBEDDER_FAILED,
+        err instanceof Error ? err : void 0
+      );
+    }
   }
   const durationMs = performance.now() - start;
   return { filesProcessed, chunksUpdated, durationMs };
 }
 var embedderInstance3 = null;
-async function loadEmbedder3() {
-  if (embedderInstance3) return embedderInstance3;
-  embedderInstance3 = await createLocalEmbedder();
+var embedderKey3 = null;
+function getCacheKey3(projectPath) {
+  const config = getProjectEmbedderConfig(projectPath);
+  return `${projectPath}:${config.provider}:${config.model}:${config.dimensions}`;
+}
+async function loadEmbedder3(projectPath) {
+  const cacheKey = getCacheKey3(projectPath);
+  if (embedderInstance3 && embedderKey3 === cacheKey) return embedderInstance3;
+  embedderInstance3 = await createProjectEmbedder(projectPath);
+  embedderKey3 = cacheKey;
   return embedderInstance3;
 }
 async function runWatch(projectPath, options = {}) {
@@ -3520,8 +3993,29 @@ async function runWatch(projectPath, options = {}) {
     );
   }
   await initParser();
-  const db = createDatabase(dbPath);
+  const embedderConfig = getProjectEmbedderConfig(absoluteRoot);
+  const db = createDatabase(dbPath, embedderConfig.dimensions);
+  const existingEmbedder = db.getIndexEmbedder();
+  if (existingEmbedder) {
+    if (!isSameEmbedderConfig2(existingEmbedder, embedderConfig)) {
+      db.close();
+      throw new IndexError(
+        `Index embedder mismatch: index uses ${existingEmbedder.provider} (${existingEmbedder.model}, ${existingEmbedder.dimensions} dims) but config requests ${embedderConfig.provider} (${embedderConfig.model}, ${embedderConfig.dimensions} dims). Rebuild the index.`,
+        ErrorCode.CONFIG_INVALID
+      );
+    }
+  } else {
+    const isEmptyIndex = db.getFileCount() === 0 && db.getChunkCount() === 0 && db.getVectorCount() === 0;
+    if (isEmptyIndex) {
+      db.setIndexEmbedder({
+        provider: embedderConfig.provider,
+        model: embedderConfig.model,
+        dimensions: embedderConfig.dimensions
+      });
+    }
+  }
   let watcherHandle = null;
+  let reindexQueue = Promise.resolve();
   const watcher = createWatcher(
     {
       projectPath: absoluteRoot,
@@ -3530,7 +4024,7 @@ async function runWatch(projectPath, options = {}) {
     },
     {
       onChange: (changes) => {
-        void (async () => {
+        reindexQueue = reindexQueue.then(async () => {
           try {
             const result = await reindexChanges(db, changes, absoluteRoot, {
               skipEmbedding: options.skipEmbedding,
@@ -3546,7 +4040,7 @@ async function runWatch(projectPath, options = {}) {
               `[${timestamp()}] Error: ${err instanceof Error ? err.message : String(err)}`
             );
           }
-        })();
+        });
       },
       onError: (err) => {
         log(`[${timestamp()}] Watcher error: ${err.message}`);
@@ -3562,6 +4056,7 @@ async function runWatch(projectPath, options = {}) {
         await watcherHandle.stop();
         watcherHandle = null;
       }
+      await reindexQueue;
       db.close();
       log("Stopped watching. Database saved.");
     }
@@ -3623,6 +4118,7 @@ function readConfig2(ctxDir) {
     const parsed = JSON.parse(raw);
     const embedder = parsed.embedder;
     return {
+      provider: embedder?.provider ?? parsed.provider ?? "unknown",
       model: embedder?.model ?? parsed.model ?? "unknown",
       dimensions: embedder?.dimensions ?? parsed.dimensions ?? 0
     };
@@ -3663,15 +4159,54 @@ function formatStatus(projectPath, output) {
       lines.push(`    ${label}${count} file${count !== 1 ? "s" : ""}`);
     }
   }
-  if (output.config) {
-    lines.push("");
-    lines.push(
-      `  Embedder: local (${output.config.model}, ${output.config.dimensions} dims)`
-    );
+  const hasConfig = output.config !== null;
+  const hasIndexEmbedder = output.indexEmbedder !== null;
+  if (hasConfig || hasIndexEmbedder) lines.push("");
+  if (hasConfig && hasIndexEmbedder) {
+    const config = output.config;
+    const indexEmbedder = output.indexEmbedder;
+    if (!config || !indexEmbedder) {
+      lines.push("  Embedder: unknown");
+    } else if (isSameEmbedder(config, indexEmbedder)) {
+      lines.push(
+        `  Embedder: ${indexEmbedder.provider} (${indexEmbedder.model}, ${indexEmbedder.dimensions} dims)`
+      );
+    } else {
+      lines.push(
+        `  Index embedder:  ${indexEmbedder.provider} (${indexEmbedder.model}, ${indexEmbedder.dimensions} dims)`
+      );
+      lines.push(
+        `  Config embedder: ${config.provider} (${config.model}, ${config.dimensions} dims)`
+      );
+    }
+  } else if (hasIndexEmbedder) {
+    const indexEmbedder = output.indexEmbedder;
+    if (!indexEmbedder) {
+      lines.push("  Index embedder: unknown");
+    } else {
+      lines.push(
+        `  Index embedder: ${indexEmbedder.provider} (${indexEmbedder.model}, ${indexEmbedder.dimensions} dims)`
+      );
+    }
+  } else if (hasConfig) {
+    const config = output.config;
+    if (!config) {
+      lines.push("  Config embedder: unknown");
+    } else {
+      lines.push(
+        `  Config embedder: ${config.provider} (${config.model}, ${config.dimensions} dims)`
+      );
+    }
+  }
+  if (output.embedderWarning) {
+    lines.push(`  Warning: ${output.embedderWarning}`);
   }
   lines.push("");
   return lines.join("\n");
 }
+function isSameEmbedder(a, b) {
+  return a.provider === b.provider && a.model === b.model && a.dimensions === b.dimensions;
+}
 async function runStatus(projectPath) {
   const absoluteRoot = path10.resolve(projectPath);
   const ctxDir = path10.join(absoluteRoot, CTX_DIR6);
@@ -3686,6 +4221,8 @@ async function runStatus(projectPath) {
       lastIndexed: null,
       languages: /* @__PURE__ */ new Map(),
       config: null,
+      indexEmbedder: null,
+      embedderWarning: null,
       text: formatNotInitialized(absoluteRoot)
     };
     return output;
@@ -3698,7 +4235,9 @@ async function runStatus(projectPath) {
     const languages = db.getLanguageBreakdown();
     const lastIndexed = db.getLastIndexed();
     const config = readConfig2(ctxDir);
+    const indexEmbedder = db.getIndexEmbedder();
     const dbSizeBytes = fs10.statSync(dbPath).size;
+    const embedderWarning = config && indexEmbedder && !isSameEmbedder(config, indexEmbedder) ? `Index built with ${indexEmbedder.provider} (${indexEmbedder.dimensions} dims), config requests ${config.provider} (${config.dimensions} dims) \u2014 rebuild needed.` : null;
     const output = {
       initialized: true,
       fileCount,
@@ -3708,6 +4247,8 @@ async function runStatus(projectPath) {
       lastIndexed,
       languages,
       config,
+      indexEmbedder,
+      embedderWarning,
       text: ""
     };
     output.text = formatStatus(absoluteRoot, output);