nlm-memory 0.20.0 → 0.21.0

package/dist/core/embedding/embed-backfill.js

@@ -1,7 +1,6 @@
 /**
  * embed-backfill — re-embed every session in canonical.sqlite into the
- * chunk + max-pool index (session_embedding_chunks). Replaces the prior
- * one-vector-per-session backfill that wrote to session_embeddings.
+ * chunk + max-pool index (session_embedding_chunks).
  *
  * For each session: chunk (label + summary + body) via chunkSessionText,
  * embed each chunk with kind="document", and write to the chunk table +
@@ -12,6 +11,12 @@
  * session ids. A session is considered "done" only when ALL its chunks
  * embed successfully — partial sessions are retried on the next run.
  *
+ * When the probed embedder dim/model/provider differs from the stored
+ * embedding_config prose row, both vec tables are dropped and recreated at
+ * the new dim before the session loop runs. Facts (superseded_by IS NULL
+ * AND retired_at IS NULL) are reembedded immediately after the session
+ * loop. Same-config reruns skip the rebuild and reembed sessions only.
+ *
  * Layering: depends on the LLMClient port. SQLite touched directly via
  * better-sqlite3 because this is a one-shot operational tool, not a hot
  * path. Lives under core/ but is invoked from the CLI composition root.
@@ -23,22 +28,26 @@ import Database from "better-sqlite3";
 import * as sqliteVec from "sqlite-vec";
 import { LLMUnreachableError } from "../../ports/llm-client.js";
 import { chunkSessionText } from "../embedding/chunk-body.js";
+import { SqliteEmbeddingConfigStore } from "../storage/sqlite-embedding-config.js";
 const DEFAULT_STATE_PATH = join(homedir(), ".nlm", "embed_reembed.state");
 const SAVE_EVERY = 25;
-function loadState(path) {
+function loadStateData(path) {
     if (!existsSync(path))
-        return new Set();
+        return { done: new Set() };
     try {
-        const data = JSON.parse(readFileSync(path, "utf8"));
-        return new Set(data.done ?? []);
+        const raw = JSON.parse(readFileSync(path, "utf8"));
+        return { done: new Set(raw.done ?? []), ...(raw.config ? { config: raw.config } : {}) };
     }
     catch {
-        return new Set();
+        return { done: new Set() };
     }
 }
-function saveState(path, done) {
+function saveState(path, done, config) {
     mkdirSync(dirname(path), { recursive: true });
-    writeFileSync(path, JSON.stringify({ done: [...done].sort() }));
+    const data = { done: [...done].sort() };
+    if (config)
+        data.config = config;
+    writeFileSync(path, JSON.stringify(data));
 }
 export async function reembedCorpus(opts) {
     const statePath = opts.statePath ?? DEFAULT_STATE_PATH;
@@ -47,31 +56,126 @@ export async function reembedCorpus(opts) {
     }
     const db = new Database(opts.dbPath);
     sqliteVec.load(db);
-    // Backfill every session with content; live ingest covers ongoing writes.
-    // The state file dedupes across runs so partial completion resumes cleanly.
-    const sql = "SELECT s.id, s.label, s.summary, s.body FROM sessions s " +
-        "WHERE s.body IS NOT NULL OR s.summary IS NOT NULL OR s.label IS NOT NULL " +
-        "ORDER BY s.started_at" +
-        (opts.limit ? ` LIMIT ${Math.trunc(opts.limit)}` : "");
-    const rows = db.prepare(sql).all();
-    const total = rows.length;
-    const done = loadState(statePath);
-    const selectChunks = db.prepare("SELECT chunk_id FROM session_chunk_map WHERE session_id = ?");
-    const delChunks = (sessionId) => {
-        const existing = selectChunks.all(sessionId);
-        if (existing.length === 0)
-            return;
-        const placeholders = existing.map(() => "?").join(",");
-        const ids = existing.map((r) => r.chunk_id);
-        db.prepare(`DELETE FROM session_embedding_chunks WHERE chunk_id IN (${placeholders})`).run(...ids);
-        db.prepare("DELETE FROM session_chunk_map WHERE session_id = ?").run(sessionId);
-    };
-    const insChunk = db.prepare("INSERT INTO session_embedding_chunks (embedding, session_id, chunk_idx) VALUES (?, ?, ?)");
-    const insMap = db.prepare("INSERT INTO session_chunk_map (chunk_id, session_id, chunk_idx) VALUES (?, ?, ?)");
-    let succeeded = 0;
-    let failed = 0;
-    let skipped = 0;
     try {
+        const configTableRow = db
+            .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='embedding_config'")
+            .get();
+        if (!configTableRow) {
+            throw new Error("embedding_config table not found; run `nlm migrate` to apply pending migrations");
+        }
+        const probeResult = await opts.embedder.embed("nlm dimension probe", "query");
+        const dim = probeResult.vector.length;
+        const model = probeResult.model;
+        const provider = opts.embedderProvider ?? process.env["NLM_EMBED_PROVIDER"] ?? "ollama";
+        const runtimeConfig = { lane: "prose", provider, model, dim };
+        const configStore = new SqliteEmbeddingConfigStore(db);
+        const storedConfig = configStore.getLane("prose");
+        const stateData = loadStateData(statePath);
+        let configMatch = false;
+        let needsRebuild = false;
+        if (storedConfig !== null) {
+            configMatch =
+                storedConfig.provider === runtimeConfig.provider &&
+                    storedConfig.model === runtimeConfig.model &&
+                    storedConfig.dim === runtimeConfig.dim;
+            if (!configMatch) {
+                // An interrupted rebuild already dropped+recreated the tables if the
+                // state file carries a matching config. Resume without re-dropping.
+                const stateConfigMatches = stateData.config !== undefined &&
+                    stateData.config.provider === runtimeConfig.provider &&
+                    stateData.config.model === runtimeConfig.model &&
+                    stateData.config.dim === runtimeConfig.dim;
+                if (!stateConfigMatches) {
+                    needsRebuild = true;
+                }
+            }
+        }
+        // Resume state: stored config differs from runtime, but the state file already
+        // carries the new config (tables were rebuilt in a prior interrupted run).
+        // No re-drop is needed, but facts must still be reembedded to finish the run.
+        const isResumeState = storedConfig !== null && !configMatch && !needsRebuild;
+        if (opts.dryRun) {
+            if (needsRebuild) {
+                const stored = storedConfig;
+                process.stderr.write(`[embed-backfill] dim mismatch: stored ${stored.provider}/${stored.model}@${stored.dim},` +
+                    ` runtime ${provider}/${model}@${dim}.` +
+                    ` Would DROP session_embedding_chunks, session_chunk_map entries, fact_embeddings.\n`);
+            }
+            else if (isResumeState) {
+                const stored = storedConfig;
+                process.stderr.write(`[embed-backfill] resume state: interrupted rebuild detected.` +
+                    ` Stored config ${stored.provider}/${stored.model}@${stored.dim}` +
+                    ` differs from runtime ${provider}/${model}@${dim},` +
+                    ` but state file already matches runtime.` +
+                    ` No re-drop; facts would be reembedded to complete the interrupted rebuild.\n`);
+            }
+            else {
+                process.stderr.write(`[embed-backfill] dry-run: config matches (${provider}/${model}@${dim}), no rebuild needed.\n`);
+            }
+            let dryRunFactCount = 0;
+            if (isResumeState) {
+                dryRunFactCount = db
+                    .prepare("SELECT id FROM facts WHERE superseded_by IS NULL AND retired_at IS NULL")
+                    .all().length;
+            }
+            return {
+                total: 0,
+                processed: 0,
+                succeeded: 0,
+                failed: 0,
+                skippedAlreadyDone: 0,
+                dbMissing: false,
+                dryRun: true,
+                rebuilt: needsRebuild,
+                factsReembedded: dryRunFactCount,
+            };
+        }
+        if (needsRebuild) {
+            const stored = storedConfig;
+            process.stderr.write(`[embed-backfill] rebuilding vec tables: stored ${stored.provider}/${stored.model}@${stored.dim}` +
+                ` vs runtime ${provider}/${model}@${dim}\n`);
+            process.stderr.write(`[embed-backfill] DROP TABLE IF EXISTS session_embedding_chunks\n`);
+            db.exec("DROP TABLE IF EXISTS session_embedding_chunks");
+            db.exec(`CREATE VIRTUAL TABLE session_embedding_chunks USING vec0(` +
+                `chunk_id INTEGER PRIMARY KEY, ` +
+                `embedding float[${dim}], ` +
+                `+session_id TEXT, ` +
+                `+chunk_idx INTEGER` +
+                `)`);
+            db.exec("DELETE FROM session_chunk_map");
+            process.stderr.write(`[embed-backfill] DROP TABLE IF EXISTS fact_embeddings\n`);
+            db.exec("DROP TABLE IF EXISTS fact_embeddings");
+            db.exec(`CREATE VIRTUAL TABLE fact_embeddings USING vec0(` +
+                `fact_id TEXT PRIMARY KEY, ` +
+                `embedding float[${dim}]` +
+                `)`);
+            // Ignore the stale state file; start fresh for this config.
+            stateData.done = new Set();
+        }
+        // Backfill every session with content; live ingest covers ongoing writes.
+        // The state file dedupes across runs so partial completion resumes cleanly.
+        const sql = "SELECT s.id, s.label, s.summary, s.body FROM sessions s " +
+            "WHERE s.body IS NOT NULL OR s.summary IS NOT NULL OR s.label IS NOT NULL " +
+            "ORDER BY s.started_at" +
+            (opts.limit ? ` LIMIT ${Math.trunc(opts.limit)}` : "");
+        const rows = db.prepare(sql).all();
+        const total = rows.length;
+        const done = stateData.done;
+        const selectChunks = db.prepare("SELECT chunk_id FROM session_chunk_map WHERE session_id = ?");
+        const delChunks = (sessionId) => {
+            const existing = selectChunks.all(sessionId);
+            if (existing.length === 0)
+                return;
+            const placeholders = existing.map(() => "?").join(",");
+            const ids = existing.map((r) => r.chunk_id);
+            db.prepare(`DELETE FROM session_embedding_chunks WHERE chunk_id IN (${placeholders})`).run(...ids);
+            db.prepare("DELETE FROM session_chunk_map WHERE session_id = ?").run(sessionId);
+        };
+        const insChunk = db.prepare("INSERT INTO session_embedding_chunks (embedding, session_id, chunk_idx) VALUES (?, ?, ?)");
+        const insMap = db.prepare("INSERT INTO session_chunk_map (chunk_id, session_id, chunk_idx) VALUES (?, ?, ?)");
+        let succeeded = 0;
+        let failed = 0;
+        let skipped = 0;
         for (let i = 0; i < rows.length; i++) {
             const row = rows[i];
             const idx = i + 1;
@@ -143,20 +247,54 @@ export async function reembedCorpus(opts) {
                 : `PARTIAL (${vectors.length}/${chunks.length} chunks, ${chunkSkipped} skipped)`;
             opts.onProgress?.(idx, total, row.id, status);
             if (succeeded % SAVE_EVERY === 0)
-                saveState(statePath, done);
+                saveState(statePath, done, { provider, model, dim });
+        }
+        // Reembed facts whenever the stored config does not match the runtime config
+        // (storedConfig exists but configMatch is false). This covers both fresh
+        // rebuilds and interrupted rebuilds that resume: after a rebuild the tables
+        // are empty and the state file carries the new config, but the embedding_config
+        // row is only updated on successful completion, so storedConfig still holds
+        // the old values. Gating on needsRebuild alone skips facts in that resume path.
+        let factsReembedded = 0;
+        if (storedConfig !== null && !configMatch) {
+            const factRows = db
+                .prepare("SELECT id, subject, predicate, value FROM facts" +
+                " WHERE superseded_by IS NULL AND retired_at IS NULL")
+                .all();
+            const delFactEmb = db.prepare("DELETE FROM fact_embeddings WHERE fact_id = ?");
+            const insFactEmb = db.prepare("INSERT INTO fact_embeddings (fact_id, embedding) VALUES (?, ?)");
+            for (const fact of factRows) {
+                const factText = `${fact.subject} ${fact.predicate} ${fact.value}`.trim();
+                if (!factText)
+                    continue;
+                try {
+                    const out = await opts.embedder.embed(factText, "document");
+                    const blob = Buffer.from(out.vector.buffer, out.vector.byteOffset, out.vector.byteLength);
+                    delFactEmb.run(fact.id);
+                    insFactEmb.run(fact.id, blob);
+                    factsReembedded += 1;
+                }
+                catch {
+                    // Best-effort: fact row survives; semantic recall misses it until re-run.
+                }
+            }
         }
-        saveState(statePath, done);
+        // Upsert prose config row on successful completion and persist final state.
+        configStore.upsertLane(runtimeConfig, new Date().toISOString());
+        saveState(statePath, done, { provider, model, dim });
+        return {
+            total,
+            processed: succeeded + failed + skipped,
+            succeeded,
+            failed,
+            skippedAlreadyDone: skipped,
+            dbMissing: false,
+            rebuilt: needsRebuild,
+            factsReembedded,
+        };
     }
     finally {
         db.close();
     }
-    return {
-        total,
-        processed: succeeded + failed + skipped,
-        succeeded,
-        failed,
-        skippedAlreadyDone: skipped,
-        dbMissing: false,
-    };
 }
 //# sourceMappingURL=embed-backfill.js.map

package/dist/core/embedding/embed-backfill.js.map

@@ -1 +1 @@
-{"version":3,"file":"embed-backfill.js","sourceRoot":"","sources":["../../../src/core/embedding/embed-backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAEjE,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,CAAC;AAC1E,MAAM,UAAU,GAAG,EAAE,CAAC;AA0BtB,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAwB,CAAC;QAC3E,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,IAAiB;IAChD,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;AAClE,CAAC;AAGD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAqB;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACrG,CAAC;IAED,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEnB,0EAA0E;IAC1E,4EAA4E;IAC5E,MAAM,GAAG,GACP,0DAA0D;QAC1D,2EAA2E;QAC3E,uBAAuB;QACvB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAiB,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IACnD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IAE1B,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IAElC,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAC7B,6DAA6D,CAC9D,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,SAAiB,EAAQ,EAAE;QAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAClC,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC5C,EAAE,CAAC,OAAO,CACR,2DAA2D,YAAY,GAAG,CAC3E,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;QACd,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAClF,CAAC,CAAC;IACF,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CACzB,0FAA0F,CAC3F,CAAC;IACF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,kFAAkF,CACnF,CAAC;IAEF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,IAAI,CAAC;QACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;YACrB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACrB,OAAO,IAAI,CAAC,CAAC;gBACb,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,gBAAgB,CAAC;gBAC9B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,IAAI,EAAE,GAAG,CAAC,IAAI;aACf,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,CAAC;gBACZ,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;gBACxD,SAAS;YACX,CAAC;YAED,qEAAqE;YACrE,kEAAkE;YAClE,kEAAkE;YAClE,kEAAkE;YAClE,uDAAuD;YACvD,MAAM,OAAO,GAAyC,EAAE,CAAC;YACzD,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBACzB,IAAI,OAAgB,CAAC;gBACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;oBAC7C,IAAI,CAAC;wBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;wBACzD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;wBAC1C,OAAO,GAAG,SAAS,CAAC;wBACpB,MAAM;oBACR,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,CAAC,CAAC;wBACZ,IAAI,CAAC,CAAC,CAAC,YAAY,mBAAmB,CAAC;4BAAE,MAAM,CAAC,CAAC;wBACjD,IAAI,OAAO,KAAK,CAAC;4BAAE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;gBACD,IAAI,OAAO,KAAK,SAAS;oBAAE,YAAY,IAAI,CAAC,CAAC;YAC/C,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,CAAC;gBACZ,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,mBAAmB,YAAY,IAAI,MAAM,CAAC,MAAM,UAAU,CAAC,CAAC;gBAClG,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAClB,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;oBACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;oBACrE,kEAAkE;oBAClE,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;oBACtD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBACzD,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,IAAI,CAAC,CAAC;gBACZ,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,cAAe,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC5E,SAAS;YACX,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjB,SAAS,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GACV,YAAY,KAAK,CAAC;gBAChB,CAAC,CAAC,OAAO,OAAO,CAAC,MAAM,UAAU;gBACjC,CAAC,CAAC,YAAY,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,YAAY,YAAY,WAAW,CAAC;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAC9C,IAAI,SAAS,GAAG,UAAU,KAAK,CAAC;gBAAE,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC/D,CAAC;QACD,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAED,OAAO;QACL,KAAK;QACL,SAAS,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO;QACvC,SAAS;QACT,MAAM;QACN,kBAAkB,EAAE,OAAO;QAC3B,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC"}
+{"version":3,"file":"embed-backfill.js","sourceRoot":"","sources":["../../../src/core/embedding/embed-backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,EAAE,0BAA0B,EAAE,MAAM,0CAA0C,CAAC;AAGtF,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,CAAC;AAC1E,MAAM,UAAU,GAAG,EAAE,CAAC;AAkDtB,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IAClD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAGhD,CAAC;QACF,OAAO,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAC1F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,IAAiB,EAAE,MAAoB;IACtE,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,IAAI,GAA6C,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IAClF,IAAI,MAAM;QAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACjC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC;AAGD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAqB;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACrG,CAAC;IAED,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEnB,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,EAAE;aACtB,OAAO,CACN,+EAA+E,CAChF;aACA,GAAG,EAAE,CAAC;QACT,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAC9E,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;QACtC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,QAAQ,CAAC;QACxF,MAAM,aAAa,GAAwB,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QAEnF,MAAM,WAAW,GAAG,IAAI,0BAA0B,CAAC,EAAE,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAElD,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;QAE3C,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,WAAW;gBACT,YAAY,CAAC,QAAQ,KAAK,aAAa,CAAC,QAAQ;oBAChD,YAAY,CAAC,KAAK,KAAK,aAAa,CAAC,KAAK;oBAC1C,YAAY,CAAC,GAAG,KAAK,aAAa,CAAC,GAAG,CAAC;YACzC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,qEAAqE;gBACrE,oEAAoE;gBACpE,MAAM,kBAAkB,GACtB,SAAS,CAAC,MAAM,KAAK,SAAS;oBAC9B,SAAS,CAAC,MAAM,CAAC,QAAQ,KAAK,aAAa,CAAC,QAAQ;oBACpD,SAAS,CAAC,MAAM,CAAC,KAAK,KAAK,aAAa,CAAC,KAAK;oBAC9C,SAAS,CAAC,MAAM,CAAC,GAAG,KAAK,aAAa,CAAC,GAAG,CAAC;gBAC7C,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBACxB,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;QAED,+EAA+E;QAC/E,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,aAAa,GAAG,YAAY,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,YAAY,CAAC;QAE7E,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,MAAM,GAAG,YAAa,CAAC;gBAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,yCAAyC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,GAAG;oBACzF,YAAY,QAAQ,IAAI,KAAK,IAAI,GAAG,GAAG;oBACvC,qFAAqF,CACtF,CAAC;YACJ,CAAC;iBAAM,IAAI,aAAa,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,YAAa,CAAC;gBAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,8DAA8D;oBAC9D,kBAAkB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,EAAE;oBACjE,yBAAyB,QAAQ,IAAI,KAAK,IAAI,GAAG,GAAG;oBACpD,0CAA0C;oBAC1C,+EAA+E,CAChF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6CAA6C,QAAQ,IAAI,KAAK,IAAI,GAAG,yBAAyB,CAC/F,CAAC;YACJ,CAAC;YACD,IAAI,eAAe,GAAG,CAAC,CAAC;YACxB,IAAI,aAAa,EAAE,CAAC;gBAClB,eAAe,GAAG,EAAE;qBACjB,OAAO,CACN,yEAAyE,CAC1E;qBACA,GAAG,EAAE,CAAC,MAAM,CAAC;YAClB,CAAC;YACD,OAAO;gBACL,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC;gBACT,kBAAkB,EAAE,CAAC;gBACrB,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,YAAY;gBACrB,eAAe,EAAE,eAAe;aACjC,CAAC;QACJ,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,MAAM,GAAG,YAAa,CAAC;YAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,kDAAkD,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,EAAE;gBACjG,eAAe,QAAQ,IAAI,KAAK,IAAI,GAAG,IAAI,CAC5C,CAAC;YACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kEAAkE,CAAC,CAAC;YACzF,EAAE,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YACzD,EAAE,CAAC,IAAI,CACL,2DAA2D;gBAC3D,gCAAgC;gBAChC,mBAAmB,GAAG,KAAK;gBAC3B,oBAAoB;gBACpB,oBAAoB;gBACpB,GAAG,CACJ,CAAC;YACF,EAAE,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YACzC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;YAChF,EAAE,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;YAChD,EAAE,CAAC,IAAI,CACL,kDAAkD;gBAClD,4BAA4B;gBAC5B,mBAAmB,GAAG,GAAG;gBACzB,GAAG,CACJ,CAAC;YACF,4DAA4D;YAC5D,SAAS,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QAC7B,CAAC;QAED,0EAA0E;QAC1E,4EAA4E;QAC5E,MAAM,GAAG,GACP,0DAA0D;YAC1D,2EAA2E;YAC3E,uBAAuB;YACvB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAiB,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAE1B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;QAE5B,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAC7B,6DAA6D,CAC9D,CAAC;QACF,MAAM,SAAS,GAAG,CAAC,SAAiB,EAAQ,EAAE;YAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAClC,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC5C,EAAE,CAAC,OAAO,CACR,2DAA2D,YAAY,GAAG,CAC3E,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;YACd,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClF,CAAC,CAAC;QACF,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CACzB,0FAA0F,CAC3F,CAAC;QACF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,kFAAkF,CACnF,CAAC;QAEF,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;YACrB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACrB,OAAO,IAAI,CAAC,CAAC;gBACb,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,gBAAgB,CAAC;gBAC9B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,IAAI,EAAE,GAAG,CAAC,IAAI;aACf,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,CAAC;gBACZ,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;gBACxD,SAAS;YACX,CAAC;YAED,qEAAqE;YACrE,kEAAkE;YAClE,kEAAkE;YAClE,kEAAkE;YAClE,uDAAuD;YACvD,MAAM,OAAO,GAAyC,EAAE,CAAC;YACzD,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBACzB,IAAI,OAAgB,CAAC;gBACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;oBAC7C,IAAI,CAAC;wBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;wBACzD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;wBAC1C,OAAO,GAAG,SAAS,CAAC;wBACpB,MAAM;oBACR,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,GAAG,CAAC,CAAC;wBACZ,IAAI,CAAC,CAAC,CAAC,YAAY,mBAAmB,CAAC;4BAAE,MAAM,CAAC,CAAC;wBACjD,IAAI,OAAO,KAAK,CAAC;4BAAE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;gBACD,IAAI,OAAO,KAAK,SAAS;oBAAE,YAAY,IAAI,CAAC,CAAC;YAC/C,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,CAAC;gBACZ,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,mBAAmB,YAAY,IAAI,MAAM,CAAC,MAAM,UAAU,CAAC,CAAC;gBAClG,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAClB,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;oBACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;oBACrE,kEAAkE;oBAClE,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;oBACtD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBACzD,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,IAAI,CAAC,CAAC;gBACZ,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,cAAe,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC5E,SAAS;YACX,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjB,SAAS,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GACV,YAAY,KAAK,CAAC;gBAChB,CAAC,CAAC,OAAO,OAAO,CAAC,MAAM,UAAU;gBACjC,CAAC,CAAC,YAAY,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,YAAY,YAAY,WAAW,CAAC;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAC9C,IAAI,SAAS,GAAG,UAAU,KAAK,CAAC;gBAAE,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACzF,CAAC;QAED,6EAA6E;QAC7E,yEAAyE;QACzE,4EAA4E;QAC5E,gFAAgF;QAChF,4EAA4E;QAC5E,gFAAgF;QAChF,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,YAAY,KAAK,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,EAAE;iBAChB,OAAO,CACN,iDAAiD;gBACjD,qDAAqD,CACtD;iBACA,GAAG,EAAE,CAAC;YACT,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,+CAA+C,CAAC,CAAC;YAC/E,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,gEAAgE,CACjE,CAAC;YACF,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;gBAC1E,IAAI,CAAC,QAAQ;oBAAE,SAAS;gBACxB,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;oBAC1F,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACxB,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBAC9B,eAAe,IAAI,CAAC,CAAC;gBACvB,CAAC;gBAAC,MAAM,CAAC;oBACP,0EAA0E;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;QAED,4EAA4E;QAC5E,WAAW,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAChE,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAErD,OAAO;YACL,KAAK;YACL,SAAS,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO;YACvC,SAAS;YACT,MAAM;YACN,kBAAkB,EAAE,OAAO;YAC3B,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,YAAY;YACrB,eAAe;SAChB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC"}

package/dist/core/embedding/embedding-config.d.ts

@@ -0,0 +1,17 @@
+export type EmbeddingLane = "prose" | "code";
+export interface EmbeddingLaneConfig {
+    readonly lane: EmbeddingLane;
+    readonly provider: string;
+    readonly model: string;
+    readonly dim: number;
+}
+export interface EmbeddingConfigStore {
+    getLane(lane: EmbeddingLane): EmbeddingLaneConfig | null;
+    upsertLane(cfg: EmbeddingLaneConfig, updatedAtIso: string): void;
+}
+export type LaneReconcileState = "recorded" | "match" | "stale";
+export interface LaneReconcileResult {
+    readonly state: LaneReconcileState;
+    readonly stored: EmbeddingLaneConfig | null;
+}
+export declare function reconcileLane(store: EmbeddingConfigStore, runtime: EmbeddingLaneConfig, nowIso: string): LaneReconcileResult;

package/dist/core/embedding/embedding-config.js

@@ -0,0 +1,12 @@
+export function reconcileLane(store, runtime, nowIso) {
+    const stored = store.getLane(runtime.lane);
+    if (!stored) {
+        store.upsertLane(runtime, nowIso);
+        return { state: "recorded", stored: null };
+    }
+    const match = stored.provider === runtime.provider &&
+        stored.model === runtime.model &&
+        stored.dim === runtime.dim;
+    return { state: match ? "match" : "stale", stored };
+}
+//# sourceMappingURL=embedding-config.js.map

package/dist/core/embedding/embedding-config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-config.js","sourceRoot":"","sources":["../../../src/core/embedding/embedding-config.ts"],"names":[],"mappings":"AAqBA,MAAM,UAAU,aAAa,CAC3B,KAA2B,EAC3B,OAA4B,EAC5B,MAAc;IAEd,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAClC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,KAAK,GACT,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ;QACpC,MAAM,CAAC,KAAK,KAAK,OAAO,CAAC,KAAK;QAC9B,MAAM,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC;IAC7B,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;AACtD,CAAC"}

package/dist/core/embedding/pg-embed-backfill.d.ts

@@ -0,0 +1,31 @@
+/**
+ * pg-embed-backfill -- re-embed every session in the Postgres store into
+ * session_embedding_chunks. Mirrors reembedCorpus from embed-backfill.ts
+ * but drives a pg Pool instead of a SQLite file.
+ *
+ * Resumable via a JSON state file at $NLM_EMBED_STATE (default
+ * ~/.nlm/embed_reembed_pg.state). Same SAVE_EVERY = 25 checkpoint cadence.
+ *
+ * When the probed dim/model/provider differs from the stored embedding_config
+ * row, both vector tables are rebuilt: rows cleared, the embedding column
+ * altered to the new dimension, and indexes recreated. Facts (superseded_by
+ * IS NULL AND retired_at IS NULL) are reembedded after the session loop.
+ * Same-config reruns skip the rebuild.
+ *
+ * CRITICAL: the final config write is an awaited pool.query() INSERT ON
+ * CONFLICT -- never routed through PgEmbeddingConfigStore.upsertLane, which
+ * is fire-and-forget and can be lost when pool.end() follows.
+ */
+import type { LLMClient } from "../../ports/llm-client.js";
+import type { BackfillReport } from "../embedding/embed-backfill.js";
+export type { BackfillReport };
+export interface PgBackfillOptions {
+    readonly pgUrl: string;
+    readonly embedder: LLMClient;
+    readonly statePath?: string;
+    readonly limit?: number;
+    readonly dryRun?: boolean;
+    readonly embedderProvider?: string;
+    readonly onProgress?: (i: number, total: number, sid: string, status: string) => void;
+}
+export declare function reembedCorpusPg(opts: PgBackfillOptions): Promise<BackfillReport>;

package/dist/core/embedding/pg-embed-backfill.js

@@ -0,0 +1,264 @@
+/**
+ * pg-embed-backfill -- re-embed every session in the Postgres store into
+ * session_embedding_chunks. Mirrors reembedCorpus from embed-backfill.ts
+ * but drives a pg Pool instead of a SQLite file.
+ *
+ * Resumable via a JSON state file at $NLM_EMBED_STATE (default
+ * ~/.nlm/embed_reembed_pg.state). Same SAVE_EVERY = 25 checkpoint cadence.
+ *
+ * When the probed dim/model/provider differs from the stored embedding_config
+ * row, both vector tables are rebuilt: rows cleared, the embedding column
+ * altered to the new dimension, and indexes recreated. Facts (superseded_by
+ * IS NULL AND retired_at IS NULL) are reembedded after the session loop.
+ * Same-config reruns skip the rebuild.
+ *
+ * CRITICAL: the final config write is an awaited pool.query() INSERT ON
+ * CONFLICT -- never routed through PgEmbeddingConfigStore.upsertLane, which
+ * is fire-and-forget and can be lost when pool.end() follows.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { homedir } from "node:os";
+import { Pool } from "pg";
+import { LLMUnreachableError } from "../../ports/llm-client.js";
+import { chunkSessionText } from "../embedding/chunk-body.js";
+const DEFAULT_STATE_PATH = join(homedir(), ".nlm", "embed_reembed_pg.state");
+const SAVE_EVERY = 25;
+function loadStateData(path) {
+    if (!existsSync(path))
+        return { done: new Set() };
+    try {
+        const raw = JSON.parse(readFileSync(path, "utf8"));
+        return { done: new Set(raw.done ?? []), ...(raw.config ? { config: raw.config } : {}) };
+    }
+    catch {
+        return { done: new Set() };
+    }
+}
+function saveState(path, done, config) {
+    mkdirSync(dirname(path), { recursive: true });
+    const data = { done: [...done].sort() };
+    if (config)
+        data.config = config;
+    writeFileSync(path, JSON.stringify(data));
+}
+export async function reembedCorpusPg(opts) {
+    const statePath = opts.statePath ?? DEFAULT_STATE_PATH;
+    const pool = new Pool({ connectionString: opts.pgUrl });
+    try {
+        const tableCheck = await pool.query("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'embedding_config') AS exists");
+        if (!tableCheck.rows[0]?.exists) {
+            throw new Error("embedding_config table not found; run `nlm migrate` to apply pending migrations");
+        }
+        const probeResult = await opts.embedder.embed("nlm dimension probe", "query");
+        const dim = probeResult.vector.length;
+        const model = probeResult.model;
+        const provider = opts.embedderProvider ?? process.env["NLM_EMBED_PROVIDER"] ?? "ollama";
+        const cfgRows = await pool.query("SELECT provider, model, dim FROM embedding_config WHERE lane = 'prose'");
+        const storedConfig = cfgRows.rows[0] ?? null;
+        const stateData = loadStateData(statePath);
+        let configMatch = false;
+        let needsRebuild = false;
+        if (storedConfig !== null) {
+            configMatch =
+                storedConfig.provider === provider &&
+                    storedConfig.model === model &&
+                    storedConfig.dim === dim;
+            if (!configMatch) {
+                const stateConfigMatches = stateData.config !== undefined &&
+                    stateData.config.provider === provider &&
+                    stateData.config.model === model &&
+                    stateData.config.dim === dim;
+                if (!stateConfigMatches) {
+                    needsRebuild = true;
+                }
+            }
+        }
+        // Resume state: stored config differs from runtime, but the state file already
+        // carries the new config (tables were rebuilt in a prior interrupted run).
+        // No column alter is needed, but facts must still be reembedded to finish the run.
+        const isResumeState = storedConfig !== null && !configMatch && !needsRebuild;
+        if (opts.dryRun) {
+            if (needsRebuild) {
+                const stored = storedConfig;
+                process.stderr.write(`[pg-embed-backfill] dim mismatch: stored ${stored.provider}/${stored.model}@${stored.dim},` +
+                    ` runtime ${provider}/${model}@${dim}.` +
+                    ` Would ALTER session_embedding_chunks and fact_embeddings to vector(${dim}).\n`);
+            }
+            else if (isResumeState) {
+                const stored = storedConfig;
+                process.stderr.write(`[pg-embed-backfill] resume state: interrupted rebuild detected.` +
+                    ` Stored config ${stored.provider}/${stored.model}@${stored.dim}` +
+                    ` differs from runtime ${provider}/${model}@${dim},` +
+                    ` but state file already matches runtime.` +
+                    ` No column alter; facts would be reembedded to complete the interrupted rebuild.\n`);
+            }
+            else {
+                process.stderr.write(`[pg-embed-backfill] dry-run: config matches (${provider}/${model}@${dim}), no rebuild needed.\n`);
+            }
+            let dryRunFactCount = 0;
+            if (isResumeState) {
+                const countResult = await pool.query("SELECT COUNT(*) AS cnt FROM facts WHERE superseded_by IS NULL AND retired_at IS NULL");
+                dryRunFactCount = Number(countResult.rows[0]?.cnt ?? 0);
+            }
+            return {
+                total: 0,
+                processed: 0,
+                succeeded: 0,
+                failed: 0,
+                skippedAlreadyDone: 0,
+                dbMissing: false,
+                dryRun: true,
+                rebuilt: needsRebuild,
+                factsReembedded: dryRunFactCount,
+            };
+        }
+        if (needsRebuild) {
+            const stored = storedConfig;
+            process.stderr.write(`[pg-embed-backfill] rebuilding vec tables: stored ${stored.provider}/${stored.model}@${stored.dim}` +
+                ` vs runtime ${provider}/${model}@${dim}\n`);
+            process.stderr.write(`[pg-embed-backfill] DELETE FROM session_embedding_chunks\n`);
+            await pool.query("DELETE FROM session_embedding_chunks");
+            await pool.query("DROP INDEX IF EXISTS session_chunks_embedding_idx");
+            await pool.query(`ALTER TABLE session_embedding_chunks ALTER COLUMN embedding TYPE vector(${dim})`);
+            await pool.query(`CREATE INDEX session_chunks_embedding_idx` +
+                ` ON session_embedding_chunks USING ivfflat (embedding vector_l2_ops) WITH (lists = 100)`);
+            process.stderr.write(`[pg-embed-backfill] rebuilding fact_embeddings to vector(${dim})\n`);
+            await pool.query("DELETE FROM fact_embeddings");
+            await pool.query("DROP INDEX IF EXISTS fact_embeddings_idx");
+            await pool.query(`ALTER TABLE fact_embeddings ALTER COLUMN embedding TYPE vector(${dim})`);
+            await pool.query(`CREATE INDEX fact_embeddings_idx` +
+                ` ON fact_embeddings USING ivfflat (embedding vector_l2_ops) WITH (lists = 100)`);
+            stateData.done = new Set();
+        }
+        const limitClause = opts.limit ? ` LIMIT ${Math.trunc(opts.limit)}` : "";
+        const sessionResult = await pool.query("SELECT id, label, summary, body FROM sessions" +
+            " WHERE body IS NOT NULL OR summary IS NOT NULL OR label IS NOT NULL" +
+            " ORDER BY started_at" +
+            limitClause);
+        const rows = sessionResult.rows;
+        const total = rows.length;
+        const done = stateData.done;
+        let succeeded = 0;
+        let failed = 0;
+        let skipped = 0;
+        for (let i = 0; i < rows.length; i++) {
+            const row = rows[i];
+            const idx = i + 1;
+            if (done.has(row.id)) {
+                skipped += 1;
+                continue;
+            }
+            const chunks = chunkSessionText({
+                label: row.label,
+                summary: row.summary,
+                body: row.body,
+            });
+            if (chunks.length === 0) {
+                failed += 1;
+                opts.onProgress?.(idx, total, row.id, "SKIP (no text)");
+                continue;
+            }
+            await pool.query("DELETE FROM session_embedding_chunks WHERE session_id = $1", [row.id]);
+            const vectors = [];
+            let chunkSkipped = 0;
+            for (let c = 0; c < chunks.length; c++) {
+                const chunk = chunks[c];
+                let lastErr;
+                for (let attempt = 0; attempt < 2; attempt++) {
+                    try {
+                        const out = await opts.embedder.embed(chunk, "document");
+                        vectors.push({ idx: c, vec: out.vector });
+                        lastErr = undefined;
+                        break;
+                    }
+                    catch (e) {
+                        lastErr = e;
+                        if (!(e instanceof LLMUnreachableError))
+                            throw e;
+                        if (attempt === 0)
+                            await new Promise((r) => setTimeout(r, 200));
+                    }
+                }
+                if (lastErr !== undefined)
+                    chunkSkipped += 1;
+            }
+            if (vectors.length === 0) {
+                failed += 1;
+                opts.onProgress?.(idx, total, row.id, `FAIL (embedder, ${chunkSkipped}/${chunks.length} chunks)`);
+                continue;
+            }
+            try {
+                for (const { idx: cidx, vec } of vectors) {
+                    const vecStr = `[${Array.from(vec).join(",")}]`;
+                    const ins = await pool.query(`INSERT INTO session_embedding_chunks (session_id, chunk_idx, embedding)
+             VALUES ($1, $2, $3::vector) RETURNING chunk_id`, [row.id, cidx, vecStr]);
+                    const chunkId = ins.rows[0].chunk_id;
+                    await pool.query("INSERT INTO session_chunk_map (chunk_id, session_id, chunk_idx) VALUES ($1, $2, $3)", [chunkId, row.id, cidx]);
+                }
+            }
+            catch (e) {
+                failed += 1;
+                opts.onProgress?.(idx, total, row.id, `FAIL (db): ${e.message}`);
+                continue;
+            }
+            done.add(row.id);
+            succeeded += 1;
+            const status = chunkSkipped === 0
+                ? `OK (${vectors.length} chunks)`
+                : `PARTIAL (${vectors.length}/${chunks.length} chunks, ${chunkSkipped} skipped)`;
+            opts.onProgress?.(idx, total, row.id, status);
+            if (succeeded % SAVE_EVERY === 0)
+                saveState(statePath, done, { provider, model, dim });
+        }
+        // Reembed facts whenever the stored config does not match the runtime config
+        // (storedConfig exists but configMatch is false). This covers both fresh
+        // rebuilds and interrupted rebuilds that resume: the embedding_config row
+        // is only updated on successful completion, so storedConfig still holds
+        // the old values during a partial run. Gating on needsRebuild alone skips
+        // facts in that resume path.
+        let factsReembedded = 0;
+        if (storedConfig !== null && !configMatch) {
+            const factResult = await pool.query("SELECT id, subject, predicate, value FROM facts" +
+                " WHERE superseded_by IS NULL AND retired_at IS NULL");
+            for (const fact of factResult.rows) {
+                const factText = `${fact.subject} ${fact.predicate} ${fact.value}`.trim();
+                if (!factText)
+                    continue;
+                try {
+                    const out = await opts.embedder.embed(factText, "document");
+                    const vecStr = `[${Array.from(out.vector).join(",")}]`;
+                    await pool.query(`INSERT INTO fact_embeddings (fact_id, embedding)
+             VALUES ($1, $2::vector)
+             ON CONFLICT (fact_id) DO UPDATE SET embedding = EXCLUDED.embedding`, [fact.id, vecStr]);
+                    factsReembedded += 1;
+                }
+                catch {
+                    // Best-effort: fact row survives; semantic recall misses it until re-run.
+                }
+            }
+        }
+        await pool.query(`INSERT INTO embedding_config (lane, provider, model, dim, updated_at)
+       VALUES ($1, $2, $3, $4, $5)
+       ON CONFLICT (lane) DO UPDATE SET
+         provider   = EXCLUDED.provider,
+         model      = EXCLUDED.model,
+         dim        = EXCLUDED.dim,
+         updated_at = EXCLUDED.updated_at`, ["prose", provider, model, dim, new Date().toISOString()]);
+        saveState(statePath, done, { provider, model, dim });
+        return {
+            total,
+            processed: succeeded + failed + skipped,
+            succeeded,
+            failed,
+            skippedAlreadyDone: skipped,
+            dbMissing: false,
+            rebuilt: needsRebuild,
+            factsReembedded,
+        };
+    }
+    finally {
+        await pool.end();
+    }
+}
+//# sourceMappingURL=pg-embed-backfill.js.map