@pella-labs/pinakes 0.1.0

package/dist/ingest/ingester.js

@@ -0,0 +1,312 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { logger } from '../observability/logger.js';
+import { countTokens } from '../gate/budget.js';
+import { emptyManifest, fileSha, manifestPathFor, readManifest, removeManifestEntry, updateManifestEntry, writeManifest, } from './manifest.js';
+import { parseMarkdown, detectConfidence } from './parse/markdown.js';
+import { detectGaps } from '../gaps/detector.js';
+import { chunkSection } from './parse/chunk.js';
+// ----------------------------------------------------------------------------
+// Single-flight gate (module-level)
+// ----------------------------------------------------------------------------
+/**
+ * Module-level single-flight map. Keyed by `${scope}:${absPath}` so the same
+ * file can be ingested concurrently for different scopes (e.g. if the wiki
+ * dir lives at the same path for both scopes — unusual but possible).
+ *
+ * The promise stored here is the FULL `ingestFile` operation; concurrent
+ * callers await the same promise and observe the same result. After the
+ * promise settles, the entry is removed so subsequent calls run fresh.
+ *
+ * Test reset hook: `__resetSingleFlightForTests()` clears the map between
+ * tests so a leftover entry from a prior test doesn't affect the next one.
+ */
+const inFlight = new Map();
+/** Test-only: clear the single-flight map. Do not call from production code. */
+export function __resetSingleFlightForTests() {
+    inFlight.clear();
+}
+// ----------------------------------------------------------------------------
+// IngesterService
+// ----------------------------------------------------------------------------
+export class IngesterService {
+    bundle;
+    embedder;
+    scope;
+    manifest;
+    manifestPath;
+    constructor(bundle, embedder, scope, wikiRoot, options = {}) {
+        this.bundle = bundle;
+        this.embedder = embedder;
+        this.scope = scope;
+        this.manifestPath = options.manifestPath ?? manifestPathFor(wikiRoot);
+        this.manifest = readManifest(this.manifestPath);
+    }
+    /**
+     * Ingest a single markdown file. Single-flight: concurrent calls for the
+     * same file return the same in-progress promise.
+     *
+     * Returns the per-chunk metrics for this run (added/skipped/embedder calls)
+     * so callers (rebuild CLI, chokidar handler) can log a summary.
+     */
+    async ingestFile(absPath) {
+        const key = `${this.scope}:${resolve(absPath)}`;
+        const existing = inFlight.get(key);
+        if (existing)
+            return existing;
+        const promise = this.ingestFileImpl(resolve(absPath))
+            .catch((err) => {
+            // Make sure single-flight cleanup runs even on error.
+            logger.error({ err, file: absPath, scope: this.scope }, 'ingest failed');
+            this.appendLog('ingest:error', absPath, {
+                error: err instanceof Error ? err.message : String(err),
+            });
+            throw err;
+        })
+            .finally(() => {
+            inFlight.delete(key);
+        });
+        inFlight.set(key, promise);
+        return promise;
+    }
+    /**
+     * Remove a file from the index (for `file:removed` chokidar events).
+     * Deletes the file's nodes (cascades to chunks/edges/vec rows via FK)
+     * and updates the manifest. Single-statement, no transaction needed.
+     */
+    removeFile(absPath) {
+        const abs = resolve(absPath);
+        const sourceUri = pathToFileURL(abs).href;
+        this.bundle.writer
+            .prepare('DELETE FROM kg_nodes WHERE scope = ? AND source_uri = ?')
+            .run(this.scope, sourceUri);
+        this.manifest = removeManifestEntry(this.manifest, abs);
+        writeManifest(this.manifestPath, this.manifest);
+        this.appendLog('ingest:removed', abs, {});
+    }
+    /**
+     * The current in-memory manifest. Exposed for tests + the startup
+     * consistency check in `cli/serve.ts`.
+     */
+    getManifest() {
+        return this.manifest;
+    }
+    /**
+     * Reload the manifest from disk. Used at startup before walking the
+     * consistency check, and by tests that mutate the manifest externally.
+     */
+    reloadManifest() {
+        this.manifest = readManifest(this.manifestPath);
+    }
+    // --------------------------------------------------------------------------
+    // Internal: the actual ingest pipeline
+    // --------------------------------------------------------------------------
+    async ingestFileImpl(absPath) {
+        // Step 1: read file + compute file-level sha
+        const text = readFileSync(absPath, 'utf8');
+        const sourceSha = sha1(text);
+        const sourceUri = pathToFileURL(absPath).href;
+        // Step 2: manifest fast path — same source_sha → no work
+        const existingEntry = this.manifest.files[absPath];
+        if (existingEntry && existingEntry.source_sha === sourceSha) {
+            return {
+                chunks_added: 0,
+                chunks_skipped: 0,
+                embedder_calls: 0,
+                nodes_written: 0,
+                noop: true,
+            };
+        }
+        // Step 3: parse + chunk + detect confidence
+        const sections = parseMarkdown(text);
+        const confidence = detectConfidence(text);
+        const planned = sections.map((section) => {
+            const nodeId = nodeIdFor(this.scope, sourceUri, section.section_path);
+            const chunks = chunkSection(section.content);
+            return {
+                nodeId,
+                section,
+                chunks: chunks.map((chunkText, idx) => ({
+                    id: chunkIdFor(nodeId, idx),
+                    text: chunkText.text,
+                    chunkSha: sha1(chunkText.text),
+                    tokenCount: chunkText.token_count,
+                    chunkIndex: idx,
+                })),
+            };
+        });
+        // Step 4: load existing chunk_shas → embedding map for skip-unchanged
+        const existingEmbeddings = this.loadExistingEmbeddings(sourceUri);
+        // Step 5: embed only new chunks
+        let embedderCalls = 0;
+        let chunksSkipped = 0;
+        const embeddings = new Map();
+        for (const node of planned) {
+            for (const chunk of node.chunks) {
+                const cached = existingEmbeddings.get(chunk.chunkSha);
+                if (cached) {
+                    embeddings.set(chunk.id, cached);
+                    chunksSkipped++;
+                }
+                else {
+                    try {
+                        const vec = await this.embedder.embed(chunk.text);
+                        embeddings.set(chunk.id, vec);
+                        embedderCalls++;
+                    }
+                    catch (err) {
+                        // Per CLAUDE.md §AI Rules #4: embedder failure is non-fatal.
+                        // Insert the chunk without a vec row; FTS5 still works.
+                        logger.warn({ err, chunkId: chunk.id, file: absPath }, 'embedder failed for chunk; inserting without vec row');
+                    }
+                }
+            }
+        }
+        // Step 6: transaction — delete-then-insert nodes/chunks/vec for this file
+        const now = Date.now();
+        const totalChunks = planned.reduce((acc, n) => acc + n.chunks.length, 0);
+        this.runInTransaction((w) => {
+            // Delete old vec rows BEFORE deleting kg_chunks rows. We need to know
+            // the rowids first because the FK cascade will drop the kg_chunks rows
+            // (and sqlite-vec doesn't have FK awareness).
+            const oldRowids = w
+                .prepare(`SELECT c.rowid AS rowid
+             FROM kg_chunks c JOIN kg_nodes n ON c.node_id = n.id
+            WHERE n.scope = ? AND n.source_uri = ?`)
+                .all(this.scope, sourceUri);
+            if (oldRowids.length > 0) {
+                const placeholders = oldRowids.map(() => '?').join(',');
+                w.prepare(`DELETE FROM kg_chunks_vec WHERE rowid IN (${placeholders})`).run(...oldRowids.map((r) => r.rowid));
+            }
+            // Delete old nodes for this file (cascades to chunks → FTS5 trigger fires).
+            w.prepare('DELETE FROM kg_nodes WHERE scope = ? AND source_uri = ?').run(this.scope, sourceUri);
+            // Insert new nodes
+            const insertNode = w.prepare(`INSERT INTO kg_nodes (id, scope, source_uri, section_path, kind, title, content, source_sha, token_count, created_at, updated_at, last_accessed_at, confidence)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+            const insertChunk = w.prepare(`INSERT INTO kg_chunks (id, node_id, chunk_index, text, chunk_sha, token_count, created_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?)`);
+            const insertVec = w.prepare('INSERT INTO kg_chunks_vec(rowid, embedding) VALUES (?, ?)');
+            for (const node of planned) {
+                const sectionTokenCount = countTokens(node.section.content);
+                insertNode.run(node.nodeId, this.scope, sourceUri, node.section.section_path, node.section.kind, node.section.title || null, node.section.content, sourceSha, sectionTokenCount, now, now, now, confidence);
+                for (const chunk of node.chunks) {
+                    // Capture lastInsertRowid directly from the chunk insert. This avoids
+                    // a follow-up SELECT.
+                    const info = insertChunk.run(chunk.id, node.nodeId, chunk.chunkIndex, chunk.text, chunk.chunkSha, chunk.tokenCount, now);
+                    // sqlite-vec's vec0 virtual table is strict: it rejects JS Number
+                    // bindings for the rowid column with "Only integers are allowed for
+                    // primary key values" — even though the value IS an integer at the
+                    // SQL level. We MUST bind as BigInt. (Verified via repro 2026-04-09.)
+                    // lastInsertRowid is `number | bigint`; coerce to BigInt either way.
+                    const rowidBig = typeof info.lastInsertRowid === 'bigint'
+                        ? info.lastInsertRowid
+                        : BigInt(info.lastInsertRowid);
+                    const vec = embeddings.get(chunk.id);
+                    if (vec) {
+                        insertVec.run(rowidBig, Buffer.from(vec.buffer));
+                    }
+                }
+            }
+        });
+        // Step 7: append kg_log row
+        this.appendLog('ingest:done', absPath, {
+            nodes: planned.length,
+            chunks_added: totalChunks - chunksSkipped,
+            chunks_skipped: chunksSkipped,
+            embedder_calls: embedderCalls,
+        });
+        // Step 7.5: gap detection (Phase 6) — runs after commit, scans for concept
+        // mentions and resolves gaps when matching nodes appear
+        try {
+            const nodeTitles = planned
+                .map((n) => n.section.title)
+                .filter((t) => !!t);
+            detectGaps(this.bundle.writer, this.scope, text, nodeTitles);
+        }
+        catch (err) {
+            // Non-fatal: gap detection failure should never block ingest
+            logger.warn({ err, file: absPath }, 'gap detection failed');
+        }
+        // Step 8: update manifest (AFTER commit)
+        const allChunkShas = planned.flatMap((n) => n.chunks.map((c) => c.chunkSha));
+        const newEntry = { source_sha: sourceSha, chunk_shas: allChunkShas };
+        this.manifest = updateManifestEntry(this.manifest, absPath, newEntry);
+        writeManifest(this.manifestPath, this.manifest);
+        return {
+            chunks_added: totalChunks - chunksSkipped,
+            chunks_skipped: chunksSkipped,
+            embedder_calls: embedderCalls,
+            nodes_written: planned.length,
+            noop: false,
+        };
+    }
+    /**
+     * Load `chunk_sha → embedding` for every existing chunk of a file. Used by
+     * the per-chunk skip optimization: if a new chunk's sha matches one in
+     * this map, we reuse the embedding instead of calling the embedder.
+     *
+     * Returns an empty map if the file isn't in the DB yet (first ingest).
+     */
+    loadExistingEmbeddings(sourceUri) {
+        const out = new Map();
+        const rows = this.bundle.writer
+            .prepare(`SELECT c.chunk_sha AS chunk_sha, vec.embedding AS embedding
+           FROM kg_chunks c
+           JOIN kg_nodes n ON c.node_id = n.id
+           LEFT JOIN kg_chunks_vec vec ON vec.rowid = c.rowid
+          WHERE n.scope = ? AND n.source_uri = ?`)
+            .all(this.scope, sourceUri);
+        for (const row of rows) {
+            if (!row.embedding)
+                continue;
+            // sqlite-vec stores Float32 as a raw bytes blob. Reinterpret the buffer
+            // (no copy needed since we won't mutate it).
+            const buf = row.embedding;
+            const view = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
+            // Make a stable copy in case better-sqlite3 reuses the buffer.
+            out.set(row.chunk_sha, new Float32Array(view));
+        }
+        return out;
+    }
+    runInTransaction(fn) {
+        const txn = this.bundle.writer.transaction(fn);
+        txn(this.bundle.writer);
+    }
+    appendLog(kind, absPath, payload) {
+        try {
+            this.bundle.writer
+                .prepare(`INSERT INTO kg_log (ts, scope, kind, source_uri, payload)
+           VALUES (?, ?, ?, ?, ?)`)
+                .run(Date.now(), this.scope, kind, pathToFileURL(absPath).href, JSON.stringify(payload));
+        }
+        catch (err) {
+            logger.warn({ err, kind, absPath }, 'failed to append kg_log row');
+        }
+    }
+}
+/** sha1(scope + ':' + source_uri + ':' + section_path) — locked id derivation */
+function nodeIdFor(scope, sourceUri, sectionPath) {
+    return sha1(`${scope}:${sourceUri}:${sectionPath}`);
+}
+/** sha1(node_id + ':' + chunk_index) — locked id derivation */
+function chunkIdFor(nodeId, chunkIndex) {
+    return sha1(`${nodeId}:${chunkIndex}`);
+}
+function sha1(input) {
+    return createHash('sha1').update(input).digest('hex');
+}
+// ----------------------------------------------------------------------------
+// Cold-start helper for the rebuild CLI
+// ----------------------------------------------------------------------------
+/**
+ * Reset the in-memory manifest to empty. Used by `kg rebuild` when the
+ * caller passes `--clean` to force a full re-ingest from scratch.
+ */
+export function freshManifest() {
+    return emptyManifest();
+}
+export { manifestPathFor };
+export { fileSha };
+//# sourceMappingURL=ingester.js.map

package/dist/ingest/ingester.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingester.js","sourceRoot":"","sources":["../../src/ingest/ingester.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAGL,aAAa,EACb,OAAO,EACP,eAAe,EACf,YAAY,EACZ,mBAAmB,EACnB,mBAAmB,EACnB,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAoB,MAAM,qBAAqB,CAAC;AACxF,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,YAAY,EAA2B,MAAM,kBAAkB,CAAC;AAqEzE,+EAA+E;AAC/E,oCAAoC;AACpC,+EAA+E;AAE/E;;;;;;;;;;;GAWG;AACH,MAAM,QAAQ,GAAuC,IAAI,GAAG,EAAE,CAAC;AAE/D,gFAAgF;AAChF,MAAM,UAAU,2BAA2B;IACzC,QAAQ,CAAC,KAAK,EAAE,CAAC;AACnB,CAAC;AAED,+EAA+E;AAC/E,kBAAkB;AAClB,+EAA+E;AAE/E,MAAM,OAAO,eAAe;IAKP;IACA;IACA;IANX,QAAQ,CAAW;IACV,YAAY,CAAS;IAEtC,YACmB,MAAgB,EAChB,QAAkB,EAClB,KAAY,EAC7B,QAAgB,EAChB,UAA2B,EAAE;QAJZ,WAAM,GAAN,MAAM,CAAU;QAChB,aAAQ,GAAR,QAAQ,CAAU;QAClB,UAAK,GAAL,KAAK,CAAO;QAI7B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC,QAAQ,CAAC,CAAC;QACtE,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAClD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,OAAe;QAC9B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;aAClD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,sDAAsD;YACtD,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,eAAe,CAAC,CAAC;YACzE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,OAAO,EAAE;gBACtC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YACH,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEL,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,UAAU,CAAC,OAAe;QACxB,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;QAE1C,IAAI,CAAC,MAAM,CAAC,MAAM;aACf,OAAO,CAAC,yDAAyD,CAAC;aAClE,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE9B,IAAI,CAAC,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACxD,aAAa,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEhD,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,cAAc;QACZ,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAClD,CAAC;IAED,6EAA6E;IAC7E,uCAAuC;IACvC,6EAA6E;IAErE,KAAK,CAAC,cAAc,CAAC,OAAe;QAC1C,6CAA6C;QAC7C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAE9C,yDAAyD;QACzD,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,aAAa,IAAI,aAAa,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC5D,OAAO;gBACL,YAAY,EAAE,CAAC;gBACf,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,CAAC;gBACjB,aAAa,EAAE,CAAC;gBAChB,IAAI,EAAE,IAAI;aACX,CAAC;QACJ,CAAC;QAED,4CAA4C;QAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAkB,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACtD,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;YACtE,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC7C,OAAO;gBACL,MAAM;gBACN,OAAO;gBACP,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;oBACtC,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;oBAC3B,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAC9B,UAAU,EAAE,SAAS,CAAC,WAAW;oBACjC,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;aACJ,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAElE,gCAAgC;QAChC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACtD,IAAI,MAAM,EAAE,CAAC;oBACX,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBACjC,aAAa,EAAE,CAAC;gBAClB,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC;wBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAClD,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;wBAC9B,aAAa,EAAE,CAAC;oBAClB,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,6DAA6D;wBAC7D,wDAAwD;wBACxD,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EACzC,sDAAsD,CACvD,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,0EAA0E;QAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAEzE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,EAAE;YAC1B,sEAAsE;YACtE,uEAAuE;YACvE,8CAA8C;YAC9C,MAAM,SAAS,GAAG,CAAC;iBAChB,OAAO,CACN;;mDAEyC,CAC1C;iBACA,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAE9B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACxD,CAAC,CAAC,OAAO,CAAC,6CAA6C,YAAY,GAAG,CAAC,CAAC,GAAG,CACzE,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACjC,CAAC;YACJ,CAAC;YAED,4EAA4E;YAC5E,CAAC,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CACtE,IAAI,CAAC,KAAK,EACV,SAAS,CACV,CAAC;YAEF,mBAAmB;YACnB,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,CAC1B;wDACgD,CACjD,CAAC;YACF,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAC3B;sCAC8B,CAC/B,CAAC;YACF,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CACzB,2DAA2D,CAC5D,CAAC;YAEF,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,iBAAiB,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC5D,UAAU,CAAC,GAAG,CACZ,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,SAAS,EACT,IAAI,CAAC,OAAO,CAAC,YAAY,EACzB,IAAI,CAAC,OAAO,CAAC,IAAI,EACjB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,EAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,SAAS,EACT,iBAAiB,EACjB,GAAG,EACH,GAAG,EACH,GAAG,EACH,UAAU,CACX,CAAC;gBAEF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChC,sEAAsE;oBACtE,sBAAsB;oBACtB,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAC1B,KAAK,CAAC,EAAE,EACR,IAAI,CAAC,MAAM,EACX,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,UAAU,EAChB,GAAG,CACJ,CAAC;oBACF,kEAAkE;oBAClE,oEAAoE;oBACpE,mEAAmE;oBACnE,sEAAsE;oBACtE,qEAAqE;oBACrE,MAAM,QAAQ,GACZ,OAAO,IAAI,CAAC,eAAe,KAAK,QAAQ;wBACtC,CAAC,CAAC,IAAI,CAAC,eAAe;wBACtB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;oBAEnC,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBACrC,IAAI,GAAG,EAAE,CAAC;wBACR,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;oBACnD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,4BAA4B;QAC5B,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE;YACrC,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,YAAY,EAAE,WAAW,GAAG,aAAa;YACzC,cAAc,EAAE,aAAa;YAC7B,cAAc,EAAE,aAAa;SAC9B,CAAC,CAAC;QAEH,2EAA2E;QAC3E,wDAAwD;QACxD,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,OAAO;iBACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC3B,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6DAA6D;YAC7D,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC9D,CAAC;QAED,yCAAyC;QACzC,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAkB,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC;QACpF,IAAI,CAAC,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QACtE,aAAa,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEhD,OAAO;YACL,YAAY,EAAE,WAAW,GAAG,aAAa;YACzC,cAAc,EAAE,aAAa;YAC7B,cAAc,EAAE,aAAa;YAC7B,aAAa,EAAE,OAAO,CAAC,MAAM;YAC7B,IAAI,EAAE,KAAK;SACZ,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,sBAAsB,CAAC,SAAiB;QAC9C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAwB,CAAC;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;aAC5B,OAAO,CAIN;;;;iDAIyC,CAC1C;aACA,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,SAAS;gBAAE,SAAS;YAC7B,wEAAwE;YACxE,6CAA6C;YAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;YAC9E,+DAA+D;YAC/D,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,gBAAgB,CAAC,EAA0C;QACjE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC/C,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,OAAe,EAAE,OAAgC;QAC/E,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,MAAM;iBACf,OAAO,CACN;kCACwB,CACzB;iBACA,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;CACF;AAoBD,iFAAiF;AACjF,SAAS,SAAS,CAAC,KAAY,EAAE,SAAiB,EAAE,WAAmB;IACrE,OAAO,IAAI,CAAC,GAAG,KAAK,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,+DAA+D;AAC/D,SAAS,UAAU,CAAC,MAAc,EAAE,UAAkB;IACpD,OAAO,IAAI,CAAC,GAAG,MAAM,IAAI,UAAU,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACxD,CAAC;AAED,+EAA+E;AAC/E,wCAAwC;AACxC,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,aAAa;IAC3B,OAAO,aAAa,EAAE,CAAC;AACzB,CAAC;AAID,OAAO,EAAE,eAAe,EAAE,CAAC;AAC3B,OAAO,EAAE,OAAO,EAAE,CAAC"}

package/dist/ingest/manifest.d.ts

@@ -0,0 +1,87 @@
+import type { Database as BetterSqliteDatabase } from 'better-sqlite3';
+export interface ManifestEntry {
+    /** sha1 of the entire source file */
+    source_sha: string;
+    /** Ordered list of chunk shas (one per chunk produced by the chunker) */
+    chunk_shas: string[];
+}
+export interface Manifest {
+    version: number;
+    /** Maps absolute file paths → entry */
+    files: Record<string, ManifestEntry>;
+}
+/**
+ * Build an empty manifest. Useful for cold starts and tests.
+ */
+export declare function emptyManifest(): Manifest;
+/**
+ * Read a manifest from disk. Returns an empty manifest if the file doesn't
+ * exist or is unreadable. Logs a warning on parse errors but doesn't throw —
+ * a corrupted manifest is treated as "no manifest", and the consistency
+ * check will conservatively re-ingest everything (which is correct).
+ */
+export declare function readManifest(path: string): Manifest;
+/**
+ * Write a manifest atomically. Creates the parent directory if missing.
+ * The write is tmpfile + rename, so a SIGKILL mid-write either leaves the
+ * old manifest intact or replaces it with the new one — never half-written.
+ */
+export declare function writeManifest(path: string, manifest: Manifest): void;
+/**
+ * Update one entry in a manifest in-memory. Mutates the input and returns it
+ * (the caller is expected to follow up with `writeManifest`). The entry's
+ * key is the absolute path to the file as it was ingested.
+ */
+export declare function updateManifestEntry(manifest: Manifest, filePath: string, entry: ManifestEntry): Manifest;
+/**
+ * Drop a file from the manifest (e.g. after a `file:removed` event).
+ */
+export declare function removeManifestEntry(manifest: Manifest, filePath: string): Manifest;
+/**
+ * Compute the sha1 of a file's contents on disk. Used by `checkConsistency`
+ * and the ingester. Synchronous because we're already in the single-flight
+ * path and parallelizing this against itself buys nothing.
+ */
+export declare function fileSha(filePath: string): string;
+/**
+ * Walk a wiki directory recursively and return absolute paths of all *.md
+ * files in deterministic (sorted) order. Used by both the consistency check
+ * and the rebuild CLI.
+ */
+export declare function listMarkdownFiles(rootDir: string): string[];
+/**
+ * Compare an in-memory manifest against the current on-disk state of a wiki
+ * directory. Returns a list of file paths that need re-ingestion:
+ *   - Files present on disk that are NOT in the manifest (new or unindexed)
+ *   - Files present in BOTH but with mismatched `source_sha` (edited externally,
+ *     or last ingest crashed mid-flight before the manifest could be updated)
+ *   - Files where the manifest says "indexed" but the DB has zero rows (DB/manifest
+ *     divergence — e.g. DB was recreated, migration dropped tables, or crash
+ *     between COMMIT and manifest write left the manifest ahead of the DB)
+ *
+ * When DB/manifest divergence is detected for a file, its manifest entry is
+ * cleared so the ingester's manifest fast-path won't noop the re-ingest.
+ *
+ * Files that exist in the manifest but no longer on disk are NOT returned —
+ * they're deletions, handled separately by the chokidar `file:removed`
+ * pathway in Pass 3. The startup consistency check is for additions and
+ * mutations, not removals.
+ *
+ * **Performance**: O(N) over markdown files, one stat + sha per file, plus
+ * one lightweight COUNT query per file that passes the manifest check when
+ * a DB writer is provided. For a typical wiki of <100 files this is <100ms.
+ */
+export declare function checkConsistency(manifest: Manifest, rootDir: string, writer?: BetterSqliteDatabase): string[];
+/**
+ * Compute the canonical manifest path for a given wiki directory and scope.
+ *
+ * - `'project'` → `<wikiPath>/../kg-manifest.json`
+ *   (lives next to the wiki dir, inside the same `.pharos/` folder)
+ * - `'personal'` → `<wikiPath>/../kg-manifest.json` as well
+ *   (lives next to `~/.pharos/profile/wiki/`, in `~/.pharos/profile/`)
+ *
+ * Both scopes use the same relative shape — the difference is the wikiPath
+ * the caller passes in.
+ */
+export declare function manifestPathFor(wikiPath: string): string;
+//# sourceMappingURL=manifest.d.ts.map

package/dist/ingest/manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/ingest/manifest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,IAAI,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAwCvE,MAAM,WAAW,aAAa;IAC5B,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CACtC;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,QAAQ,CAExC;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAqBnD;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAOpE;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,aAAa,GACnB,QAAQ,CAIV;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAIlF;AAED;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGhD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CA4B3D;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,oBAAoB,GAC5B,MAAM,EAAE,CAsCV;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAExD"}

package/dist/ingest/manifest.js

@@ -0,0 +1,223 @@
+import { createHash } from 'node:crypto';
+import { mkdirSync, readFileSync, renameSync, writeFileSync, existsSync, readdirSync, statSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { logger } from '../observability/logger.js';
+/**
+ * Consistency manifest for KG-MCP Phase 2.
+ *
+ * Lives at:
+ *   - `<wikiPath>/../kg-manifest.json` for project scope
+ *     (e.g. `.pharos/kg-manifest.json` next to `.pharos/wiki/`)
+ *   - `<profilePath>/kg-manifest.json` for personal scope
+ *     (e.g. `~/.pharos/profile/kg-manifest.json`)
+ *
+ * **What it stores**: per ingested file, the file-level `source_sha` and
+ * the list of per-chunk `chunk_shas` that ingest produced. This is the
+ * mid-ingest crash recovery surface — if the process dies during a vec
+ * insert, the manifest may disagree with the DB on startup, and the
+ * consistency check (`checkConsistency`) detects which files need re-ingest.
+ *
+ * **Why we need this**: pre-v1 sqlite-vec has untested crash semantics
+ * (presearch.md F9). The DB may end up in a state where vec rows exist for
+ * a chunk_sha that doesn't match the on-disk file. The manifest is the
+ * tiebreaker — it records what we INTENDED to be on disk. On startup we
+ * compare it against the actual on-disk markdown file_sha; mismatches mean
+ * the file was edited externally (or the previous run crashed before
+ * completing). Either way, re-ingest is the right move.
+ *
+ * **Atomic writes**: writeManifest() writes to a tmpfile and renames over
+ * the target. SQLite uses the same pattern; rename is atomic on Linux/macOS
+ * for files on the same filesystem. This avoids the failure mode where a
+ * SIGKILL mid-write leaves a half-written manifest that fails to parse on
+ * the next startup.
+ *
+ * **Format**: JSON, hand-readable, future-proofed with a `version` field
+ * so we can detect old manifests if the structure ever changes (e.g. adding
+ * per-chunk embedding metadata).
+ */
+const MANIFEST_VERSION = 1;
+/**
+ * Build an empty manifest. Useful for cold starts and tests.
+ */
+export function emptyManifest() {
+    return { version: MANIFEST_VERSION, files: {} };
+}
+/**
+ * Read a manifest from disk. Returns an empty manifest if the file doesn't
+ * exist or is unreadable. Logs a warning on parse errors but doesn't throw —
+ * a corrupted manifest is treated as "no manifest", and the consistency
+ * check will conservatively re-ingest everything (which is correct).
+ */
+export function readManifest(path) {
+    if (!existsSync(path))
+        return emptyManifest();
+    try {
+        const raw = readFileSync(path, 'utf8');
+        const parsed = JSON.parse(raw);
+        if (typeof parsed !== 'object' || parsed === null)
+            return emptyManifest();
+        if (parsed.version !== MANIFEST_VERSION) {
+            logger.warn({ path, foundVersion: parsed.version, expectedVersion: MANIFEST_VERSION }, 'manifest version mismatch — discarding');
+            return emptyManifest();
+        }
+        if (typeof parsed.files !== 'object' || parsed.files === null) {
+            return emptyManifest();
+        }
+        return parsed;
+    }
+    catch (err) {
+        logger.warn({ err, path }, 'failed to parse manifest — starting fresh');
+        return emptyManifest();
+    }
+}
+/**
+ * Write a manifest atomically. Creates the parent directory if missing.
+ * The write is tmpfile + rename, so a SIGKILL mid-write either leaves the
+ * old manifest intact or replaces it with the new one — never half-written.
+ */
+export function writeManifest(path, manifest) {
+    const abs = resolve(path);
+    mkdirSync(dirname(abs), { recursive: true });
+    const tmp = `${abs}.tmp.${process.pid}.${Date.now()}`;
+    const json = JSON.stringify(manifest, null, 2);
+    writeFileSync(tmp, json, 'utf8');
+    renameSync(tmp, abs);
+}
+/**
+ * Update one entry in a manifest in-memory. Mutates the input and returns it
+ * (the caller is expected to follow up with `writeManifest`). The entry's
+ * key is the absolute path to the file as it was ingested.
+ */
+export function updateManifestEntry(manifest, filePath, entry) {
+    const abs = resolve(filePath);
+    manifest.files[abs] = entry;
+    return manifest;
+}
+/**
+ * Drop a file from the manifest (e.g. after a `file:removed` event).
+ */
+export function removeManifestEntry(manifest, filePath) {
+    const abs = resolve(filePath);
+    delete manifest.files[abs];
+    return manifest;
+}
+/**
+ * Compute the sha1 of a file's contents on disk. Used by `checkConsistency`
+ * and the ingester. Synchronous because we're already in the single-flight
+ * path and parallelizing this against itself buys nothing.
+ */
+export function fileSha(filePath) {
+    const buf = readFileSync(filePath);
+    return createHash('sha1').update(buf).digest('hex');
+}
+/**
+ * Walk a wiki directory recursively and return absolute paths of all *.md
+ * files in deterministic (sorted) order. Used by both the consistency check
+ * and the rebuild CLI.
+ */
+export function listMarkdownFiles(rootDir) {
+    const out = [];
+    const stack = [resolve(rootDir)];
+    while (stack.length > 0) {
+        const dir = stack.pop();
+        let entries;
+        try {
+            entries = readdirSync(dir);
+        }
+        catch {
+            continue;
+        }
+        for (const name of entries) {
+            const full = join(dir, name);
+            let st;
+            try {
+                st = statSync(full);
+            }
+            catch {
+                continue;
+            }
+            if (st.isDirectory()) {
+                stack.push(full);
+            }
+            else if (st.isFile() && name.toLowerCase().endsWith('.md')) {
+                out.push(full);
+            }
+        }
+    }
+    out.sort();
+    return out;
+}
+/**
+ * Compare an in-memory manifest against the current on-disk state of a wiki
+ * directory. Returns a list of file paths that need re-ingestion:
+ *   - Files present on disk that are NOT in the manifest (new or unindexed)
+ *   - Files present in BOTH but with mismatched `source_sha` (edited externally,
+ *     or last ingest crashed mid-flight before the manifest could be updated)
+ *   - Files where the manifest says "indexed" but the DB has zero rows (DB/manifest
+ *     divergence — e.g. DB was recreated, migration dropped tables, or crash
+ *     between COMMIT and manifest write left the manifest ahead of the DB)
+ *
+ * When DB/manifest divergence is detected for a file, its manifest entry is
+ * cleared so the ingester's manifest fast-path won't noop the re-ingest.
+ *
+ * Files that exist in the manifest but no longer on disk are NOT returned —
+ * they're deletions, handled separately by the chokidar `file:removed`
+ * pathway in Pass 3. The startup consistency check is for additions and
+ * mutations, not removals.
+ *
+ * **Performance**: O(N) over markdown files, one stat + sha per file, plus
+ * one lightweight COUNT query per file that passes the manifest check when
+ * a DB writer is provided. For a typical wiki of <100 files this is <100ms.
+ */
+export function checkConsistency(manifest, rootDir, writer) {
+    const stale = [];
+    const files = listMarkdownFiles(rootDir);
+    // Pre-build a set of indexed source_uris for O(1) lookup when DB is available.
+    // A single query is cheaper than N per-file COUNT queries.
+    let indexedUris = null;
+    if (writer) {
+        try {
+            const rows = writer
+                .prepare('SELECT DISTINCT source_uri FROM kg_nodes')
+                .all();
+            indexedUris = new Set(rows.map((r) => r.source_uri));
+        }
+        catch (err) {
+            logger.warn({ err }, 'checkConsistency: failed to query kg_nodes — falling back to manifest-only');
+        }
+    }
+    for (const f of files) {
+        const entry = manifest.files[f];
+        const onDiskSha = fileSha(f);
+        if (!entry || entry.source_sha !== onDiskSha) {
+            stale.push(f);
+        }
+        else if (indexedUris) {
+            // Manifest says this file is current — verify the DB agrees.
+            const sourceUri = pathToFileURL(resolve(f)).href;
+            if (!indexedUris.has(sourceUri)) {
+                logger.info({ file: f }, 'checkConsistency: manifest says indexed but DB has no rows — marking stale');
+                // Clear the manifest entry so the ingester's fast-path doesn't noop.
+                delete manifest.files[f];
+                stale.push(f);
+            }
+        }
+    }
+    return stale;
+}
+/**
+ * Compute the canonical manifest path for a given wiki directory and scope.
+ *
+ * - `'project'` → `<wikiPath>/../kg-manifest.json`
+ *   (lives next to the wiki dir, inside the same `.pharos/` folder)
+ * - `'personal'` → `<wikiPath>/../kg-manifest.json` as well
+ *   (lives next to `~/.pharos/profile/wiki/`, in `~/.pharos/profile/`)
+ *
+ * Both scopes use the same relative shape — the difference is the wikiPath
+ * the caller passes in.
+ */
+export function manifestPathFor(wikiPath) {
+    return resolve(dirname(resolve(wikiPath)), 'kg-manifest.json');
+}
+//# sourceMappingURL=manifest.js.map

package/dist/ingest/manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/ingest/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAChH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEpD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAe3B;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;AAClD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,aAAa,EAAE,CAAC;IAC9C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;QAC3C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,aAAa,EAAE,CAAC;QAC1E,IAAI,MAAM,CAAC,OAAO,KAAK,gBAAgB,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CACT,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,EACzE,wCAAwC,CACzC,CAAC;YACF,OAAO,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC9D,OAAO,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,2CAA2C,CAAC,CAAC;QACxE,OAAO,aAAa,EAAE,CAAC;IACzB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,QAAkB;IAC5D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,GAAG,GAAG,GAAG,GAAG,QAAQ,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC/C,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAkB,EAClB,QAAgB,EAChB,KAAoB;IAEpB,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAC5B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAkB,EAAE,QAAgB;IACtE,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,QAAgB;IACtC,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACnC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QACzB,IAAI,OAAiB,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC7B,IAAI,EAAE,CAAC;YACP,IAAI,CAAC;gBACH,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;iBAAM,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IACD,GAAG,CAAC,IAAI,EAAE,CAAC;IACX,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,gBAAgB,CAC9B,QAAkB,EAClB,OAAe,EACf,MAA6B;IAE7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAEzC,+EAA+E;IAC/E,2DAA2D;IAC3D,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM;iBAChB,OAAO,CAAC,0CAA0C,CAAC;iBACnD,GAAG,EAAmC,CAAC;YAC1C,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,4EAA4E,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,WAAW,EAAE,CAAC;YACvB,6DAA6D;YAC7D,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACjD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CACT,EAAE,IAAI,EAAE,CAAC,EAAE,EACX,4EAA4E,CAC7E,CAAC;gBACF,qEAAqE;gBACrE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;AACjE,CAAC"}