@pella-labs/pinakes 0.1.0

package/dist/gate/budget.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"budget.d.ts","sourceRoot":"","sources":["../../src/gate/budget.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,aAAa,MAAM,CAAC;AA4BjC;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKhD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAG/D;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,IAAI,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,SAAS,CAAC,CAAC;IAC1B,IAAI,EAAE,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC;IAClC,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAC1B,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC9B,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EACzB,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,GACzB,SAAS,CAAC,CAAC,CAAC,CA4Cd;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAEhE"}

package/dist/gate/budget.js

@@ -0,0 +1,145 @@
+import { getEncoding } from 'js-tiktoken';
+/**
+ * Token-counting budget gate.
+ *
+ * Implements CLAUDE.md §API Rules #6 budget math:
+ *
+ *   envelope_reserve = 500          // bytes set aside for meta/logs/stale_files
+ *   safety_margin    = 0.9          // js-tiktoken is an estimator, not an oracle
+ *   available        = floor((max_tokens - envelope_reserve) * safety_margin)
+ *
+ * At the default `max_tokens=5000` the available budget for result bodies is:
+ *   floor((5000 - 500) * 0.9) = 4050 tokens
+ *
+ * Truncation is greedy by rank: keep the highest-ranked item whole if it fits;
+ * otherwise emit a `too_large` sentinel so the caller can re-query with a
+ * higher `max_tokens` or fetch the node directly by id.
+ *
+ * The sentinel pattern is Loop 6.5 A3 / presearch.md D22. A single oversize
+ * item must NOT blackhole the whole response — we report its id + uri and
+ * let the LLM decide what to do next.
+ *
+ * Token counting uses the `p50k_base` encoder — close enough to Claude's
+ * tokenization for budgeting purposes, and the 10% safety margin absorbs the
+ * estimation error between tokenizers.
+ */
+export const ENVELOPE_RESERVE_TOKENS = 500;
+export const SAFETY_MARGIN = 0.9;
+/**
+ * Length threshold above which we skip the real tokenizer and use a
+ * conservative character-based estimate instead.
+ *
+ * js-tiktoken's BPE merge loop is O(n²) on long runs — a 60K-char string
+ * takes ~200 seconds to tokenize on current hardware (measured), which is
+ * a DoS vector on the budget gate. For any string longer than this
+ * threshold we estimate `ceil(length / CHARS_PER_TOKEN_LOWER)`, which
+ * always over-counts (since real English is 4+ chars/token), keeping us
+ * safely conservative with respect to the budget.
+ *
+ * Why 8000: at that size tiktoken takes ~2.5s which is already too slow
+ * for a request path. Anything under the threshold tokenizes in <50ms,
+ * which is acceptable.
+ */
+const EXACT_TOKENIZE_MAX_CHARS = 8_000;
+/**
+ * Pessimistic chars-per-token ratio for the estimation path. Real English
+ * text runs at 4+ chars per token; we use 3.0 to over-count on purpose so
+ * the budget gate stays safe even on token-dense content (code, URLs).
+ */
+const CHARS_PER_TOKEN_LOWER = 3.0;
+const encoder = getEncoding('p50k_base');
+/**
+ * Count tokens in a UTF-8 string.
+ *
+ * Fast path (long strings): return a character-based over-estimate. This
+ * is strictly a ceiling — we'd rather emit a few extra `results_truncated`
+ * responses than block the event loop for minutes on tokenization.
+ *
+ * Slow path (short strings): use the real p50k_base encoder for an exact
+ * count. This is what matters for normal-size response bodies.
+ *
+ * The encoder is initialized once at module load and shared across calls.
+ */
+export function countTokens(text) {
+    if (text.length > EXACT_TOKENIZE_MAX_CHARS) {
+        return Math.ceil(text.length / CHARS_PER_TOKEN_LOWER);
+    }
+    return encoder.encode(text).length;
+}
+/**
+ * Given a user-facing `max_tokens` budget, compute the internal result-body
+ * budget after subtracting the envelope reserve and applying the safety
+ * margin. Always returns a non-negative integer.
+ */
+export function computeInternalBudget(maxTokens) {
+    const raw = Math.floor((maxTokens - ENVELOPE_RESERVE_TOKENS) * SAFETY_MARGIN);
+    return Math.max(0, raw);
+}
+/**
+ * Greedy rank-order truncation. Iterates `items` in the order given (caller
+ * is responsible for ranking first), measures each one's serialized token
+ * count, and keeps items until the next one would exceed the internal
+ * budget.
+ *
+ * If a single item's token count alone exceeds the budget, it is replaced
+ * with a `too_large` sentinel and counted as zero body tokens (the sentinel
+ * itself is tiny — ~20 tokens). The iteration then continues so that smaller
+ * items after the oversize one can still land in the response.
+ *
+ * @param items       Results, pre-ranked (highest rank first).
+ * @param maxTokens   User-facing `max_tokens` budget from the tool call.
+ * @param serialize   How to turn one item into the text we'll count. Usually
+ *                    `JSON.stringify`. Broken out so the caller can include
+ *                    framing (commas, wrapping object keys) in the count.
+ * @param idOf        Read the item's id for sentinel construction.
+ * @param uriOf       Read the item's source uri for sentinel construction.
+ */
+export function fitResults(items, maxTokens, serialize, idOf, uriOf) {
+    const budget = computeInternalBudget(maxTokens);
+    const kept = [];
+    let used = 0;
+    let truncated = false;
+    for (const item of items) {
+        const serialized = serialize(item);
+        const cost = countTokens(serialized);
+        if (cost > budget) {
+            // Single-oversize case — emit a sentinel so the caller can re-query.
+            const sentinel = {
+                too_large: true,
+                id: idOf(item),
+                source_uri: uriOf(item),
+                tokens: cost,
+            };
+            const sentinelCost = countTokens(JSON.stringify(sentinel));
+            if (used + sentinelCost > budget) {
+                truncated = true;
+                break;
+            }
+            kept.push(sentinel);
+            used += sentinelCost;
+            truncated = true; // we dropped the actual body
+            continue;
+        }
+        if (used + cost > budget) {
+            truncated = true;
+            break;
+        }
+        kept.push(item);
+        used += cost;
+    }
+    return {
+        kept,
+        truncated,
+        tokensUsed: used,
+        tokensBudgeted: budget,
+    };
+}
+/**
+ * Count tokens in an already-serialized response body without running the
+ * fit loop. Used by the tool handlers to populate `meta.tokens_used` after
+ * the envelope has been built.
+ */
+export function countEnvelopeTokens(envelopeJson) {
+    return countTokens(envelopeJson);
+}
+//# sourceMappingURL=budget.js.map

package/dist/gate/budget.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"budget.js","sourceRoot":"","sources":["../../src/gate/budget.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAiB,MAAM,aAAa,CAAC;AAEzD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAC3C,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AAEjC;;;;;;;;;;;;;;GAcG;AACH,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,OAAO,GAAa,WAAW,CAAC,WAAW,CAAC,CAAC;AAEnD;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,IAAI,CAAC,MAAM,GAAG,wBAAwB,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,qBAAqB,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,SAAiB;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,uBAAuB,CAAC,GAAG,aAAa,CAAC,CAAC;IAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC1B,CAAC;AAsBD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,UAAU,CACxB,KAAU,EACV,SAAiB,EACjB,SAA8B,EAC9B,IAAyB,EACzB,KAA0B;IAE1B,MAAM,MAAM,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC;IAChD,MAAM,IAAI,GAAgC,EAAE,CAAC;IAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QAErC,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;YAClB,qEAAqE;YACrE,MAAM,QAAQ,GAAqB;gBACjC,SAAS,EAAE,IAAI;gBACf,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC;gBACd,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC;gBACvB,MAAM,EAAE,IAAI;aACb,CAAC;YACF,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3D,IAAI,IAAI,GAAG,YAAY,GAAG,MAAM,EAAE,CAAC;gBACjC,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpB,IAAI,IAAI,YAAY,CAAC;YACrB,SAAS,GAAG,IAAI,CAAC,CAAC,6BAA6B;YAC/C,SAAS;QACX,CAAC;QAED,IAAI,IAAI,GAAG,IAAI,GAAG,MAAM,EAAE,CAAC;YACzB,SAAS,GAAG,IAAI,CAAC;YACjB,MAAM;QACR,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,IAAI,IAAI,IAAI,CAAC;IACf,CAAC;IAED,OAAO;QACL,IAAI;QACJ,SAAS;QACT,UAAU,EAAE,IAAI;QAChB,cAAc,EAAE,MAAM;KACvB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,YAAoB;IACtD,OAAO,WAAW,CAAC,YAAY,CAAC,CAAC;AACnC,CAAC"}

package/dist/ingest/chokidar.d.ts

@@ -0,0 +1,33 @@
+import type { IngestEvent, IngestSource, Scope } from './source.js';
+export interface ChokidarWatcherOptions {
+    /** Absolute path to the wiki root directory */
+    rootDir: string;
+    /** Which scope to tag events with */
+    scope: Scope;
+    /** Override debounce window — tests use this with fake timers */
+    debounceMs?: number;
+}
+export declare class ChokidarWatcher implements IngestSource {
+    private readonly options;
+    private watcher;
+    private pending;
+    private readonly debounceMs;
+    private onEvent;
+    constructor(options: ChokidarWatcherOptions);
+    start(onEvent: (ev: IngestEvent) => Promise<void>): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * Test-only: dispatch all pending events immediately, bypassing the
+     * debounce timer. Used by chokidar.test.ts so the test doesn't have to
+     * wait 2 real seconds.
+     */
+    __flushForTests(): Promise<void>;
+    /**
+     * Test-only: number of currently-queued events. Used to assert
+     * drop-oldest semantics.
+     */
+    __pendingCountForTests(): number;
+    private queueEvent;
+    private dispatch;
+}
+//# sourceMappingURL=chokidar.d.ts.map

package/dist/ingest/chokidar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chokidar.d.ts","sourceRoot":"","sources":["../../src/ingest/chokidar.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAmB,YAAY,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAgCrF,MAAM,WAAW,sBAAsB;IACrC,+CAA+C;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,KAAK,EAAE,KAAK,CAAC;IACb,iEAAiE;IACjE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAeD,qBAAa,eAAgB,YAAW,YAAY;IAMtC,OAAO,CAAC,QAAQ,CAAC,OAAO;IALpC,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,OAAO,CAAwC;IACvD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,OAAO,CAAqD;gBAEvC,OAAO,EAAE,sBAAsB;IAItD,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4CjE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B;;;;OAIG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAStC;;;OAGG;IACH,sBAAsB,IAAI,MAAM;IAIhC,OAAO,CAAC,UAAU;YA8BJ,QAAQ;CAQvB"}

package/dist/ingest/chokidar.js

@@ -0,0 +1,152 @@
+import { resolve } from 'node:path';
+import chokidar from 'chokidar';
+import { logger } from '../observability/logger.js';
+/**
+ * `ChokidarWatcher` — file-system implementation of `IngestSource` for KG-MCP Phase 2.
+ *
+ * Watches a wiki directory recursively for `*.md` changes, with the two
+ * critical adaptations for Pharos's wiki-updater (CLAUDE.md §Database Rules #4,
+ * §Architecture #3, Loop 6.5 A4):
+ *
+ * **2-second debounce per path** (NOT chokidar's default 50ms). The
+ * wiki-updater writes via atomic rename, which fires chokidar events in
+ * rapid bursts (`add` immediately followed by `change` from a follow-up
+ * write to the same file). A 50ms debounce coalesces some of these but
+ * misses the longer-tail bursts where the wiki-updater appends to log.md
+ * a few hundred milliseconds after the initial atomic rename. 2 seconds
+ * is the empirically-derived sweet spot.
+ *
+ * **Bounded queue with drop-oldest per source path**. If 3 events arrive
+ * for the same file before the 2s timer fires, only the latest event is
+ * dispatched. Earlier events are silently dropped. This is correct
+ * because each event represents the *current* state of the file — older
+ * events are already obsolete by the time we'd ingest them.
+ *
+ * **Initial scan**: chokidar emits `add` events for every existing file
+ * during its first pass (when `ignoreInitial: false`). These are dispatched
+ * just like change events, so a fresh `serve` start will trigger the
+ * ingester for every wiki file. The ingester's manifest fast-path
+ * (source_sha unchanged → noop) makes this cheap on warm starts.
+ */
+const DEBOUNCE_MS = 2_000;
+export class ChokidarWatcher {
+    options;
+    watcher = null;
+    pending = new Map();
+    debounceMs;
+    onEvent = null;
+    constructor(options) {
+        this.options = options;
+        this.debounceMs = options.debounceMs ?? DEBOUNCE_MS;
+    }
+    async start(onEvent) {
+        if (this.watcher) {
+            throw new Error('ChokidarWatcher.start called twice');
+        }
+        this.onEvent = onEvent;
+        const root = resolve(this.options.rootDir);
+        // Note: chokidar 4.x removed glob support — pass the root directory
+        // and filter paths in the event handlers. We deliberately do NOT use
+        // the `ignored` option here because in chokidar 4.x it's matched against
+        // every traversed path including the root directory itself, and a naive
+        // "not .md → ignore" check ignores the wiki dir before recursing into it.
+        // Filtering inside the per-event handlers is simpler and bug-free.
+        this.watcher = chokidar.watch(root, {
+            ignoreInitial: false,
+            // No awaitWriteFinish — we have our own debounce that's better-suited
+            // to the wiki-updater's atomic-rename pattern.
+            awaitWriteFinish: false,
+        });
+        const isMarkdown = (path) => path.toLowerCase().endsWith('.md');
+        this.watcher.on('add', (path) => {
+            if (isMarkdown(path))
+                this.queueEvent('file:added', path);
+        });
+        this.watcher.on('change', (path) => {
+            if (isMarkdown(path))
+                this.queueEvent('file:changed', path);
+        });
+        this.watcher.on('unlink', (path) => {
+            if (isMarkdown(path))
+                this.queueEvent('file:removed', path);
+        });
+        this.watcher.on('error', (err) => {
+            logger.error({ err, root }, 'chokidar error');
+        });
+        // Wait for the initial scan to complete so callers know all `add` events
+        // have been queued before start() resolves. chokidar emits `ready` once
+        // the initial scan finishes.
+        await new Promise((resolveWait) => {
+            this.watcher.once('ready', () => resolveWait());
+        });
+    }
+    async stop() {
+        // Cancel every pending debounce timer so we don't fire after stop.
+        for (const pending of this.pending.values()) {
+            clearTimeout(pending.timer);
+        }
+        this.pending.clear();
+        if (this.watcher) {
+            await this.watcher.close();
+            this.watcher = null;
+        }
+        this.onEvent = null;
+    }
+    /**
+     * Test-only: dispatch all pending events immediately, bypassing the
+     * debounce timer. Used by chokidar.test.ts so the test doesn't have to
+     * wait 2 real seconds.
+     */
+    async __flushForTests() {
+        const entries = Array.from(this.pending.entries());
+        this.pending.clear();
+        for (const [, pending] of entries) {
+            clearTimeout(pending.timer);
+            await this.dispatch(pending.event);
+        }
+    }
+    /**
+     * Test-only: number of currently-queued events. Used to assert
+     * drop-oldest semantics.
+     */
+    __pendingCountForTests() {
+        return this.pending.size;
+    }
+    queueEvent(kind, path) {
+        const abs = resolve(path);
+        const event = { kind, path: abs, scope: this.options.scope };
+        const existing = this.pending.get(abs);
+        if (existing) {
+            // Drop-oldest: replace the queued event but DON'T restart the timer.
+            // This caps a single file's queue depth at 1 even under heavy load,
+            // and bounds the worst-case latency to `debounceMs` after the first
+            // event in a burst.
+            existing.event = event;
+            return;
+        }
+        // First event for this path — start a fresh debounce timer.
+        const timer = setTimeout(() => {
+            const pending = this.pending.get(abs);
+            this.pending.delete(abs);
+            if (pending) {
+                // Fire-and-forget: dispatch errors are logged inside dispatch().
+                // We don't await here because the chokidar event loop must keep
+                // pumping new events; the ingester's single-flight gate handles
+                // concurrent dispatches for the same path.
+                void this.dispatch(pending.event);
+            }
+        }, this.debounceMs);
+        this.pending.set(abs, { event, timer });
+    }
+    async dispatch(event) {
+        if (!this.onEvent)
+            return;
+        try {
+            await this.onEvent(event);
+        }
+        catch (err) {
+            logger.error({ err, event }, 'ingest event handler failed');
+        }
+    }
+}
+//# sourceMappingURL=chokidar.js.map

package/dist/ingest/chokidar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chokidar.js","sourceRoot":"","sources":["../../src/ingest/chokidar.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,QAA4B,MAAM,UAAU,CAAC;AAEpD,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAGpD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,MAAM,WAAW,GAAG,KAAK,CAAC;AAwB1B,MAAM,OAAO,eAAe;IAMG;IALrB,OAAO,GAAqB,IAAI,CAAC;IACjC,OAAO,GAA8B,IAAI,GAAG,EAAE,CAAC;IACtC,UAAU,CAAS;IAC5B,OAAO,GAAgD,IAAI,CAAC;IAEpE,YAA6B,OAA+B;QAA/B,YAAO,GAAP,OAAO,CAAwB;QAC1D,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,WAAW,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAA2C;QACrD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QAEvB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAE3C,oEAAoE;QACpE,qEAAqE;QACrE,yEAAyE;QACzE,wEAAwE;QACxE,0EAA0E;QAC1E,mEAAmE;QACnE,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE;YAClC,aAAa,EAAE,KAAK;YACpB,sEAAsE;YACtE,+CAA+C;YAC/C,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEjF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,UAAU,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;YACjC,IAAI,UAAU,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;YACjC,IAAI,UAAU,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC/B,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,gBAAgB,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,yEAAyE;QACzE,wEAAwE;QACxE,6BAA6B;QAC7B,MAAM,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;YACtC,IAAI,CAAC,OAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAI;QACR,mEAAmE;QACnE,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,eAAe;QACnB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,KAAK,MAAM,CAAC,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;YAClC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,sBAAsB;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAEO,UAAU,CAAC,IAAqB,EAAE,IAAY;QACpD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,KAAK,GAAgB,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAE1E,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YACb,qEAAqE;YACrE,oEAAoE;YACpE,oEAAoE;YACpE,oBAAoB;YACpB,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;YACvB,OAAO;QACT,CAAC;QAED,4DAA4D;QAC5D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,OAAO,EAAE,CAAC;gBACZ,iEAAiE;gBACjE,gEAAgE;gBAChE,gEAAgE;gBAChE,2CAA2C;gBAC3C,KAAK,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEpB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,KAAkB;QACvC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAC1B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,6BAA6B,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;CACF"}

package/dist/ingest/ingester.d.ts

@@ -0,0 +1,117 @@
+import type { DbBundle } from '../db/client.js';
+import type { Embedder } from '../retrieval/embedder.js';
+import { type Manifest, fileSha, manifestPathFor } from './manifest.js';
+import { type Chunk as ChunkText } from './parse/chunk.js';
+/**
+ * IngesterService — the load-bearing single-flight writer for KG-MCP Phase 2.
+ *
+ * **What it does**: given an absolute path to a markdown file, parse it into
+ * sections, chunk each section, embed only the chunks whose sha changed, and
+ * upsert the result into SQLite under one transaction. Update the consistency
+ * manifest, append a row to `kg_log`. Errors trigger a clean rollback and an
+ * error log entry, but never throw past the caller's await — they're caught,
+ * logged, and re-thrown so the rebuild CLI can decide whether to abort or
+ * continue.
+ *
+ * **Key invariants** (from CLAUDE.md §Database Rules):
+ *
+ * - **Single writer per source_uri**: a module-level `Map<string, Promise>`
+ *   gates concurrent calls to `ingestFile(samePath)`. The second caller waits
+ *   on the first's promise, then exits early (no double-ingest). This is the
+ *   single-flight pattern; CLAUDE.md §Architecture #3 / §Database Rules #5.
+ *
+ * - **Per-chunk skip-unchanged**: on a re-ingest of an existing file, we
+ *   load the existing `kg_chunks` rows for that file's nodes, build a
+ *   `Set<chunk_sha>` of already-embedded chunks, and only call `embedder.embed()`
+ *   for chunks whose sha is NOT in the set. This is THE optimization that
+ *   makes Pharos's whole-file-rewrite-per-turn pattern viable — without it,
+ *   60 chunks × 50ms embed = 3s of blocking work per turn (Loop 6.5 A4).
+ *
+ * - **Transaction per file**: all DML for one file lives inside one
+ *   `BEGIN ... COMMIT` block. Partial failures roll back cleanly so the DB
+ *   never observes a half-ingested file. The FTS5 triggers fire inside the
+ *   transaction; sqlite-vec inserts also happen inside.
+ *
+ * - **Idempotent upserts**: node ids and chunk ids are deterministic
+ *   `sha1()` of stable inputs, so re-ingesting the same content produces
+ *   identical row ids. We use `INSERT OR REPLACE` so a re-ingest with
+ *   different content cleanly overwrites.
+ *
+ * - **Manifest is the tiebreaker on crash recovery**: written AFTER the
+ *   transaction commits. If we crash between COMMIT and writeManifest, the
+ *   next startup's consistency check will detect the divergence (DB has
+ *   the new chunks but manifest still has old chunk_shas) and re-ingest.
+ *   The work is duplicated but not wrong — that's the right safety/perf
+ *   tradeoff for pre-v1 sqlite-vec.
+ */
+export type Scope = 'project' | 'personal';
+export interface IngestResult {
+    /** Number of new chunks written (post-skip) */
+    chunks_added: number;
+    /** Number of chunks reused via per-chunk skip (no embedder call) */
+    chunks_skipped: number;
+    /** Number of embedder.embed() calls actually made */
+    embedder_calls: number;
+    /** Number of nodes (sections) written for this file */
+    nodes_written: number;
+    /** True if the file was unchanged and we exited early via the manifest fast path */
+    noop: boolean;
+}
+export interface IngesterOptions {
+    /** Optional override for the manifest file path (default: derived from wikiRoot) */
+    manifestPath?: string;
+}
+/** Test-only: clear the single-flight map. Do not call from production code. */
+export declare function __resetSingleFlightForTests(): void;
+export declare class IngesterService {
+    private readonly bundle;
+    private readonly embedder;
+    private readonly scope;
+    private manifest;
+    private readonly manifestPath;
+    constructor(bundle: DbBundle, embedder: Embedder, scope: Scope, wikiRoot: string, options?: IngesterOptions);
+    /**
+     * Ingest a single markdown file. Single-flight: concurrent calls for the
+     * same file return the same in-progress promise.
+     *
+     * Returns the per-chunk metrics for this run (added/skipped/embedder calls)
+     * so callers (rebuild CLI, chokidar handler) can log a summary.
+     */
+    ingestFile(absPath: string): Promise<IngestResult>;
+    /**
+     * Remove a file from the index (for `file:removed` chokidar events).
+     * Deletes the file's nodes (cascades to chunks/edges/vec rows via FK)
+     * and updates the manifest. Single-statement, no transaction needed.
+     */
+    removeFile(absPath: string): void;
+    /**
+     * The current in-memory manifest. Exposed for tests + the startup
+     * consistency check in `cli/serve.ts`.
+     */
+    getManifest(): Manifest;
+    /**
+     * Reload the manifest from disk. Used at startup before walking the
+     * consistency check, and by tests that mutate the manifest externally.
+     */
+    reloadManifest(): void;
+    private ingestFileImpl;
+    /**
+     * Load `chunk_sha → embedding` for every existing chunk of a file. Used by
+     * the per-chunk skip optimization: if a new chunk's sha matches one in
+     * this map, we reuse the embedding instead of calling the embedder.
+     *
+     * Returns an empty map if the file isn't in the DB yet (first ingest).
+     */
+    private loadExistingEmbeddings;
+    private runInTransaction;
+    private appendLog;
+}
+/**
+ * Reset the in-memory manifest to empty. Used by `kg rebuild` when the
+ * caller passes `--clean` to force a full re-ingest from scratch.
+ */
+export declare function freshManifest(): Manifest;
+export type { ChunkText };
+export { manifestPathFor };
+export { fileSha };
+//# sourceMappingURL=ingester.d.ts.map

package/dist/ingest/ingester.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingester.d.ts","sourceRoot":"","sources":["../../src/ingest/ingester.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAGzD,OAAO,EACL,KAAK,QAAQ,EAGb,OAAO,EACP,eAAe,EAKhB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAgB,KAAK,KAAK,IAAI,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEzE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAMH,MAAM,MAAM,KAAK,GAAG,SAAS,GAAG,UAAU,CAAC;AAE3C,MAAM,WAAW,YAAY;IAC3B,+CAA+C;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,cAAc,EAAE,MAAM,CAAC;IACvB,uDAAuD;IACvD,aAAa,EAAE,MAAM,CAAC;IACtB,oFAAoF;IACpF,IAAI,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAoBD,gFAAgF;AAChF,wBAAgB,2BAA2B,IAAI,IAAI,CAElD;AAMD,qBAAa,eAAe;IAKxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,KAAK;IANxB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAGnB,MAAM,EAAE,QAAQ,EAChB,QAAQ,EAAE,QAAQ,EAClB,KAAK,EAAE,KAAK,EAC7B,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB;IAM/B;;;;;;OAMG;IACG,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAsBxD;;;;OAIG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAcjC;;;OAGG;IACH,WAAW,IAAI,QAAQ;IAIvB;;;OAGG;IACH,cAAc,IAAI,IAAI;YAQR,cAAc;IAiM5B;;;;;;OAMG;IACH,OAAO,CAAC,sBAAsB;IA4B9B,OAAO,CAAC,gBAAgB;IAKxB,OAAO,CAAC,SAAS;CAYlB;AAsCD;;;GAGG;AACH,wBAAgB,aAAa,IAAI,QAAQ,CAExC;AAGD,YAAY,EAAE,SAAS,EAAE,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,CAAC;AAC3B,OAAO,EAAE,OAAO,EAAE,CAAC"}