@seanhogg/builderforce-memory 2026.10.27 → 2026.10.29

package/src/knowledge/review.test.ts

@@ -0,0 +1,180 @@
+/**
+ * src/knowledge/review.test.ts
+ * planReview / reviewDirectory with fake ports: unchanged files are never read, extracted
+ * output is never written as memory, a changed hash marks only that source stale, the
+ * budget stops judgement (and counts the rest), invalid judgement is dropped and counted,
+ * and abort keeps what was written consistent per file.
+ */
+
+import { InMemoryIngestManifest } from '../ingest/IngestionPipeline.js';
+import { hashText } from '../ingest/hash.js';
+import { buildGraph } from './build.js';
+import type { Extracted, JudgementRequest, KnowledgeStore, ReviewArgs, StoredFact, StoredRelation } from './model.js';
+import { DOCS_PACK } from './packs/docs.js';
+import { planReview } from './plan.js';
+import { assertStoredBatch, reviewDirectory } from './review.js';
+import { fakeJudge, memSource, memStore, ROOT, type SpySource } from './testkit.test-util.js';
+
+const BUDGET = { modelCalls: 200, ms: 60_000 };
+
+function args(source: SpySource, store: KnowledgeStore, manifest: InMemoryIngestManifest, extra: Partial<ReviewArgs> = {}): ReviewArgs {
+    return { source, store, manifest, packs: [DOCS_PACK], budget: BUDGET, ...extra };
+}
+
+async function graphOf(source: SpySource, store: KnowledgeStore) {
+    const extracted: Extracted[] = [];
+    for await (const f of source.list()) {
+        const hit = (await store.getExtracted!(f.hash))?.find((e) => e.sourceRef === `src:${ROOT}::${f.path}`);
+        if (hit) extracted.push(hit);
+    }
+    return buildGraph({
+        rootId: ROOT, extracted, packs: [DOCS_PACK],
+        facts: await store.facts({ rootId: ROOT }), relations: await store.relations({ rootId: ROOT }),
+    });
+}
+
+const anchoredTo = (facts: StoredFact[], path: string) => facts.filter((f) => f.fileAnchors.some((a) => a.path === path));
+
+describe('planReview', () => {
+    it('reads only the ignore files — never an unchanged file — and reports skips', async () => {
+        const source = memSource({
+            '.gitignore': 'skip/\n', 'a.md': '# A\nalpha\n', 'b.md': '# B\nbeta\n', 'img.png': 'x', 'skip/x.md': '# X\n',
+        });
+        const manifest = new InMemoryIngestManifest();
+        await reviewDirectory(args(source, memStore(), manifest));
+
+        source.files['a.md'] = '# A\nalpha changed\n';
+        source.reads.length = 0;
+        const plan = await planReview(source, manifest, [DOCS_PACK]);
+        expect(source.reads).toEqual(['.gitignore']);
+        expect(plan.changed).toEqual(['a.md']);
+        expect(plan.unchanged).toBe(1);
+        expect(plan.byPack).toEqual({ docs: 2 });
+        expect(plan.estimatedModelCalls).toBeGreaterThan(0);
+        expect(plan.skipped).toEqual(expect.arrayContaining([
+            { path: '.gitignore', reason: 'ignored' }, { path: 'img.png', reason: 'binary' }, { path: 'skip/x.md', reason: 'ignored' },
+        ]));
+
+        source.reads.length = 0;
+        await reviewDirectory(args(source, memStore(), manifest));
+        expect(source.reads).toEqual(['.gitignore', 'a.md']);
+    });
+
+    it('enforces the size cap and lists removed files', async () => {
+        const source = memSource({ 'a.md': '# A\nalpha\n', 'big.md': 'x'.repeat(50) });
+        const manifest = new InMemoryIngestManifest();
+        await reviewDirectory(args(source, memStore(), manifest));
+        delete source.files['a.md'];
+        const plan = await planReview(source, manifest, [DOCS_PACK], { maxFileBytes: 10 });
+        expect(plan.removed).toEqual(['a.md']);
+        expect(plan.skipped).toContainEqual({ path: 'big.md', reason: 'too-large' });
+    });
+});
+
+describe('reviewDirectory', () => {
+    it('never writes extracted output as memory; caches it by content hash instead', async () => {
+        const source = memSource({ 'a.md': '# A\nalpha\n', 'b.md': '# B\nbeta\n' });
+        const store = memStore();
+        const batches: Array<{ facts: StoredFact[]; relations: StoredRelation[] }> = [];
+        const write = store.write.bind(store);
+        store.write = (b) => { batches.push(b); return write(b); };
+        const judge = fakeJudge();
+        const report = await reviewDirectory(args(source, store, new InMemoryIngestManifest(), {
+            judgement: judge,
+            similar: async (ids) => (ids.length >= 2 ? [{ a: ids[0]!, b: ids[1]!, similarity: 0.9 }] : []),
+        }));
+
+        const written = batches.flatMap((b) => [...b.facts, ...b.relations]);
+        expect(written.length).toBeGreaterThan(0);
+        expect(written.every((x) => x.origin === 'model')).toBe(true);
+        expect(report.extracted).toBe(2);
+        expect(report.judged).toBe(3); // two sections + one compare
+        expect(report.factsWritten).toBeGreaterThan(0);
+        expect(await store.getExtracted(hashText('# A\nalpha\n'))).toHaveLength(1);
+        expect(await store.relations({ rootId: ROOT })).toContainEqual(expect.objectContaining({ kind: 'contradicts', confidence: 0.9 }));
+        expect((await store.scopes())[0]).toMatchObject({ rootId: ROOT, lastDigest: report.digest, packs: ['docs'] });
+    });
+
+    it('the store guard rejects extracted items', async () => {
+        const bad = { from: 'src:r::a', to: 'src:r::b', kind: 'links', weight: 1, origin: 'extracted' } as unknown as StoredRelation;
+        expect(() => assertStoredBatch({ facts: [], relations: [bad] })).toThrow();
+        await expect(memStore().write({ scope: { rootId: ROOT, title: 't', packs: [] }, facts: [], relations: [bad] })).rejects.toThrow();
+    });
+
+    it('a changed hash marks only that source stale; unchanged sections are re-anchored, not re-judged', async () => {
+        const source = memSource({ 'a.md': '# A\nalpha\n# A2\nsame\n', 'b.md': '# B\nbeta\n' });
+        const store = memStore();
+        const manifest = new InMemoryIngestManifest();
+        await reviewDirectory(args(source, store, manifest, { judgement: fakeJudge() }));
+
+        source.files['a.md'] = '# A\nalpha two\n# A2\nsame\n';
+        await reviewDirectory(args(source, store, manifest)); // extracted-only: no model
+        let g = await graphOf(source, store);
+        const facts = await store.facts({ rootId: ROOT });
+        const claimsA = anchoredTo(facts, 'a.md').filter((f) => f.key.startsWith('claim:'));
+        const claimsB = anchoredTo(facts, 'b.md').filter((f) => f.key.startsWith('claim:'));
+        expect(claimsA).toHaveLength(2);
+        for (const c of claimsA) expect(g.nodes.find((n) => n.ref === `fact:${c.key}`)?.stale).toBe(true);
+        for (const c of claimsB) expect(g.nodes.find((n) => n.ref === `fact:${c.key}`)?.stale).toBeUndefined();
+        const rels = await store.relations({ rootId: ROOT });
+        expect(rels.filter((r) => r.to.startsWith(`src:${ROOT}::a.md#`) && r.kind === 'about').every((r) => r.staleAt)).toBe(true);
+        expect(rels.filter((r) => r.to.startsWith(`src:${ROOT}::b.md#`)).every((r) => !r.staleAt)).toBe(true);
+
+        // Re-judging: only the changed section costs a call; the unchanged one is fresh again.
+        source.files['a.md'] = '# A\nalpha three\n# A2\nsame\n';
+        const judge = fakeJudge();
+        await reviewDirectory(args(source, store, manifest, { judgement: judge }));
+        expect(judge.requests.filter((r) => r.kind === 'extract').map((r) => r.items[0]!.text)).toEqual(['alpha three']);
+        g = await graphOf(source, store);
+        const same = (await store.facts({ rootId: ROOT })).find((f) => f.key.startsWith('claim:') && f.content.startsWith('Lease ends same'))!;
+        expect(g.nodes.find((n) => n.ref === `fact:${same.key}`)?.stale).toBeUndefined();
+    });
+
+    it('the budget stops judgement; the rest is counted unjudged and stays changed', async () => {
+        const source = memSource({ 'a.md': '# A\nalpha\n# B\nbeta\n', 'b.md': '# C\ngamma\n' });
+        const manifest = new InMemoryIngestManifest();
+        const judge = fakeJudge();
+        const report = await reviewDirectory(args(source, memStore(), manifest, { judgement: judge, budget: { modelCalls: 1, ms: 60_000 } }));
+        expect(judge.requests).toHaveLength(1);
+        expect(report.judged).toBe(1);
+        expect(report.unjudged).toBe(2);
+        expect(report.extracted).toBe(2);
+        expect((await planReview(source, manifest, [DOCS_PACK])).changed).toEqual(['a.md', 'b.md']);
+    });
+
+    it('drops invalid judgement output and counts it', async () => {
+        const judge = fakeJudge((req: JudgementRequest) => ({
+            extract: [{
+                ref: req.items[0]!.ref,
+                claims: [
+                    { subject: 'Lease', predicate: 'ends', object: '2027', quote: 'ends 2027' },
+                    { subject: 'Lease', object: 'x', quote: '' } as never,
+                ],
+                entities: [{ name: '', type: 'thing' }, { name: 'Lease', type: 'contract' }],
+            }],
+            invalid: 1,
+        }));
+        const store = memStore();
+        const report = await reviewDirectory(args(memSource({ 'a.md': '# A\nalpha\n' }), store, new InMemoryIngestManifest(), { judgement: judge }));
+        expect(report.invalidJudgements).toBe(3);
+        const facts = await store.facts({ rootId: ROOT });
+        expect(facts.filter((f) => f.key.startsWith('claim:'))).toHaveLength(1);
+        expect(facts.map((f) => f.key)).toContain('ent:docs:lease');
+    });
+
+    it('abort between files keeps the written file whole and stops before the next', async () => {
+        const source = memSource({ 'a.md': '# A\nalpha\n', 'b.md': '# B\nbeta\n' });
+        const store = memStore();
+        const controller = new AbortController();
+        const report = await reviewDirectory(args(source, store, new InMemoryIngestManifest(), {
+            judgement: fakeJudge(),
+            signal: controller.signal,
+            onProgress: (p) => { if (p.phase === 'write') controller.abort(); },
+        }));
+        expect(report.aborted).toBe(true);
+        const facts = await store.facts({ rootId: ROOT });
+        expect(anchoredTo(facts, 'a.md').length).toBeGreaterThan(0);
+        expect(anchoredTo(facts, 'b.md')).toHaveLength(0);
+        expect(source.reads).not.toContain('b.md');
+    });
+});

package/src/knowledge/review.ts

@@ -0,0 +1,235 @@
+/**
+ * knowledge/review.ts — `reviewDirectory`: point it at a folder and it learns what the
+ * folder says (PRD 38 §4.3).
+ *
+ *   plan (walk + manifest diff) → per changed file: read · parse (the ONE ParserRegistry,
+ *   composed from the packs) · extract (cached by content hash) · judge (changed sections
+ *   of judged kinds, within budget) · write · markStale → analyzers → compare near-duplicate
+ *   claims → manifest + scope digest.
+ *
+ * Store judgement, compute perception: only `model` facts and relations are written —
+ * `extracted` output goes to the extracted cache, never to memory ({@link assertStoredBatch}).
+ * Abort is honoured between files; a file is written whole or not at all. A file whose
+ * sections were cut by the budget stays "changed" in the manifest, so continuing re-plans it
+ * and judges only the sections not judged yet.
+ */
+
+import { hashText } from '../ingest/hash.js';
+import { sha256Hex } from '../retrieval/knowledge-index/sha256.js';
+import type {
+    ExtractContext, Extracted, KnowledgeParser, KnowledgeStore, NodeRef, ReviewArgs, ReviewReport, ScopeRecord,
+    StoredFact, StoredRelation,
+} from './model.js';
+import type { SourceDocument, ParsedSection } from '../ingest/types.js';
+import { SECTION_INDEX_BADGE } from '../ingest/docsExtract.js';
+import { planDetail } from './plan.js';
+import {
+    asPathSource, composePacks, filesManifestKey, isBinaryText, judgedManifestKey, manifestMap, resolveRelativePath,
+    saveManifestMap,
+} from './sources.js';
+import {
+    budgetLeft, compareClaims, DEFAULT_REVIEW_MODEL_CALLS, JudgedBatch, judgeSections, type JudgeBudget,
+} from './judge.js';
+import { entityName, knownKeysOf, resolveEntities } from './resolve.js';
+import { basenameOf, canonicalJson, compareStr, factKeyOf, srcRef } from './refs.js';
+
+/** Throws if a batch carries `extracted` items — perception is never memory. */
+export function assertStoredBatch(batch: { facts: readonly StoredFact[]; relations: readonly StoredRelation[] }): void {
+    const bad = [...batch.facts, ...batch.relations].find((x) => (x.origin as string) === 'extracted');
+    if (bad) throw new Error('knowledge: extracted output is never written to a KnowledgeStore');
+}
+
+async function extractedFor(
+    store: KnowledgeStore, parser: KnowledgeParser, pack: string, src: SourceDocument, sections: ParsedSection[],
+    ctx: ExtractContext, hash: string,
+): Promise<Extracted> {
+    const ref = srcRef(ctx.rootId, src.id);
+    const cached = await store.getExtracted?.(hash);
+    const hit = cached?.find((e) => e.sourceRef === ref);
+    if (hit) return hit;
+    const ex: Extracted = parser.extract
+        ? { ...parser.extract(src, sections, ctx), sourceHash: hash }
+        : {
+            sourceRef: ref, sourceHash: hash, edges: [], keys: [{ ref, key: src.id }],
+            nodes: [{ ref, kind: 'file', label: basenameOf(src.id), pack, importance: 0, origin: 'extracted', sourceHash: hash }],
+        };
+    await store.putExtracted?.(hash, [...(cached ?? []).filter((e) => e.sourceRef !== ref), ex]);
+    return ex;
+}
+
+/** Stored knowledge, indexed for re-anchoring unchanged sections without a model call. */
+class StoredIndex {
+    readonly facts: Map<string, StoredFact>;
+    private readonly from = new Map<string, StoredRelation[]>();
+    private readonly to = new Map<string, StoredRelation[]>();
+
+    constructor(facts: StoredFact[], relations: StoredRelation[]) {
+        this.facts = new Map(facts.map((f) => [f.key, f] as const));
+        for (const r of relations) {
+            if (r.origin !== 'model') continue;
+            (this.from.get(r.from) ?? this.from.set(r.from, []).get(r.from)!).push(r);
+            (this.to.get(r.to) ?? this.to.set(r.to, []).get(r.to)!).push(r);
+        }
+    }
+
+    /** An unchanged section keeps its claims and mentions; they are re-anchored to the file's new hash. */
+    refresh(section: NodeRef, path: string, hash: string, batch: JudgedBatch, entities: Map<string, StoredFact>): void {
+        const reanchor = (f: StoredFact): StoredFact => ({
+            ...f, fileAnchors: f.fileAnchors.map((a) => (a.path === path ? { ...a, contentHash: hash } : a)),
+        });
+        const relation = (r: StoredRelation) => batch.relation({ ...r, sourceHash: hash, staleAt: null });
+        for (const r of this.to.get(section) ?? []) {
+            if (r.kind !== 'about') continue;
+            relation(r);
+            const key = factKeyOf(r.from);
+            const fact = key && (batch.facts.get(key) ?? this.facts.get(key));
+            if (fact && fact.origin === 'model') batch.fact(reanchor(fact));
+            for (const m of this.from.get(r.from) ?? []) if (m.kind === 'mentions') relation(m);
+        }
+        for (const r of this.from.get(section) ?? []) {
+            if (r.kind !== 'mentions') continue;
+            relation(r);
+            const ent = entities.get(r.to);
+            if (ent && ent.origin === 'model') {
+                const next = reanchor(ent);
+                entities.set(r.to, next);
+                batch.fact(next);
+            }
+        }
+    }
+}
+
+export async function reviewDirectory(args: ReviewArgs): Promise<ReviewReport> {
+    const now = () => Date.now();
+    const started = now();
+    const { source, store, packs, manifest, signal } = args;
+    const rootId = source.rootId;
+    const scope = { rootId };
+    const progress = (p: Parameters<NonNullable<ReviewArgs['onProgress']>>[0]) => args.onProgress?.(p);
+    const report: ReviewReport = {
+        rootId, digest: '', extracted: 0, judged: 0, unjudged: 0, invalidJudgements: 0,
+        factsWritten: 0, relationsWritten: 0, staleMarked: 0, aborted: false, durationMs: 0,
+    };
+
+    const detail = await planDetail(source, manifest, packs);
+    progress({ phase: 'plan', done: detail.files.length, total: detail.files.length });
+    const composed = composePacks(packs);
+    const packIds = packs.map((p) => p.id).sort(compareStr);
+    const scopeRecord: ScopeRecord = { rootId, title: source.title, packs: packIds };
+    const [storedFacts, storedRelations] = await Promise.all([store.facts(scope), store.relations(scope)]);
+    const stored = new StoredIndex(storedFacts, storedRelations);
+    const entities = new Map(storedFacts.filter((f) => f.key.startsWith('ent:')).map((f) => [f.key, f] as const));
+    const ctx: ExtractContext = {
+        rootId,
+        knownKeys: knownKeysOf(resolveEntities([...entities.keys()].map((k) => ({ ref: k as NodeRef, key: entityName(k) })))),
+        resolvePath: resolveRelativePath,
+    };
+    const budget: JudgeBudget = {
+        modelCalls: args.budget?.modelCalls ?? DEFAULT_REVIEW_MODEL_CALLS,
+        deadline: started + (Number.isFinite(args.budget?.ms) ? args.budget.ms : Number.POSITIVE_INFINITY),
+        used: 0, now,
+    };
+    const batch = new JudgedBatch();
+    const judgedPrior = await manifestMap(manifest, judgedManifestKey(rootId));
+    const judgedNow = new Map(judgedPrior);
+    const filesNow = new Map(detail.prior);
+    const newClaims = new Map<string, { text: string; hash: string }>();
+
+    const write = async (withScope: ScopeRecord = scopeRecord) => {
+        const out = batch.take();
+        assertStoredBatch(out);
+        if (out.facts.length === 0 && out.relations.length === 0 && withScope === scopeRecord) return;
+        await store.write({ scope: withScope, ...out });
+        report.factsWritten += out.facts.length;
+        report.relationsWritten += out.relations.length;
+    };
+
+    const changed = detail.files.filter((f) => f.changed);
+    for (const [i, f] of changed.entries()) {
+        if (signal?.aborted) { report.aborted = true; break; }
+        progress({ phase: 'extract', done: i, total: changed.length, path: f.path });
+        const doc = await source.read(f.path);
+        if (doc.content.kind === 'text' && isBinaryText(doc.content.text)) continue;
+        const src = asPathSource(doc, f.path, f.hash);
+        const hit = composed.resolve(src);
+        if (!hit) continue;
+        const sections = hit.parser.parse(src);
+        const ex = await extractedFor(store, hit.parser, hit.pack.id, src, sections, ctx, f.hash);
+        report.extracted += 1;
+
+        const spec = hit.pack.judgement;
+        let complete = true;
+        if (spec && args.judgement && (spec.claims || spec.entities)) {
+            const kinds = new Set(spec.nodeKinds);
+            const toJudge: Array<{ ref: NodeRef; text: string }> = [];
+            for (const node of ex.nodes) {
+                const index = node.badges?.[SECTION_INDEX_BADGE];
+                const text = typeof index === 'number' ? sections[index]?.text : undefined;
+                if (!kinds.has(node.kind) || !text) continue;
+                if (judgedPrior.get(node.ref) === hashText(text)) stored.refresh(node.ref, f.path, f.hash, batch, entities);
+                else toJudge.push({ ref: node.ref, text });
+            }
+            const unjudgedBefore = batch.unjudged;
+            progress({ phase: 'judge', done: i, total: changed.length, path: f.path });
+            try {
+                const claims = await judgeSections({
+                    port: args.judgement, pack: hit.pack.id, path: f.path, fileHash: f.hash, sections: toJudge,
+                    budget, batch, entities, ...(signal ? { signal } : {}),
+                    onJudged: (ref) => judgedNow.set(ref, hashText(toJudge.find((s) => s.ref === ref)!.text)),
+                });
+                for (const c of claims) newClaims.set(c.ref, { text: c.text, hash: f.hash });
+            } catch (err) {
+                if (!signal?.aborted) throw err;
+                batch.take();
+                report.aborted = true;
+                break;
+            }
+            complete = batch.unjudged === unjudgedBefore;
+        }
+        progress({ phase: 'write', done: i, total: changed.length, path: f.path });
+        await write();
+        await store.markStale(scope, srcRef(rootId, f.path), f.hash);
+        report.staleMarked += 1;
+        if (complete) filesNow.set(f.path, f.hash);
+    }
+
+    if (!report.aborted) {
+        for (const path of detail.plan.removed) {
+            await store.markStale(scope, srcRef(rootId, path), '');
+            report.staleMarked += 1;
+            filesNow.delete(path);
+        }
+        for (const pack of packs) {
+            if (!pack.analyzers) continue;
+            for (const ex of await pack.analyzers.analyse(rootId, signal)) {
+                await store.putExtracted?.(ex.sourceHash, [ex]);
+                report.extracted += 1;
+            }
+        }
+        progress({ phase: 'resolve', done: 0, total: newClaims.size });
+        const wantsCompare = packs.some((p) => p.judgement?.contradictions);
+        if (wantsCompare && args.judgement && args.similar && newClaims.size > 0 && budgetLeft(budget)) {
+            const pairs = await args.similar([...newClaims.keys()].sort(compareStr));
+            const claims = new Map(newClaims);
+            for (const p of pairs) {
+                for (const id of [p.a, p.b]) {
+                    const fact = !claims.has(id) ? stored.facts.get(factKeyOf(id as NodeRef) ?? '') : undefined;
+                    if (fact?.key.startsWith('claim:')) claims.set(id, { text: fact.content, hash: fact.fileAnchors[0]?.contentHash ?? '' });
+                }
+            }
+            progress({ phase: 'judge', done: 0, total: pairs.length });
+            await compareClaims({ port: args.judgement, pairs, claims, budget, batch, ...(signal ? { signal } : {}) });
+        }
+    }
+
+    report.judged = batch.judged;
+    report.unjudged = batch.unjudged;
+    report.invalidJudgements = batch.invalid;
+    report.digest = sha256Hex(canonicalJson({ rootId, packs: packIds, files: [...filesNow].sort((a, b) => compareStr(a[0], b[0])) }));
+    await saveManifestMap(manifest, filesManifestKey(rootId), filesNow);
+    await saveManifestMap(manifest, judgedManifestKey(rootId), judgedNow);
+    await write(report.aborted ? scopeRecord : { ...scopeRecord, lastDigest: report.digest, lastReviewedAt: new Date(now()).toISOString() });
+    report.durationMs = now() - started;
+    progress({ phase: 'done', done: changed.length, total: changed.length });
+    return report;
+}

package/src/knowledge/sources.ts

@@ -0,0 +1,146 @@
+/**
+ * knowledge/sources.ts — what a review needs to know about a file BEFORE (or without)
+ * reading it: its pack, its media type, whether it is binary or ignored. Shared by
+ * `planReview` and `reviewDirectory`, so the plan's numbers and the run agree.
+ */
+
+import { ParserRegistry } from '../ingest/parsers.js';
+import { CONTENT_HASH_FIELD, type ChunkFingerprint, type IngestManifest, type SourceDocument } from '../ingest/types.js';
+import type { KnowledgePack, KnowledgeParser } from './model.js';
+import { matchGlob } from './partitions.js';
+import { compareStr } from './refs.js';
+
+/** Default per-file size cap (bytes). */
+export const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024;
+/** Ignore files read through the source, when present. */
+export const IGNORE_FILES = ['.gitignore', '.builderforce/ignore'] as const;
+/** Always ignored: VCS internals, installed dependencies, builderforce config. */
+const DEFAULT_IGNORES = ['.git/', 'node_modules/', '.builderforce/', '/.gitignore'];
+
+const MEDIA_TYPES: Record<string, string> = {
+    md: 'text/markdown', markdown: 'text/markdown', mdx: 'text/markdown',
+    html: 'text/html', htm: 'text/html', xhtml: 'application/xhtml+xml', xml: 'application/xml',
+    json: 'application/json', csv: 'text/csv', tsv: 'text/tab-separated-values',
+};
+
+const BINARY_EXT = new Set([
+    'png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'ico', 'icns', 'tif', 'tiff', 'psd', 'avif', 'heic',
+    'mp3', 'mp4', 'mov', 'avi', 'mkv', 'webm', 'wav', 'flac', 'ogg', 'm4a',
+    'zip', 'gz', 'tgz', 'bz2', 'xz', '7z', 'rar', 'tar', 'jar', 'war',
+    'exe', 'dll', 'so', 'dylib', 'bin', 'o', 'a', 'class', 'wasm', 'node', 'pyc',
+    'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'odt', 'ods', 'odp',
+    'ttf', 'otf', 'woff', 'woff2', 'eot', 'sqlite', 'db', 'vsix', 'lock',
+]);
+
+const extOf = (path: string): string => {
+    const m = /\.([A-Za-z0-9]+)$/.exec(path);
+    return m ? m[1]!.toLowerCase() : '';
+};
+
+export function mediaTypeOfPath(path: string): string {
+    return MEDIA_TYPES[extOf(path)] ?? 'text/plain';
+}
+
+/** By extension — the plan never reads a file to decide. (pdf/office text needs a host extractor.) */
+export function isBinaryPath(path: string): boolean {
+    return BINARY_EXT.has(extOf(path));
+}
+
+/** A NUL in the first 8 KB means binary, whatever the extension said. */
+export function isBinaryText(text: string): boolean {
+    return text.slice(0, 8192).includes('\u0000');
+}
+
+/** A read source with its id set to the root-relative path and a media type filled in. */
+export function asPathSource(doc: SourceDocument, path: string, hash: string): SourceDocument {
+    const content = doc.content.kind === 'text' && !doc.content.mediaType
+        ? { ...doc.content, mediaType: mediaTypeOfPath(path) }
+        : doc.content;
+    return { ...doc, id: path, content, metadata: { ...(doc.metadata ?? {}), [CONTENT_HASH_FIELD]: hash } };
+}
+
+/** A content-free stand-in, so `accepts` can pick a pack without a read. */
+export function stubSource(path: string): SourceDocument {
+    return { id: path, tenantId: '', content: { kind: 'text', text: '', mediaType: mediaTypeOfPath(path) } };
+}
+
+export interface ComposedPacks {
+    /** The ONE registry, composed from the packs' parsers in pack order. */
+    registry: ParserRegistry;
+    resolve(source: SourceDocument): { parser: KnowledgeParser; pack: KnowledgePack } | undefined;
+}
+
+export function composePacks(packs: readonly KnowledgePack[]): ComposedPacks {
+    const owner = new Map<KnowledgeParser, KnowledgePack>();
+    for (const pack of packs) for (const p of pack.parsers) if (!owner.has(p)) owner.set(p, pack);
+    const registry = new ParserRegistry([...owner.keys()]);
+    return {
+        registry,
+        resolve(source) {
+            const parser = registry.resolve(source) as KnowledgeParser | undefined;
+            const pack = parser && owner.get(parser);
+            return parser && pack ? { parser, pack } : undefined;
+        },
+    };
+}
+
+/**
+ * A `.gitignore`-style matcher: `#` comments, `!` negation (last match wins), a trailing
+ * `/` for directories only, a `/` elsewhere anchors to the root, otherwise any depth.
+ */
+export function ignoreMatcher(texts: readonly string[]): (path: string) => boolean {
+    const rules: Array<{ glob: string; negate: boolean; dirOnly: boolean }> = [];
+    for (const line of [...DEFAULT_IGNORES, ...texts.flatMap((t) => t.split(/\r?\n/))]) {
+        let p = line.replace(/\s+$/, '');
+        if (!p || p.startsWith('#')) continue;
+        const negate = p.startsWith('!');
+        if (negate) p = p.slice(1);
+        const dirOnly = p.endsWith('/');
+        if (dirOnly) p = p.slice(0, -1);
+        const anchored = p.includes('/');
+        p = p.replace(/^\//, '');
+        if (p) rules.push({ glob: anchored ? p : `**/${p}`, negate, dirOnly });
+    }
+    return (path) => {
+        const segments = path.split('/');
+        const dirs = segments.slice(0, -1).map((_, i) => segments.slice(0, i + 1).join('/'));
+        let ignored = false;
+        for (const r of rules) {
+            const hit = (r.dirOnly ? dirs : [...dirs, path]).some((c) => matchGlob(c, r.glob));
+            if (hit) ignored = !r.negate;
+        }
+        return ignored;
+    };
+}
+
+/** Manifest entry holding `{ id: path, hash }` for every reviewed file of a root. */
+export const filesManifestKey = (rootId: string): string => `knowledge:${rootId}`;
+/** Manifest entry holding `{ id: sectionRef, hash: textHash }` for every judged section. */
+export const judgedManifestKey = (rootId: string): string => `knowledge:${rootId}:judged`;
+
+export async function manifestMap(manifest: IngestManifest, key: string): Promise<Map<string, string>> {
+    const prior: ChunkFingerprint[] = (await manifest.get(key)) ?? [];
+    return new Map(prior.map((f) => [f.id, f.hash] as const));
+}
+
+export async function saveManifestMap(manifest: IngestManifest, key: string, map: ReadonlyMap<string, string>): Promise<void> {
+    const entries = [...map].sort((a, b) => compareStr(a[0], b[0])).map(([id, hash]) => ({ id, hash }));
+    await manifest.set(key, entries);
+}
+
+/**
+ * Resolves a link/import target against the linking file's folder (a leading `/` is the
+ * root). Pure path arithmetic: undefined only when the target escapes the root.
+ */
+export function resolveRelativePath(fromPath: string, target: string): string | undefined {
+    const t = target.replace(/\\/g, '/');
+    const out = t.startsWith('/') ? [] : fromPath.split('/').slice(0, -1);
+    for (const seg of t.split('/')) {
+        if (seg === '' || seg === '.') continue;
+        if (seg === '..') {
+            if (out.length === 0) return undefined;
+            out.pop();
+        } else out.push(seg);
+    }
+    return out.length ? out.join('/') : undefined;
+}

package/src/knowledge/stores/localSnapshotStore.test.ts

@@ -0,0 +1,82 @@
+/**
+ * src/knowledge/stores/localSnapshotStore.test.ts
+ * Upserts on the platform's unique keys (fact key; relation from+to+kind; scope rootId),
+ * markStale scoped to one source, canonical bytes, and the extracted guard.
+ */
+
+import type { ScopeRecord, StoredFact, StoredRelation } from '../model.js';
+import { LocalSnapshotStore } from './localSnapshotStore.js';
+import { memPersistence } from '../testkit.test-util.js';
+
+const SCOPE: ScopeRecord = { rootId: 'r', title: 'R', packs: ['docs'] };
+const fact = (key: string, content: string): StoredFact => ({
+    key, content, source: 's', importance: 0.5, origin: 'model', fileAnchors: [{ path: 'a.md', contentHash: 'h1' }],
+});
+const rel = (to: string, kind: string, extra: Partial<StoredRelation> = {}): StoredRelation => ({
+    from: 'fact:claim:1', to: to as StoredRelation['to'], kind, weight: 1, origin: 'model', sourceHash: 'h1', ...extra,
+});
+
+const store = () => new LocalSnapshotStore(memPersistence(), { now: () => 'T' });
+
+describe('LocalSnapshotStore', () => {
+    it('upserts facts by key and relations by (from, to, kind)', async () => {
+        const s = store();
+        await s.write({ scope: SCOPE, facts: [fact('claim:1', 'v1'), fact('claim:2', 'x')], relations: [rel('src:r::a.md#a', 'about')] });
+        await s.write({
+            scope: SCOPE,
+            facts: [fact('claim:1', 'v2')],
+            relations: [rel('src:r::a.md#a', 'about', { weight: 3 }), rel('src:r::a.md#a', 'mentions')],
+        });
+        const facts = await s.facts({ rootId: 'r' });
+        expect(facts.map((f) => [f.key, f.content])).toEqual(expect.arrayContaining([['claim:1', 'v2'], ['claim:2', 'x']]));
+        expect(facts).toHaveLength(2);
+        const rels = await s.relations({ rootId: 'r' });
+        expect(rels).toHaveLength(2);
+        expect(rels.find((r) => r.kind === 'about')?.weight).toBe(3);
+        expect(await s.facts({ rootId: 'other' })).toEqual([]);
+    });
+
+    it('upserts the scope by rootId without dropping fields a later write omits', async () => {
+        const s = store();
+        await s.write({ scope: { ...SCOPE, lastDigest: 'd1', lastReviewedAt: 'T1' }, facts: [], relations: [] });
+        await s.write({ scope: SCOPE, facts: [], relations: [] });
+        expect(await s.scopes()).toEqual([{ ...SCOPE, lastDigest: 'd1', lastReviewedAt: 'T1' }]);
+    });
+
+    it('markStale touches only model relations on that source judged from another hash', async () => {
+        const s = store();
+        await s.write({
+            scope: SCOPE, facts: [],
+            relations: [
+                rel('src:r::a.md#a', 'about'),
+                rel('src:r::a.md#b', 'about', { sourceHash: 'h2' }),
+                rel('src:r::b.md#c', 'about'),
+                rel('src:r::a.md#d', 'about', { origin: 'human', sourceHash: undefined }),
+            ],
+        });
+        await s.markStale({ rootId: 'r' }, 'src:r::a.md', 'h2');
+        const byTo = new Map((await s.relations({ rootId: 'r' })).map((r) => [r.to, r.staleAt] as const));
+        expect(byTo.get('src:r::a.md#a')).toBe('T');
+        expect(byTo.get('src:r::a.md#b')).toBeUndefined();
+        expect(byTo.get('src:r::b.md#c')).toBeUndefined();
+        expect(byTo.get('src:r::a.md#d')).toBeUndefined();
+    });
+
+    it('persists canonically: the same content in any order is the same bytes, and reloads', async () => {
+        const p1 = memPersistence(), p2 = memPersistence();
+        const a = new LocalSnapshotStore(p1), b = new LocalSnapshotStore(p2);
+        await a.write({ scope: SCOPE, facts: [fact('k1', '1'), fact('k2', '2')], relations: [] });
+        await b.write({ scope: SCOPE, facts: [fact('k2', '2'), fact('k1', '1')], relations: [] });
+        expect(p1.value).toBe(p2.value);
+        expect(await new LocalSnapshotStore(p1).facts({ rootId: 'r' })).toHaveLength(2);
+    });
+
+    it('caches extracted output by hash and refuses extracted items as memory', async () => {
+        const s = store();
+        await s.putExtracted('h', [{ sourceRef: 'src:r::a.md', sourceHash: 'h', nodes: [], edges: [], keys: [] }]);
+        expect(await s.getExtracted('h')).toHaveLength(1);
+        expect(await s.getExtracted('nope')).toBeUndefined();
+        const bad = { ...fact('k', 'c'), origin: 'extracted' } as unknown as StoredFact;
+        await expect(s.write({ scope: SCOPE, facts: [bad], relations: [] })).rejects.toThrow();
+    });
+});