@seanhogg/builderforce-memory 2026.10.28 → 2026.10.29

package/src/ingest/docsExtract.ts

@@ -0,0 +1,219 @@
+/**
+ * ingest/docsExtract.ts — the knowledge-graph `extract` of the builtin text parsers
+ * (markdown, HTML, plain text). PRD 38 §4.5, the docs pack.
+ *
+ * Pure: a function of the source text, its root-relative path (`source.id`), the parsed
+ * sections and the extract context. Emits, all with `origin: 'extracted'`:
+ *
+ *   • a `file` node, and a `section` node per heading breadcrumb, joined by `contains`;
+ *   • `links` edges for relative markdown links, `[[wiki]]` links and HTML hrefs, resolved
+ *     through `ctx.resolvePath` (an unresolvable target is skipped; a missing one stays an
+ *     edge, so the graph can report it broken);
+ *   • `mentions` edges where a section's normalized text contains a known entity key;
+ *   • keys: path, basename, heading slugs and front-matter values.
+ *
+ * A section node carries `badges.section` (its index in the parser's output) so the review
+ * can hand exactly that text to judgement, and `badges.excerpt` for the index card.
+ */
+
+import type { ExtractContext, Extracted, KEdge, KNode, NodeRef } from '../knowledge/model.js';
+import { basenameOf, compareStr, normalizeKey, slugOf, srcRef } from '../knowledge/refs.js';
+import { hashText } from './hash.js';
+import { CONTENT_HASH_FIELD, type ParsedSection, type SourceDocument } from './types.js';
+
+export const DOCS_PACK_ID = 'docs';
+/** Badge: index of the section in the parser's output. */
+export const SECTION_INDEX_BADGE = 'section';
+const EXCERPT_CHARS = 400;
+const BREADCRUMB_SEPARATOR = ' › ';
+
+type Ctx = ExtractContext;
+
+const textOf = (source: SourceDocument): string => (source.content.kind === 'text' ? source.content.text : '');
+
+class ExtractBuilder {
+    readonly path: string;
+    readonly hash: string;
+    readonly fileRef: NodeRef;
+    private readonly nodes = new Map<NodeRef, KNode>();
+    private readonly edges = new Map<string, KEdge>();
+    private readonly keys = new Map<string, { ref: NodeRef; key: string }>();
+
+    constructor(source: SourceDocument, private readonly ctx: Ctx, label?: string) {
+        this.path = source.id;
+        const stated = source.metadata?.[CONTENT_HASH_FIELD];
+        this.hash = typeof stated === 'string' ? stated : hashText(textOf(source));
+        this.fileRef = srcRef(ctx.rootId, this.path);
+        const base = basenameOf(this.path);
+        this.node(this.fileRef, 'file', label || base);
+        this.key(this.fileRef, this.path);
+        this.key(this.fileRef, base.toLowerCase());
+        this.key(this.fileRef, normalizeKey(base.replace(/\.[^.]+$/, '')));
+    }
+
+    node(ref: NodeRef, kind: string, label: string, badges?: Record<string, number | string>): KNode {
+        const node: KNode = {
+            ref, kind, label, pack: DOCS_PACK_ID, importance: 0, origin: 'extracted', sourceHash: this.hash,
+            ...(badges ? { badges } : {}),
+        };
+        this.nodes.set(ref, node);
+        return node;
+    }
+
+    edge(from: NodeRef, to: NodeRef, kind: string): void {
+        if (from === to) return;
+        const id = `${from}\u0000${to}\u0000${kind}`;
+        const prior = this.edges.get(id);
+        if (prior) prior.weight += 1;
+        else this.edges.set(id, { from, to, kind, weight: 1, origin: 'extracted' });
+    }
+
+    key(ref: NodeRef, key: string): void {
+        if (key) this.keys.set(`${ref}\u0000${key}`, { ref, key });
+    }
+
+    /** `links` edges to each resolvable target. */
+    link(from: NodeRef, target: string): void {
+        const clean = target.split('#')[0]!.split('?')[0]!.trim();
+        if (!clean || /^[a-z][a-z0-9+.-]*:/i.test(clean)) return;
+        let decoded = clean;
+        try { decoded = decodeURI(clean); } catch { /* keep the raw target */ }
+        const resolved = this.ctx.resolvePath(this.path, decoded);
+        if (resolved !== undefined && resolved !== this.path) this.edge(from, srcRef(this.ctx.rootId, resolved), 'links');
+    }
+
+    wikiLinks(from: NodeRef, text: string): void {
+        for (const m of text.matchAll(/\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|[^\]]*)?\]\]/g)) {
+            const name = m[1]!.trim();
+            if (name) this.link(from, /\.[A-Za-z][A-Za-z0-9]{0,7}$/.test(name) ? name : `${name}.md`);
+        }
+    }
+
+    /** `mentions` edges where the normalized text contains a known key as whole words. */
+    mentions(from: NodeRef, text: string): void {
+        if (this.ctx.knownKeys.size === 0) return;
+        const padded = ` ${normalizeKey(text)} `;
+        const known = [...this.ctx.knownKeys.entries()].sort((a, b) => compareStr(a[0], b[0]));
+        for (const [key, ref] of known) {
+            if (key && padded.includes(` ${key} `)) this.edge(from, ref, 'mentions');
+        }
+    }
+
+    /** A whole-body `section` (HTML, plain text): judged like a markdown section. */
+    bodySection(index: number, text: string): NodeRef {
+        const ref: NodeRef = `${this.fileRef}#${index === 0 ? 'body' : `body-${index + 1}`}`;
+        this.node(ref, 'section', basenameOf(this.path), { [SECTION_INDEX_BADGE]: index, excerpt: excerpt(text) });
+        this.edge(this.fileRef, ref, 'contains');
+        return ref;
+    }
+
+    build(): Extracted {
+        const byRef = <T extends { ref: NodeRef }>(a: T, b: T) => compareStr(a.ref, b.ref);
+        return {
+            sourceRef: this.fileRef,
+            sourceHash: this.hash,
+            nodes: [...this.nodes.values()].sort(byRef),
+            edges: [...this.edges.values()].sort((a, b) =>
+                compareStr(a.from, b.from) || compareStr(a.to, b.to) || compareStr(a.kind, b.kind)),
+            keys: [...this.keys.values()].sort((a, b) => byRef(a, b) || compareStr(a.key, b.key)),
+        };
+    }
+}
+
+/** `key: value` and `- item` front matter between leading `---` fences. */
+export function frontMatterOf(text: string): Record<string, string[]> {
+    const out: Record<string, string[]> = {};
+    const m = /^---\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|$)/.exec(text);
+    if (!m) return out;
+    let current: string | undefined;
+    for (const line of m[1]!.split(/\r?\n/)) {
+        const item = /^\s+-\s+(.*)$/.exec(line);
+        if (item && current) { out[current]!.push(unquote(item[1]!)); continue; }
+        const kv = /^([A-Za-z0-9_.-]+)\s*:\s*(.*)$/.exec(line);
+        if (!kv) continue;
+        current = kv[1]!;
+        const raw = kv[2]!.trim();
+        const list = /^\[(.*)\]$/.exec(raw);
+        out[current] = list ? list[1]!.split(',').map(unquote).filter(Boolean) : raw ? [unquote(raw)] : [];
+    }
+    return out;
+}
+
+function unquote(s: string): string {
+    return s.trim().replace(/^["']|["']$/g, '').trim();
+}
+
+function excerpt(text: string): string {
+    return text.replace(/\s+/g, ' ').trim().slice(0, EXCERPT_CHARS);
+}
+
+const MD_LINK = /(?<!!)\[[^\]]*\]\(\s*<?([^)\s>]+)>?(?:\s+"[^"]*")?\s*\)/g;
+
+export function extractMarkdown(source: SourceDocument, sections: ParsedSection[], ctx: Ctx): Extracted {
+    const b = new ExtractBuilder(source, ctx);
+    const fm = frontMatterOf(textOf(source));
+    const fileBadges: Record<string, string> = {};
+    for (const k of Object.keys(fm).sort(compareStr)) {
+        fileBadges[`fm.${k}`] = fm[k]!.join(', ');
+        for (const v of fm[k]!) b.key(b.fileRef, normalizeKey(v));
+    }
+    if (Object.keys(fileBadges).length) b.node(b.fileRef, 'file', fm['title']?.[0] || basenameOf(b.path), fileBadges);
+
+    const refByCrumb = new Map<string, NodeRef>();
+    const occurrences = new Map<string, number>();
+    const sectionNode = (crumbs: string[], index?: number, text?: string): NodeRef => {
+        const slugPath = crumbs.map(slugOf).join('/');
+        const n = (occurrences.get(slugPath) ?? 0) + 1;
+        occurrences.set(slugPath, n);
+        const ref: NodeRef = `${b.fileRef}#${n > 1 ? `${slugPath}-${n}` : slugPath}`;
+        const badges: Record<string, number | string> = { depth: crumbs.length, breadcrumb: crumbs.join(BREADCRUMB_SEPARATOR) };
+        if (index !== undefined) badges[SECTION_INDEX_BADGE] = index;
+        if (text) badges['excerpt'] = excerpt(text);
+        b.node(ref, 'section', crumbs[crumbs.length - 1]!, badges);
+        b.key(ref, slugOf(crumbs[crumbs.length - 1]!));
+        const parent = crumbs.length > 1 ? refByCrumb.get(crumbs.slice(0, -1).join('\u0000')) : b.fileRef;
+        b.edge(parent ?? b.fileRef, ref, 'contains');
+        refByCrumb.set(crumbs.join('\u0000'), ref);
+        return ref;
+    };
+
+    sections.forEach((section, index) => {
+        const headingPath = section.metadata?.['headingPath'];
+        let from = b.fileRef;
+        if (typeof headingPath === 'string' && headingPath) {
+            const crumbs = headingPath.split(BREADCRUMB_SEPARATOR);
+            // Headings with no body of their own are not emitted by the parser; recover them.
+            for (let k = 1; k < crumbs.length; k++) {
+                if (!refByCrumb.has(crumbs.slice(0, k).join('\u0000'))) sectionNode(crumbs.slice(0, k));
+            }
+            from = sectionNode(crumbs, index, section.text);
+        }
+        for (const m of section.text.matchAll(MD_LINK)) {
+            if (!m[1]!.startsWith('#')) b.link(from, m[1]!);
+        }
+        b.wikiLinks(from, section.text);
+        b.mentions(from, section.text);
+    });
+    return b.build();
+}
+
+export function extractHtml(source: SourceDocument, sections: ParsedSection[], ctx: Ctx): Extracted {
+    const title = sections[0]?.metadata?.['htmlTitle'];
+    const b = new ExtractBuilder(source, ctx, typeof title === 'string' ? title : undefined);
+    if (typeof title === 'string' && title) b.key(b.fileRef, normalizeKey(title));
+    for (const m of textOf(source).matchAll(/<a\s[^>]*href\s*=\s*["']([^"']+)["']/gi)) {
+        if (!m[1]!.startsWith('#')) b.link(b.fileRef, m[1]!);
+    }
+    sections.forEach((section, index) => b.mentions(b.bodySection(index, section.text), section.text));
+    return b.build();
+}
+
+export function extractPlainText(source: SourceDocument, sections: ParsedSection[], ctx: Ctx): Extracted {
+    const b = new ExtractBuilder(source, ctx);
+    sections.forEach((section, index) => {
+        const from = b.bodySection(index, section.text);
+        b.wikiLinks(from, section.text);
+        b.mentions(from, section.text);
+    });
+    return b.build();
+}

package/src/ingest/hash.ts

@@ -0,0 +1,22 @@
+/**
+ * ingest/hash.ts — the content hash every ingest consumer shares (the pipeline's chunk
+ * hashes, the knowledge graph's extracted output and card hashes). A leaf module, so
+ * `ingest/docsExtract.ts` can use it without an import cycle through the pipeline.
+ * (SHA-256 digests live in `retrieval/knowledge-index/sha256.ts`.)
+ */
+
+/**
+ * FNV-1a, 32-bit, rendered as hex.
+ *
+ * A content hash here answers "did this chunk change", not "was this chunk
+ * tampered with", so a fast non-cryptographic hash is the right tool — and it is
+ * synchronous, unlike WebCrypto, which keeps chunking a pure function.
+ */
+export function hashText(text: string): string {
+    let hash = 0x811c9dc5;
+    for (let i = 0; i < text.length; i++) {
+        hash ^= text.charCodeAt(i);
+        hash = Math.imul(hash, 0x01000193) >>> 0;
+    }
+    return hash.toString(16).padStart(8, '0');
+}

package/src/ingest/parsers.ts

@@ -17,6 +17,7 @@
  */
 
 import type { MetadataRecord, MetadataValue } from '../vectorstore/types.js';
+import { extractHtml, extractMarkdown, extractPlainText } from './docsExtract.js';
 import type { DocumentParser, ParsedSection, SourceDocument, StructuredRow } from './types.js';
 
 const textOf = (source: SourceDocument): string =>
@@ -237,6 +238,7 @@ export const markdownParser: DocumentParser = {
         flush();
         return sections;
     },
+    extract: extractMarkdown,
 };
 
 // ── HTML ──────────────────────────────────────────────────────────────────────
@@ -270,6 +272,7 @@ export const htmlParser: DocumentParser = {
         if (titleMatch?.[1]) metadata['htmlTitle'] = titleMatch[1].trim();
         return [{ text, metadata }];
     },
+    extract: extractHtml,
 };
 
 // ── plain text ────────────────────────────────────────────────────────────────
@@ -281,6 +284,7 @@ export const plainTextParser: DocumentParser = {
         const text = textOf(source).trim();
         return text ? [{ text }] : [];
     },
+    extract: extractPlainText,
 };
 
 // ── registry ──────────────────────────────────────────────────────────────────

package/src/ingest/types.ts

@@ -11,6 +11,7 @@
  */
 
 import type { MetadataRecord, MetadataValue } from '../vectorstore/types.js';
+import type { ExtractContext, Extracted } from '../knowledge/model.js';
 
 /** One row of a structured source (a DB result set, a CSV, a spreadsheet). */
 export type StructuredRow = Record<string, MetadataValue>;
@@ -57,6 +58,12 @@ export interface DocumentParser {
     /** True when this parser handles the source. First match in registry order wins. */
     accepts(source: SourceDocument): boolean;
     parse(source: SourceDocument): ParsedSection[];
+    /**
+     * Optional, pure: the knowledge-graph structure of a parsed source (file/section nodes,
+     * links, mentions, keys). Perception, never memory — recomputed from the source and
+     * cached by content hash. Parsers without it contribute a bare file node.
+     */
+    extract?(source: SourceDocument, sections: ParsedSection[], ctx: ExtractContext): Extracted;
 }
 
 /** Identity of one stored chunk, for incremental sync. */

package/src/knowledge/build.test.ts

@@ -0,0 +1,150 @@
+/**
+ * src/knowledge/build.test.ts
+ * buildGraph: byte-identical output for equal input, the fact join and staleness, the
+ * near-duplicate/duplicate rule, live overlay, and the truncation keep-rules.
+ */
+
+import { buildGraph, graphDigest } from './build.js';
+import { brokenLinkRule } from './packs/docs.js';
+import type { BuildGraphInput, KnowledgePack, StoredFact } from './model.js';
+import { edge, extractedOf, fileNode, ref, ROOT } from './testkit.test-util.js';
+
+// The broken-link rule makes the truncation fixture's dangling link a finding.
+const PACK: KnowledgePack = { id: 'docs', parsers: [], findings: [brokenLinkRule] };
+
+function fixture(): BuildGraphInput {
+    return {
+        rootId: ROOT,
+        packs: [PACK],
+        extracted: [
+            extractedOf(fileNode('a/one.md'), [edge('a/one.md', 'a/two.md'), edge('a/one.md', 'b/three.md')]),
+            extractedOf(fileNode('a/two.md'), [edge('a/two.md', 'b/three.md')]),
+            extractedOf(fileNode('b/three.md'), [edge('b/three.md', 'a/one.md', 'duplicate')]),
+        ],
+        facts: [
+            { key: 'note', content: 'A human note', source: 's', importance: 1, origin: 'human', fileAnchors: [{ path: 'a/one.md', contentHash: 'old' }] },
+            { key: 'claim:1', content: 'one ends 2027', source: 's', importance: 1, origin: 'model', fileAnchors: [{ path: 'a/one.md', contentHash: 'old' }] },
+            { key: 'claim:2', content: 'two is fine', source: 's', importance: 1, origin: 'model', fileAnchors: [{ path: 'a/two.md', contentHash: 'h:a/two.md' }] },
+            { key: 'floating', content: 'No anchor', source: 's', importance: 1, origin: 'agent', fileAnchors: [] },
+        ],
+        relations: [{ from: 'fact:claim:1', to: 'fact:claim:2', kind: 'contradicts', weight: 1, origin: 'model', confidence: 0.8 }],
+        nearDuplicates: [
+            { a: ref('a/one.md'), b: ref('a/two.md'), similarity: 0.91 },
+            { a: ref('a/one.md'), b: ref('b/three.md'), similarity: 0.95 },
+        ],
+    };
+}
+
+function reversed(input: BuildGraphInput): BuildGraphInput {
+    return {
+        ...input,
+        extracted: [...input.extracted].reverse().map((e) => ({ ...e, edges: [...e.edges].reverse() })),
+        facts: [...input.facts].reverse(),
+        relations: [...input.relations].reverse(),
+        nearDuplicates: [...(input.nearDuplicates ?? [])].reverse(),
+    };
+}
+
+describe('buildGraph — determinism', () => {
+    it('equal input (in any order) gives byte-identical JSON and digest', () => {
+        const a = JSON.stringify(buildGraph(fixture()));
+        const b = JSON.stringify(buildGraph(reversed(fixture())));
+        expect(a).toBe(b);
+        expect(graphDigest(fixture())).toBe(graphDigest(reversed(fixture())));
+        expect(graphDigest(fixture())).toMatch(/^[0-9a-f]{64}$/);
+    });
+
+    it('a different input gives a different digest', () => {
+        const changed = fixture();
+        changed.facts = changed.facts.slice(1);
+        expect(graphDigest(changed)).not.toBe(graphDigest(fixture()));
+    });
+});
+
+describe('buildGraph — the join', () => {
+    const g = buildGraph(fixture());
+    const node = (r: string) => g.nodes.find((n) => n.ref === r);
+
+    it('joins facts to sources by anchor path', () => {
+        expect(g.edges).toEqual(expect.arrayContaining([
+            expect.objectContaining({ from: 'fact:note', to: ref('a/one.md'), kind: 'about', origin: 'human' }),
+            expect.objectContaining({ from: 'fact:claim:2', to: ref('a/two.md'), kind: 'about', origin: 'model' }),
+        ]));
+    });
+
+    it('a fact with no anchor is a node with no about edge', () => {
+        expect(node('fact:floating')).toBeDefined();
+        expect(g.edges.some((e) => e.from === 'fact:floating')).toBe(false);
+    });
+
+    it('flags a model fact judged from another hash stale — never a human one', () => {
+        expect(node('fact:claim:1')?.stale).toBe(true);
+        expect(node('fact:claim:2')?.stale).toBeUndefined();
+        expect(node('fact:note')?.stale).toBeUndefined();
+        expect(g.stats.stale).toBe(1);
+        expect(g.findings.some((f) => f.kind === 'stale' && f.nodes.includes('fact:claim:1'))).toBe(true);
+    });
+
+    it('stored relations become edges', () => {
+        expect(g.edges).toContainEqual(expect.objectContaining({ from: 'fact:claim:1', to: 'fact:claim:2', kind: 'contradicts', confidence: 0.8 }));
+    });
+
+    it('near-duplicates become edges unless the pair is an exact duplicate', () => {
+        expect(g.edges).toContainEqual(expect.objectContaining({ from: ref('a/one.md'), to: ref('a/two.md'), kind: 'near-duplicate' }));
+        expect(g.edges.some((e) => e.kind === 'near-duplicate' && e.to === ref('b/three.md'))).toBe(false);
+        expect(g.findings.some((f) => f.kind === 'clone-group')).toBe(true);
+    });
+
+    it('extracted items keep origin extracted, importance is normalized to 0–1', () => {
+        expect(node(ref('a/one.md'))?.origin).toBe('extracted');
+        expect(Math.max(...g.nodes.map((n) => n.importance))).toBe(1);
+        expect(g.nodes.every((n) => n.importance >= 0 && n.importance <= 1)).toBe(true);
+    });
+
+    it('live changes badge nodes and run files become co-changed edges', () => {
+        const live = buildGraph({
+            ...fixture(),
+            live: { changes: [{ path: 'a/two.md', status: 'M', added: 3, removed: 1 }], runFiles: ['a/two.md', 'a/one.md'] },
+        });
+        expect(live.nodes.find((n) => n.ref === ref('a/two.md'))?.badges).toEqual({ added: 3, removed: 1, status: 'M' });
+        expect(live.edges).toContainEqual(expect.objectContaining({ from: ref('a/one.md'), to: ref('a/two.md'), kind: 'co-changed', origin: 'agent' }));
+    });
+});
+
+describe('buildGraph — truncation keep-rules', () => {
+    // A chain of 40 files in 4 folders; one broken link (a finding) far from the focus.
+    function chain(): BuildGraphInput {
+        const paths = Array.from({ length: 40 }, (_, i) => `d${i % 4}/f${String(i).padStart(2, '0')}.md`);
+        const extracted = paths.map((p, i) => extractedOf(fileNode(p), i + 1 < paths.length ? [edge(p, paths[i + 1]!)] : []));
+        extracted[30]!.edges.push(edge(paths[30]!, 'gone/missing.md'));
+        return { rootId: ROOT, packs: [PACK], extracted, facts: [] as StoredFact[], relations: [] };
+    }
+
+    it('keeps focus, finding and changed nodes plus their 1-hop neighbours, and folds the rest into areas', () => {
+        const input = { ...chain(), live: { changes: [{ path: 'd2/f10.md', status: 'M', added: 1, removed: 0 }] } };
+        const g = buildGraph(input, { maxNodes: 8, focus: [ref('d0/f20.md')] });
+        const refs = new Set(g.nodes.map((n) => n.ref));
+        expect(g.stats.truncated).toBe(true);
+        for (const r of ['d0/f20.md', 'd3/f19.md', 'd1/f21.md', 'd2/f10.md', 'd1/f09.md', 'd3/f11.md', 'd2/f30.md', 'gone/missing.md']) {
+            expect(refs.has(ref(r))).toBe(true);
+        }
+        const areas = g.nodes.filter((n) => n.kind === 'area');
+        expect(areas.length).toBeGreaterThan(0);
+        expect(areas.every((a) => typeof a.badges?.['members'] === 'number')).toBe(true);
+        expect(g.edges.every((e) => refs.has(e.from) && refs.has(e.to))).toBe(true);
+        expect(g.edges.every((e) => e.from !== e.to)).toBe(true);
+        expect(g.stats.nodes).toBe(g.nodes.length);
+    });
+
+    it('does not truncate under the budget', () => {
+        const g = buildGraph(chain());
+        expect(g.stats.truncated).toBe(false);
+        expect(g.nodes.some((n) => n.kind === 'area')).toBe(false);
+    });
+
+    it('truncation is deterministic too', () => {
+        const a = JSON.stringify(buildGraph(chain(), { maxNodes: 6, focus: [ref('d0/f00.md')] }));
+        const b = JSON.stringify(buildGraph(reversed(chain()), { maxNodes: 6, focus: [ref('d0/f00.md')] }));
+        expect(a).toBe(b);
+    });
+});