@seanhogg/builderforce-memory 2026.10.27 → 2026.10.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ingest/IngestionPipeline.d.ts +1 -8
- package/dist/ingest/IngestionPipeline.d.ts.map +1 -1
- package/dist/ingest/IngestionPipeline.js +3 -15
- package/dist/ingest/IngestionPipeline.js.map +1 -1
- package/dist/ingest/docsExtract.d.ts +30 -0
- package/dist/ingest/docsExtract.d.ts.map +1 -0
- package/dist/ingest/docsExtract.js +221 -0
- package/dist/ingest/docsExtract.js.map +1 -0
- package/dist/ingest/hash.d.ts +15 -0
- package/dist/ingest/hash.d.ts.map +1 -0
- package/dist/ingest/hash.js +22 -0
- package/dist/ingest/hash.js.map +1 -0
- package/dist/ingest/parsers.d.ts.map +1 -1
- package/dist/ingest/parsers.js +4 -0
- package/dist/ingest/parsers.js.map +1 -1
- package/dist/ingest/types.d.ts +7 -0
- package/dist/ingest/types.d.ts.map +1 -1
- package/dist/ingest/types.js.map +1 -1
- package/dist/knowledge/build.d.ts +24 -0
- package/dist/knowledge/build.d.ts.map +1 -0
- package/dist/knowledge/build.js +263 -0
- package/dist/knowledge/build.js.map +1 -0
- package/dist/knowledge/cards.d.ts +17 -0
- package/dist/knowledge/cards.d.ts.map +1 -0
- package/dist/knowledge/cards.js +118 -0
- package/dist/knowledge/cards.js.map +1 -0
- package/dist/knowledge/findings.d.ts +24 -0
- package/dist/knowledge/findings.d.ts.map +1 -0
- package/dist/knowledge/findings.js +152 -0
- package/dist/knowledge/findings.js.map +1 -0
- package/dist/knowledge/importance.d.ts +21 -0
- package/dist/knowledge/importance.d.ts.map +1 -0
- package/dist/knowledge/importance.js +70 -0
- package/dist/knowledge/importance.js.map +1 -0
- package/dist/knowledge/index.d.ts +31 -0
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/index.js +26 -0
- package/dist/knowledge/index.js.map +1 -0
- package/dist/knowledge/judge.d.ts +86 -0
- package/dist/knowledge/judge.d.ts.map +1 -0
- package/dist/knowledge/judge.js +213 -0
- package/dist/knowledge/judge.js.map +1 -0
- package/dist/knowledge/model.d.ts +333 -0
- package/dist/knowledge/model.d.ts.map +1 -0
- package/dist/knowledge/model.js +13 -0
- package/dist/knowledge/model.js.map +1 -0
- package/dist/knowledge/packs/docs.d.ts +17 -0
- package/dist/knowledge/packs/docs.d.ts.map +1 -0
- package/dist/knowledge/packs/docs.js +89 -0
- package/dist/knowledge/packs/docs.js.map +1 -0
- package/dist/knowledge/partitions.d.ts +31 -0
- package/dist/knowledge/partitions.d.ts.map +1 -0
- package/dist/knowledge/partitions.js +135 -0
- package/dist/knowledge/partitions.js.map +1 -0
- package/dist/knowledge/plan.d.ts +33 -0
- package/dist/knowledge/plan.d.ts.map +1 -0
- package/dist/knowledge/plan.js +89 -0
- package/dist/knowledge/plan.js.map +1 -0
- package/dist/knowledge/refs.d.ts +40 -0
- package/dist/knowledge/refs.d.ts.map +1 -0
- package/dist/knowledge/refs.js +102 -0
- package/dist/knowledge/refs.js.map +1 -0
- package/dist/knowledge/resolve.d.ts +35 -0
- package/dist/knowledge/resolve.d.ts.map +1 -0
- package/dist/knowledge/resolve.js +67 -0
- package/dist/knowledge/resolve.js.map +1 -0
- package/dist/knowledge/review.d.ts +23 -0
- package/dist/knowledge/review.d.ts.map +1 -0
- package/dist/knowledge/review.js +233 -0
- package/dist/knowledge/review.js.map +1 -0
- package/dist/knowledge/sources.d.ts +47 -0
- package/dist/knowledge/sources.d.ts.map +1 -0
- package/dist/knowledge/sources.js +136 -0
- package/dist/knowledge/sources.js.map +1 -0
- package/dist/knowledge/stores/localSnapshotStore.d.ts +50 -0
- package/dist/knowledge/stores/localSnapshotStore.d.ts.map +1 -0
- package/dist/knowledge/stores/localSnapshotStore.js +125 -0
- package/dist/knowledge/stores/localSnapshotStore.js.map +1 -0
- package/dist/knowledge/trace.d.ts +32 -0
- package/dist/knowledge/trace.d.ts.map +1 -0
- package/dist/knowledge/trace.js +102 -0
- package/dist/knowledge/trace.js.map +1 -0
- package/dist/knowledge/truncate.d.ts +24 -0
- package/dist/knowledge/truncate.d.ts.map +1 -0
- package/dist/knowledge/truncate.js +128 -0
- package/dist/knowledge/truncate.js.map +1 -0
- package/dist/memory/SemanticRecall.d.ts.map +1 -1
- package/dist/memory/SemanticRecall.js +3 -1
- package/dist/memory/SemanticRecall.js.map +1 -1
- package/dist/retrieval/HybridRetriever.d.ts.map +1 -1
- package/dist/retrieval/HybridRetriever.js +3 -1
- package/dist/retrieval/HybridRetriever.js.map +1 -1
- package/dist/retrieval/bm25.d.ts +3 -2
- package/dist/retrieval/bm25.d.ts.map +1 -1
- package/dist/retrieval/bm25.js +5 -3
- package/dist/retrieval/bm25.js.map +1 -1
- package/dist/retrieval/fusion.d.ts +2 -1
- package/dist/retrieval/fusion.d.ts.map +1 -1
- package/dist/retrieval/fusion.js +4 -2
- package/dist/retrieval/fusion.js.map +1 -1
- package/dist/retrieval/hnsw.d.ts +49 -11
- package/dist/retrieval/hnsw.d.ts.map +1 -1
- package/dist/retrieval/hnsw.js +57 -29
- package/dist/retrieval/hnsw.js.map +1 -1
- package/dist/retrieval/index.d.ts +7 -3
- package/dist/retrieval/index.d.ts.map +1 -1
- package/dist/retrieval/index.js +5 -2
- package/dist/retrieval/index.js.map +1 -1
- package/dist/retrieval/knowledge-index/bytes.d.ts +43 -0
- package/dist/retrieval/knowledge-index/bytes.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/bytes.js +151 -0
- package/dist/retrieval/knowledge-index/bytes.js.map +1 -0
- package/dist/retrieval/knowledge-index/cascade.d.ts +37 -0
- package/dist/retrieval/knowledge-index/cascade.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/cascade.js +111 -0
- package/dist/retrieval/knowledge-index/cascade.js.map +1 -0
- package/dist/retrieval/knowledge-index/compact.d.ts +18 -0
- package/dist/retrieval/knowledge-index/compact.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/compact.js +64 -0
- package/dist/retrieval/knowledge-index/compact.js.map +1 -0
- package/dist/retrieval/knowledge-index/csr.d.ts +41 -0
- package/dist/retrieval/knowledge-index/csr.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/csr.js +76 -0
- package/dist/retrieval/knowledge-index/csr.js.map +1 -0
- package/dist/retrieval/knowledge-index/digest.d.ts +21 -0
- package/dist/retrieval/knowledge-index/digest.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/digest.js +50 -0
- package/dist/retrieval/knowledge-index/digest.js.map +1 -0
- package/dist/retrieval/knowledge-index/embeddings.d.ts +32 -0
- package/dist/retrieval/knowledge-index/embeddings.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/embeddings.js +78 -0
- package/dist/retrieval/knowledge-index/embeddings.js.map +1 -0
- package/dist/retrieval/knowledge-index/expand.d.ts +19 -0
- package/dist/retrieval/knowledge-index/expand.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/expand.js +49 -0
- package/dist/retrieval/knowledge-index/expand.js.map +1 -0
- package/dist/retrieval/knowledge-index/index.d.ts +40 -0
- package/dist/retrieval/knowledge-index/index.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/index.js +134 -0
- package/dist/retrieval/knowledge-index/index.js.map +1 -0
- package/dist/retrieval/knowledge-index/lexical.d.ts +26 -0
- package/dist/retrieval/knowledge-index/lexical.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/lexical.js +76 -0
- package/dist/retrieval/knowledge-index/lexical.js.map +1 -0
- package/dist/retrieval/knowledge-index/pairs.d.ts +22 -0
- package/dist/retrieval/knowledge-index/pairs.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/pairs.js +51 -0
- package/dist/retrieval/knowledge-index/pairs.js.map +1 -0
- package/dist/retrieval/knowledge-index/prune.d.ts +21 -0
- package/dist/retrieval/knowledge-index/prune.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/prune.js +43 -0
- package/dist/retrieval/knowledge-index/prune.js.map +1 -0
- package/dist/retrieval/knowledge-index/quantize.d.ts +37 -0
- package/dist/retrieval/knowledge-index/quantize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/quantize.js +77 -0
- package/dist/retrieval/knowledge-index/quantize.js.map +1 -0
- package/dist/retrieval/knowledge-index/serialize.d.ts +18 -0
- package/dist/retrieval/knowledge-index/serialize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/serialize.js +123 -0
- package/dist/retrieval/knowledge-index/serialize.js.map +1 -0
- package/dist/retrieval/knowledge-index/sha256.d.ts +30 -0
- package/dist/retrieval/knowledge-index/sha256.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/sha256.js +130 -0
- package/dist/retrieval/knowledge-index/sha256.js.map +1 -0
- package/dist/retrieval/knowledge-index/state.d.ts +55 -0
- package/dist/retrieval/knowledge-index/state.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/state.js +67 -0
- package/dist/retrieval/knowledge-index/state.js.map +1 -0
- package/dist/retrieval/knowledge-index/tokenize.d.ts +20 -0
- package/dist/retrieval/knowledge-index/tokenize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/tokenize.js +42 -0
- package/dist/retrieval/knowledge-index/tokenize.js.map +1 -0
- package/dist/retrieval/knowledge-index/traverse.d.ts +26 -0
- package/dist/retrieval/knowledge-index/traverse.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/traverse.js +90 -0
- package/dist/retrieval/knowledge-index/traverse.js.map +1 -0
- package/dist/retrieval/knowledge-index/types.d.ts +85 -0
- package/dist/retrieval/knowledge-index/types.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/types.js +9 -0
- package/dist/retrieval/knowledge-index/types.js.map +1 -0
- package/dist/retrieval/mmr.d.ts +11 -4
- package/dist/retrieval/mmr.d.ts.map +1 -1
- package/dist/retrieval/mmr.js +28 -8
- package/dist/retrieval/mmr.js.map +1 -1
- package/dist/retrieval/order.d.ts +41 -0
- package/dist/retrieval/order.d.ts.map +1 -0
- package/dist/retrieval/order.js +54 -0
- package/dist/retrieval/order.js.map +1 -0
- package/dist/retrieval/weightedFusion.d.ts +2 -1
- package/dist/retrieval/weightedFusion.d.ts.map +1 -1
- package/dist/retrieval/weightedFusion.js +7 -5
- package/dist/retrieval/weightedFusion.js.map +1 -1
- package/dist/vectorstore/MemoryVectorStore.d.ts.map +1 -1
- package/dist/vectorstore/MemoryVectorStore.js +2 -1
- package/dist/vectorstore/MemoryVectorStore.js.map +1 -1
- package/package.json +16 -5
- package/src/ingest/IngestionPipeline.ts +3 -15
- package/src/ingest/docsExtract.ts +219 -0
- package/src/ingest/hash.ts +22 -0
- package/src/ingest/parsers.ts +4 -0
- package/src/ingest/types.ts +7 -0
- package/src/knowledge/build.test.ts +150 -0
- package/src/knowledge/build.ts +266 -0
- package/src/knowledge/cards.ts +130 -0
- package/src/knowledge/docs.test.ts +118 -0
- package/src/knowledge/findings.ts +159 -0
- package/src/knowledge/importance.ts +72 -0
- package/src/knowledge/index.ts +53 -0
- package/src/knowledge/judge.ts +231 -0
- package/src/knowledge/model.ts +312 -0
- package/src/knowledge/packs/docs.ts +91 -0
- package/src/knowledge/partitions.test.ts +98 -0
- package/src/knowledge/partitions.ts +119 -0
- package/src/knowledge/plan.ts +115 -0
- package/src/knowledge/refs.ts +108 -0
- package/src/knowledge/resolve.ts +78 -0
- package/src/knowledge/review.test.ts +180 -0
- package/src/knowledge/review.ts +235 -0
- package/src/knowledge/sources.ts +146 -0
- package/src/knowledge/stores/localSnapshotStore.test.ts +82 -0
- package/src/knowledge/stores/localSnapshotStore.ts +159 -0
- package/src/knowledge/testkit.test-util.ts +92 -0
- package/src/knowledge/trace.test.ts +91 -0
- package/src/knowledge/trace.ts +111 -0
- package/src/knowledge/truncate.ts +118 -0
- package/src/memory/SemanticRecall.ts +3 -1
- package/src/retrieval/HybridRetriever.ts +3 -1
- package/src/retrieval/bm25.ts +5 -3
- package/src/retrieval/fusion.ts +5 -2
- package/src/retrieval/hnsw.ts +82 -35
- package/src/retrieval/index.ts +17 -3
- package/src/retrieval/knowledge-index/bytes.ts +166 -0
- package/src/retrieval/knowledge-index/cascade.ts +125 -0
- package/src/retrieval/knowledge-index/compact.ts +68 -0
- package/src/retrieval/knowledge-index/csr.ts +102 -0
- package/src/retrieval/knowledge-index/digest.ts +54 -0
- package/src/retrieval/knowledge-index/embeddings.ts +85 -0
- package/src/retrieval/knowledge-index/expand.ts +55 -0
- package/src/retrieval/knowledge-index/index.ts +162 -0
- package/src/retrieval/knowledge-index/lexical.ts +78 -0
- package/src/retrieval/knowledge-index/pairs.ts +57 -0
- package/src/retrieval/knowledge-index/prune.ts +48 -0
- package/src/retrieval/knowledge-index/quantize.ts +78 -0
- package/src/retrieval/knowledge-index/serialize.ts +124 -0
- package/src/retrieval/knowledge-index/sha256.ts +122 -0
- package/src/retrieval/knowledge-index/state.ts +108 -0
- package/src/retrieval/knowledge-index/tokenize.ts +41 -0
- package/src/retrieval/knowledge-index/traverse.ts +95 -0
- package/src/retrieval/knowledge-index/types.ts +78 -0
- package/src/retrieval/mmr.ts +35 -10
- package/src/retrieval/order.ts +68 -0
- package/src/retrieval/weightedFusion.ts +8 -5
- package/src/vectorstore/MemoryVectorStore.ts +2 -1
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* knowledge/build.ts — `buildGraph`: join perception (extracted, computed) with memory
|
|
3
|
+
* (stored facts and relations) into one bounded {@link KnowledgeGraph} (PRD 38 §4.1).
|
|
4
|
+
*
|
|
5
|
+
* 1. extracted nodes/edges (analyzer output merges its badges into the same refs);
|
|
6
|
+
* 2. facts become nodes, joined to sources by `fileAnchors.path`; a model fact whose
|
|
7
|
+
* anchor hash differs from the source's current hash is `stale`;
|
|
8
|
+
* 3. stored relations become edges; near-duplicate pairs become `near-duplicate` edges
|
|
9
|
+
* (an exact `duplicate` between the same pair wins); live changes badge nodes and
|
|
10
|
+
* `runFiles` become `co-changed` edges;
|
|
11
|
+
* 4. partitions, areas, verdicts; PageRank importance; findings; then truncation.
|
|
12
|
+
*
|
|
13
|
+
* Deterministic: equal input ⇒ byte-identical JSON. Every list is sorted, every sort ends
|
|
14
|
+
* on ref, objects are built with a fixed key order.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { areaOfPath } from '../working/codemapAreas.js';
|
|
18
|
+
import { sha256Hex } from '../retrieval/knowledge-index/sha256.js';
|
|
19
|
+
import type {
|
|
20
|
+
BuildGraphInput, BuildGraphOptions, Extracted, KEdge, KNode, KnowledgeGraph, NodeRef,
|
|
21
|
+
PartitionRegistry, StoredFact,
|
|
22
|
+
} from './model.js';
|
|
23
|
+
import { computeFindings } from './findings.js';
|
|
24
|
+
import { pageRank } from './importance.js';
|
|
25
|
+
import { loadPartitions, partitionOf, verdictOf } from './partitions.js';
|
|
26
|
+
import { entityName } from './resolve.js';
|
|
27
|
+
import {
|
|
28
|
+
basenameOf, canonicalJson, compareStr, factRef, pathOfRef, round6, sortBy, srcRef,
|
|
29
|
+
} from './refs.js';
|
|
30
|
+
import { truncateGraph } from './truncate.js';
|
|
31
|
+
|
|
32
|
+
export const DEFAULT_MAX_NODES = 400;
|
|
33
|
+
export const MAX_NODES_CAP = 2000;
|
|
34
|
+
const LABEL_CHARS = 120;
|
|
35
|
+
|
|
36
|
+
const edgeId = (from: string, to: string, kind: string) => `${from}\u0000${to}\u0000${kind}`;
|
|
37
|
+
export const compareEdges = (a: KEdge, b: KEdge): number =>
|
|
38
|
+
compareStr(a.from, b.from) || compareStr(a.to, b.to) || compareStr(a.kind, b.kind);
|
|
39
|
+
|
|
40
|
+
/** sha-256 over the canonical input: equal inputs (in any order) ⇒ equal digests. */
|
|
41
|
+
export function graphDigest(input: BuildGraphInput): string {
|
|
42
|
+
return sha256Hex(canonicalJson({
|
|
43
|
+
rootId: input.rootId,
|
|
44
|
+
packs: input.packs.map((p) => p.id).sort(compareStr),
|
|
45
|
+
extracted: sortBy(input.extracted, (e) => e.sourceRef, (e) => e.sourceHash).map(canonicalExtracted),
|
|
46
|
+
facts: sortBy(input.facts, (f) => f.key).map((f) => ({ ...f, fileAnchors: sortBy(f.fileAnchors, (a) => a.path) })),
|
|
47
|
+
relations: sortBy(input.relations, (r) => r.from, (r) => r.to, (r) => r.kind),
|
|
48
|
+
live: input.live && {
|
|
49
|
+
changes: sortBy(input.live.changes, (c) => c.path),
|
|
50
|
+
runFiles: nonEmpty(input.live.runFiles) && [...input.live.runFiles!].sort(compareStr),
|
|
51
|
+
},
|
|
52
|
+
nearDuplicates: nonEmpty(input.nearDuplicates) && sortBy(input.nearDuplicates!, (p) => p.a, (p) => p.b),
|
|
53
|
+
}));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** An absent list and an empty list mean the same input, so they hash the same. */
|
|
57
|
+
const nonEmpty = (xs: readonly unknown[] | undefined): boolean | undefined => (xs && xs.length > 0) || undefined;
|
|
58
|
+
|
|
59
|
+
function canonicalExtracted(e: Extracted) {
|
|
60
|
+
return {
|
|
61
|
+
sourceRef: e.sourceRef, sourceHash: e.sourceHash,
|
|
62
|
+
nodes: sortBy(e.nodes, (n) => n.ref),
|
|
63
|
+
edges: [...e.edges].sort(compareEdges),
|
|
64
|
+
keys: sortBy(e.keys, (k) => k.ref, (k) => k.key),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function mergeBadges(a?: Record<string, number | string>, b?: Record<string, number | string>) {
|
|
69
|
+
if (!a) return b && { ...b };
|
|
70
|
+
if (!b) return { ...a };
|
|
71
|
+
const out = { ...a };
|
|
72
|
+
// Commutative: on a conflict the larger value (as a string) wins, whatever the order.
|
|
73
|
+
for (const [k, v] of Object.entries(b)) out[k] = k in out && String(out[k]) >= String(v) ? out[k]! : v;
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function oneLine(text: string): string {
|
|
78
|
+
const flat = text.replace(/\s+/g, ' ').trim();
|
|
79
|
+
return flat.length > LABEL_CHARS ? `${flat.slice(0, LABEL_CHARS - 1)}…` : flat;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
class GraphBuilder {
|
|
83
|
+
readonly nodes = new Map<NodeRef, KNode>();
|
|
84
|
+
readonly edges = new Map<string, KEdge>();
|
|
85
|
+
|
|
86
|
+
node(n: KNode): void {
|
|
87
|
+
const prior = this.nodes.get(n.ref);
|
|
88
|
+
if (!prior) { this.nodes.set(n.ref, { ...n, ...(n.badges ? { badges: { ...n.badges } } : {}) }); return; }
|
|
89
|
+
const badges = mergeBadges(prior.badges, n.badges);
|
|
90
|
+
if (badges) prior.badges = badges;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
edge(e: KEdge): void {
|
|
94
|
+
const id = edgeId(e.from, e.to, e.kind);
|
|
95
|
+
const prior = this.edges.get(id);
|
|
96
|
+
if (!prior) { this.edges.set(id, { ...e }); return; }
|
|
97
|
+
prior.weight = Math.max(prior.weight, e.weight);
|
|
98
|
+
if (e.cycle) prior.cycle = true;
|
|
99
|
+
if (e.confidence !== undefined) prior.confidence = Math.max(prior.confidence ?? 0, e.confidence);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
has(from: NodeRef, to: NodeRef, kind: string): boolean {
|
|
103
|
+
return this.edges.has(edgeId(from, to, kind));
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function factNode(f: StoredFact, rootId: string, g: GraphBuilder): KNode {
|
|
108
|
+
const ref = factRef(f.key);
|
|
109
|
+
const kind = f.key.startsWith('ent:') ? 'entity' : f.key.startsWith('claim:') ? 'claim' : 'fact';
|
|
110
|
+
const anchors = sortBy(f.fileAnchors, (a) => a.path);
|
|
111
|
+
const anchoredPack = anchors.map((a) => g.nodes.get(srcRef(rootId, a.path))?.pack).find(Boolean);
|
|
112
|
+
const pack = kind === 'entity' ? f.key.split(':')[1] || 'memory' : anchoredPack ?? 'memory';
|
|
113
|
+
const stale = f.origin === 'model' && anchors.some((a) => {
|
|
114
|
+
const current = g.nodes.get(srcRef(rootId, a.path))?.sourceHash;
|
|
115
|
+
return !!a.contentHash && !!current && current !== a.contentHash;
|
|
116
|
+
});
|
|
117
|
+
const first = anchors[0];
|
|
118
|
+
return {
|
|
119
|
+
ref, kind, label: oneLine(kind === 'entity' ? f.content || entityName(ref) : f.content), pack,
|
|
120
|
+
importance: 0, origin: f.origin,
|
|
121
|
+
...(first?.contentHash ? { sourceHash: first.contentHash } : {}),
|
|
122
|
+
...(stale ? { stale: true } : {}),
|
|
123
|
+
...(first?.line !== undefined ? { badges: { line: first.line } } : {}),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function buildGraph(input: BuildGraphInput, opts: BuildGraphOptions = {}): KnowledgeGraph {
|
|
128
|
+
const { rootId } = input;
|
|
129
|
+
const g = new GraphBuilder();
|
|
130
|
+
const sources = new Set<string>();
|
|
131
|
+
|
|
132
|
+
for (const ex of sortBy(input.extracted, (e) => e.sourceRef, (e) => e.sourceHash)) {
|
|
133
|
+
if (ex.sourceRef.startsWith('src:')) sources.add(ex.sourceRef);
|
|
134
|
+
for (const n of sortBy(ex.nodes, (x) => x.ref)) g.node({ ...n, origin: 'extracted' });
|
|
135
|
+
for (const e of [...ex.edges].sort(compareEdges)) g.edge({ ...e, origin: 'extracted' });
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const facts = sortBy(input.facts, (f) => f.key);
|
|
139
|
+
for (const f of facts) g.node(factNode(f, rootId, g));
|
|
140
|
+
for (const f of facts) {
|
|
141
|
+
for (const a of sortBy(f.fileAnchors, (x) => x.path)) {
|
|
142
|
+
const to = srcRef(rootId, a.path);
|
|
143
|
+
if (g.nodes.has(to)) g.edge({ from: factRef(f.key), to, kind: 'about', weight: 1, origin: f.origin });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
for (const r of sortBy(input.relations, (x) => x.from, (x) => x.to, (x) => x.kind)) {
|
|
148
|
+
g.edge({
|
|
149
|
+
from: r.from, to: r.to, kind: r.kind, weight: r.weight, origin: r.origin,
|
|
150
|
+
...(r.confidence !== undefined ? { confidence: r.confidence } : {}),
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
for (const p of sortBy(input.nearDuplicates ?? [], (x) => x.a, (x) => x.b)) {
|
|
155
|
+
if (p.a === p.b || p.a.startsWith('ent:') || p.b.startsWith('ent:')) continue;
|
|
156
|
+
const [a, b] = (compareStr(p.a, p.b) < 0 ? [p.a, p.b] : [p.b, p.a]) as [NodeRef, NodeRef];
|
|
157
|
+
if (!g.nodes.has(a) || !g.nodes.has(b) || g.has(a, b, 'duplicate') || g.has(b, a, 'duplicate')) continue;
|
|
158
|
+
g.edge({ from: a, to: b, kind: 'near-duplicate', weight: round6(p.similarity), origin: 'extracted' });
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const changed = applyLive(input, g);
|
|
162
|
+
closeEndpoints(g);
|
|
163
|
+
const registries = new Map(input.packs.map((p) => [p.id, loadPartitions(p)] as const));
|
|
164
|
+
placeNodes(rootId, g, registries);
|
|
165
|
+
|
|
166
|
+
const rank = pageRank([...g.nodes.keys()], [...g.edges.values()]);
|
|
167
|
+
for (const n of g.nodes.values()) n.importance = rank.get(n.ref) ?? 0;
|
|
168
|
+
|
|
169
|
+
const allNodes = [...g.nodes.values()].map(finalNode).sort((a, b) => compareStr(a.ref, b.ref));
|
|
170
|
+
const allEdges = [...g.edges.values()].map(finalEdge).sort(compareEdges);
|
|
171
|
+
const packs = input.packs.map((p) => p.id).sort(compareStr);
|
|
172
|
+
const full: KnowledgeGraph = {
|
|
173
|
+
scope: { rootId, packs, digest: graphDigest(input) },
|
|
174
|
+
nodes: allNodes, edges: allEdges, findings: [],
|
|
175
|
+
stats: { sources: sources.size, nodes: allNodes.length, edges: allEdges.length, stale: 0, truncated: false },
|
|
176
|
+
};
|
|
177
|
+
full.findings = computeFindings(full, input.packs, registries, input.nearDuplicates ?? []);
|
|
178
|
+
full.stats.stale = allNodes.filter((n) => n.stale).length;
|
|
179
|
+
|
|
180
|
+
const maxNodes = Math.min(MAX_NODES_CAP, Math.max(1, Math.floor(opts.maxNodes ?? DEFAULT_MAX_NODES)));
|
|
181
|
+
const cut = truncateGraph(full, { maxNodes, focus: opts.focus ?? [], changed });
|
|
182
|
+
return {
|
|
183
|
+
...full,
|
|
184
|
+
nodes: cut.nodes,
|
|
185
|
+
edges: cut.edges,
|
|
186
|
+
stats: { ...full.stats, nodes: cut.nodes.length, edges: cut.edges.length, truncated: cut.truncated },
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function applyLive(input: BuildGraphInput, g: GraphBuilder): NodeRef[] {
|
|
191
|
+
const changed: NodeRef[] = [];
|
|
192
|
+
if (!input.live) return changed;
|
|
193
|
+
for (const c of sortBy(input.live.changes, (x) => x.path)) {
|
|
194
|
+
const ref = srcRef(input.rootId, c.path);
|
|
195
|
+
if (!g.nodes.has(ref)) continue;
|
|
196
|
+
g.node({ ...g.nodes.get(ref)!, badges: { added: c.added, removed: c.removed, status: c.status } });
|
|
197
|
+
changed.push(ref);
|
|
198
|
+
}
|
|
199
|
+
const run = [...new Set(input.live.runFiles ?? [])].sort(compareStr)
|
|
200
|
+
.map((p) => srcRef(input.rootId, p)).filter((r) => g.nodes.has(r));
|
|
201
|
+
for (let i = 0; i < run.length; i++) {
|
|
202
|
+
for (let j = i + 1; j < run.length; j++) {
|
|
203
|
+
g.edge({ from: run[i]!, to: run[j]!, kind: 'co-changed', weight: 1, origin: 'agent' });
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return changed;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** A missing `src:` endpoint becomes a `missing` placeholder (a broken link); a missing fact drops the edge. */
|
|
210
|
+
function closeEndpoints(g: GraphBuilder): void {
|
|
211
|
+
for (const [id, e] of [...g.edges.entries()].sort((a, b) => compareStr(a[0], b[0]))) {
|
|
212
|
+
const ends = [e.from, e.to].filter((end) => !g.nodes.has(end));
|
|
213
|
+
if (ends.some((end) => pathOfRef(end) === undefined)) { g.edges.delete(id); continue; }
|
|
214
|
+
for (const end of ends) {
|
|
215
|
+
const path = pathOfRef(end)!;
|
|
216
|
+
const other = g.nodes.get(end === e.from ? e.to : e.from);
|
|
217
|
+
g.node({
|
|
218
|
+
ref: end, kind: 'missing', label: basenameOf(path) || path, pack: other?.pack ?? 'memory',
|
|
219
|
+
importance: 0, origin: 'extracted', badges: { missing: 1 },
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function placeNodes(rootId: string, g: GraphBuilder, registries: Map<string, PartitionRegistry>): void {
|
|
226
|
+
for (const n of g.nodes.values()) {
|
|
227
|
+
const path = pathOfRef(n.ref);
|
|
228
|
+
if (path === undefined) continue;
|
|
229
|
+
const reg = registries.get(n.pack);
|
|
230
|
+
const partition = reg ? partitionOf(path, reg) : undefined;
|
|
231
|
+
if (partition !== undefined) n.partition = partition;
|
|
232
|
+
const area = areaOfPath(path);
|
|
233
|
+
if (area) n.area = srcRef(rootId, area);
|
|
234
|
+
}
|
|
235
|
+
for (const e of g.edges.values()) {
|
|
236
|
+
const from = g.nodes.get(e.from), to = g.nodes.get(e.to);
|
|
237
|
+
const reg = from && registries.get(from.pack);
|
|
238
|
+
if (!from || !to || !reg || from.pack !== to.pack) continue;
|
|
239
|
+
const verdict = verdictOf(from.partition, to.partition, reg);
|
|
240
|
+
if (verdict) e.verdict = verdict;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function finalNode(n: KNode): KNode {
|
|
245
|
+
const badges = n.badges && Object.keys(n.badges).length
|
|
246
|
+
? Object.fromEntries(Object.keys(n.badges).sort(compareStr).map((k) => [k, n.badges![k]!]))
|
|
247
|
+
: undefined;
|
|
248
|
+
return {
|
|
249
|
+
ref: n.ref, kind: n.kind, label: n.label, pack: n.pack,
|
|
250
|
+
...(n.partition !== undefined ? { partition: n.partition } : {}),
|
|
251
|
+
...(n.area !== undefined ? { area: n.area } : {}),
|
|
252
|
+
importance: n.importance, origin: n.origin,
|
|
253
|
+
...(n.sourceHash !== undefined ? { sourceHash: n.sourceHash } : {}),
|
|
254
|
+
...(n.stale ? { stale: true } : {}),
|
|
255
|
+
...(badges ? { badges } : {}),
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export function finalEdge(e: KEdge): KEdge {
|
|
260
|
+
return {
|
|
261
|
+
from: e.from, to: e.to, kind: e.kind, weight: round6(e.weight), origin: e.origin,
|
|
262
|
+
...(e.confidence !== undefined ? { confidence: round6(e.confidence) } : {}),
|
|
263
|
+
...(e.verdict ? { verdict: e.verdict } : {}),
|
|
264
|
+
...(e.cycle ? { cycle: true } : {}),
|
|
265
|
+
};
|
|
266
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* knowledge/cards.ts — `toCards`: one knowledge-index card per graph node
|
|
3
|
+
* (design: Builderforce.ai/docs/design/deterministic-knowledge-index.md, "Cards").
|
|
4
|
+
*
|
|
5
|
+
* | kind | text | keys |
|
|
6
|
+
* |-----------------|-------------------------------------------------------------------|-------------------------------|
|
|
7
|
+
* | file | path · area · partition · symbols · doc line · top-5 neighbours | path, basename, symbols, area |
|
|
8
|
+
* | section | file · breadcrumb · first 400 chars · outgoing link targets | file path, slug, front matter |
|
|
9
|
+
* | entity (ent:) | name · type · top-5 mentioning sources | normalized name, aliases |
|
|
10
|
+
* | claim | subject predicate object · quote · source section | subject/object entity keys |
|
|
11
|
+
* | area | path · partition · top-10 members | path |
|
|
12
|
+
*
|
|
13
|
+
* `hash` is the index's own text hash (`hashDocText`), so an unchanged card is never re-embedded.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { hashDocText } from '../retrieval/knowledge-index/digest.js';
|
|
17
|
+
import type { IndexDoc, KEdge, KNode, KnowledgeGraph, NodeRef } from './model.js';
|
|
18
|
+
import { byImportance } from './importance.js';
|
|
19
|
+
import { entityName } from './resolve.js';
|
|
20
|
+
import { basenameOf, compareStr, fileRefOf, normalizeKey, pathOfRef, slugOf } from './refs.js';
|
|
21
|
+
|
|
22
|
+
const SEP = ' · ';
|
|
23
|
+
const EXCERPT_CHARS = 400;
|
|
24
|
+
|
|
25
|
+
export function toCards(g: KnowledgeGraph): IndexDoc[] {
|
|
26
|
+
const byRef = new Map(g.nodes.map((n) => [n.ref, n] as const));
|
|
27
|
+
const outEdges = new Map<NodeRef, KEdge[]>();
|
|
28
|
+
const inEdges = new Map<NodeRef, KEdge[]>();
|
|
29
|
+
const push = (m: Map<NodeRef, KEdge[]>, k: NodeRef, e: KEdge) => {
|
|
30
|
+
const l = m.get(k) ?? [];
|
|
31
|
+
l.push(e);
|
|
32
|
+
m.set(k, l);
|
|
33
|
+
};
|
|
34
|
+
for (const e of g.edges) { push(outEdges, e.from, e); push(inEdges, e.to, e); }
|
|
35
|
+
const label = (r: NodeRef) => byRef.get(r)?.label ?? pathOfRef(r) ?? r;
|
|
36
|
+
const ctx: CardCtx = { byRef, outEdges, inEdges, label, members: membersByArea(g.nodes) };
|
|
37
|
+
|
|
38
|
+
return g.nodes.map((n) => {
|
|
39
|
+
const { text, keys } = cardOf(n, ctx);
|
|
40
|
+
const uniqueKeys = [...new Set(keys.filter(Boolean))].sort(compareStr);
|
|
41
|
+
return { id: n.ref, text, ...(uniqueKeys.length ? { keys: uniqueKeys } : {}), hash: hashDocText(text) };
|
|
42
|
+
}).sort((a, b) => compareStr(a.id, b.id));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
interface CardCtx {
|
|
46
|
+
byRef: Map<NodeRef, KNode>;
|
|
47
|
+
outEdges: Map<NodeRef, KEdge[]>;
|
|
48
|
+
inEdges: Map<NodeRef, KEdge[]>;
|
|
49
|
+
label(r: NodeRef): string;
|
|
50
|
+
members: Map<NodeRef, KNode[]>;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function membersByArea(nodes: readonly KNode[]): Map<NodeRef, KNode[]> {
|
|
54
|
+
const out = new Map<NodeRef, KNode[]>();
|
|
55
|
+
for (const n of nodes) {
|
|
56
|
+
if (!n.area || n.kind !== 'file') continue;
|
|
57
|
+
const l = out.get(n.area) ?? [];
|
|
58
|
+
l.push(n);
|
|
59
|
+
out.set(n.area, l);
|
|
60
|
+
}
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Neighbour refs (either direction), heaviest first, then by ref. */
|
|
65
|
+
function topNeighbours(ref: NodeRef, ctx: CardCtx, n: number): NodeRef[] {
|
|
66
|
+
const scored = new Map<NodeRef, number>();
|
|
67
|
+
for (const e of ctx.outEdges.get(ref) ?? []) scored.set(e.to, Math.max(scored.get(e.to) ?? 0, e.weight));
|
|
68
|
+
for (const e of ctx.inEdges.get(ref) ?? []) scored.set(e.from, Math.max(scored.get(e.from) ?? 0, e.weight));
|
|
69
|
+
return [...scored].sort((a, b) => b[1] - a[1] || compareStr(a[0], b[0])).slice(0, n).map(([r]) => r);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const badge = (n: KNode | undefined, k: string): string | undefined => {
|
|
73
|
+
const v = n?.badges?.[k];
|
|
74
|
+
return v === undefined || v === '' ? undefined : String(v);
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const frontMatter = (n: KNode | undefined): string[] =>
|
|
78
|
+
Object.keys(n?.badges ?? {}).filter((k) => k.startsWith('fm.')).sort(compareStr)
|
|
79
|
+
.flatMap((k) => String(n!.badges![k]).split(',').map((v) => normalizeKey(v)));
|
|
80
|
+
|
|
81
|
+
function cardOf(n: KNode, ctx: CardCtx): { text: string; keys: string[] } {
|
|
82
|
+
const path = pathOfRef(n.ref) ?? '';
|
|
83
|
+
const join = (parts: Array<string | undefined>) => parts.filter((p) => p !== undefined && p !== '').join(SEP);
|
|
84
|
+
switch (n.kind) {
|
|
85
|
+
case 'file': {
|
|
86
|
+
const contained = (ctx.outEdges.get(n.ref) ?? []).filter((e) => e.kind === 'contains')
|
|
87
|
+
.map((e) => ctx.byRef.get(e.to)).filter((c): c is KNode => c?.kind === 'symbol');
|
|
88
|
+
const symbols = contained.map((s) => s.label).sort(compareStr);
|
|
89
|
+
return {
|
|
90
|
+
text: join([path, n.area && pathOfRef(n.area), n.partition, symbols.join(', '), badge(n, 'doc'),
|
|
91
|
+
topNeighbours(n.ref, ctx, 5).map(ctx.label).join(', ')]),
|
|
92
|
+
keys: [path, basenameOf(path), ...symbols, n.area ? pathOfRef(n.area) ?? '' : ''],
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
case 'section': {
|
|
96
|
+
const file = ctx.byRef.get(fileRefOf(n.ref));
|
|
97
|
+
const links = (ctx.outEdges.get(n.ref) ?? []).filter((e) => e.kind === 'links')
|
|
98
|
+
.map((e) => pathOfRef(e.to) ?? e.to).sort(compareStr);
|
|
99
|
+
return {
|
|
100
|
+
text: join([path, badge(n, 'breadcrumb') ?? n.label, badge(n, 'excerpt')?.slice(0, EXCERPT_CHARS), links.join(', ')]),
|
|
101
|
+
keys: [path, slugOf(n.label), ...frontMatter(file)],
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
case 'entity': {
|
|
105
|
+
const sources = (ctx.inEdges.get(n.ref) ?? []).filter((e) => e.kind === 'mentions' || e.kind === 'about')
|
|
106
|
+
.map((e) => ctx.byRef.get(e.from)).filter((s): s is KNode => !!s);
|
|
107
|
+
const about = (ctx.outEdges.get(n.ref) ?? []).filter((e) => e.kind === 'about')
|
|
108
|
+
.map((e) => ctx.byRef.get(e.to)).filter((s): s is KNode => !!s);
|
|
109
|
+
const top = [...new Map([...sources, ...about].map((s) => [s.ref, s])).values()].sort(byImportance).slice(0, 5);
|
|
110
|
+
const name = entityName(n.ref);
|
|
111
|
+
return {
|
|
112
|
+
text: join([n.label, badge(n, 'type'), top.map((s) => s.label).join(', ')]),
|
|
113
|
+
keys: [normalizeKey(name), normalizeKey(n.label.replace(/\s*\([^)]*\)\s*$/, '')),
|
|
114
|
+
...String(badge(n, 'aliases') ?? '').split(',').map(normalizeKey)],
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
case 'claim': {
|
|
118
|
+
const out = ctx.outEdges.get(n.ref) ?? [];
|
|
119
|
+
const sections = out.filter((e) => e.kind === 'about').map((e) => ctx.label(e.to)).sort(compareStr);
|
|
120
|
+
const entities = out.filter((e) => e.kind === 'mentions' && e.to.startsWith('ent:')).map((e) => normalizeKey(entityName(e.to)));
|
|
121
|
+
return { text: join([n.label, sections.join(', ')]), keys: entities };
|
|
122
|
+
}
|
|
123
|
+
case 'area': {
|
|
124
|
+
const members = [...(ctx.members.get(n.ref) ?? [])].sort(byImportance).slice(0, 10).map((m) => m.label);
|
|
125
|
+
return { text: join([path, n.partition, members.join(', ')]), keys: [path] };
|
|
126
|
+
}
|
|
127
|
+
default:
|
|
128
|
+
return { text: join([n.label, n.kind, path]), keys: [normalizeKey(n.label), path] };
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/knowledge/docs.test.ts
|
|
3
|
+
* The docs pack: the builtin parsers' `extract` (sections, contains, relative and wiki
|
|
4
|
+
* links, mentions, front-matter keys) and its findings (broken link, orphan), plus cards.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { htmlParser, markdownParser, plainTextParser } from '../ingest/parsers.js';
|
|
8
|
+
import type { SourceDocument } from '../ingest/types.js';
|
|
9
|
+
import { buildGraph } from './build.js';
|
|
10
|
+
import { toCards } from './cards.js';
|
|
11
|
+
import type { ExtractContext, Extracted, NodeRef } from './model.js';
|
|
12
|
+
import { DOCS_PACK } from './packs/docs.js';
|
|
13
|
+
import { asPathSource, resolveRelativePath } from './sources.js';
|
|
14
|
+
import { ref, ROOT } from './testkit.test-util.js';
|
|
15
|
+
|
|
16
|
+
const FILES: Record<string, string> = {
|
|
17
|
+
'README.md': '# Home\nSee [the guide](docs/guide.md) and [[notes]].\n',
|
|
18
|
+
'docs/guide.md': '---\ntitle: The Guide\ntags: [Alpha, beta]\n---\n# Guide\nIntro about the Lease.\n## Setup\nSee [missing](./nope.md), [ext](https://x.com) and .\n',
|
|
19
|
+
'notes.md': '# Notes\nBack to [home](README.md), [the page](docs/page.html) and [[docs/plain.txt]].\n',
|
|
20
|
+
'docs/orphan.md': '# Lonely\nNobody links here.\n',
|
|
21
|
+
'docs/page.html': '<html><title>Page</title><body><a href="guide.md">g</a></body></html>',
|
|
22
|
+
'docs/plain.txt': 'Plain text pointing at [[guide]].',
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
const ctx: ExtractContext = {
|
|
26
|
+
rootId: ROOT,
|
|
27
|
+
knownKeys: new Map<string, NodeRef>([['lease', 'ent:docs:lease']]),
|
|
28
|
+
resolvePath: resolveRelativePath,
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
function source(path: string): SourceDocument {
|
|
32
|
+
return asPathSource({ id: path, tenantId: 't', content: { kind: 'text', text: FILES[path]! } }, path, `h:${path}`);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function extract(path: string): Extracted {
|
|
36
|
+
const src = source(path);
|
|
37
|
+
const parser = [markdownParser, htmlParser, plainTextParser].find((p) => p.accepts(src))!;
|
|
38
|
+
return parser.extract!(src, parser.parse(src), ctx);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const all = Object.keys(FILES).map(extract);
|
|
42
|
+
const kinds = (ex: Extracted, kind: string) => ex.edges.filter((e) => e.kind === kind).map((e) => `${e.from} -> ${e.to}`);
|
|
43
|
+
|
|
44
|
+
describe('docs extract', () => {
|
|
45
|
+
it('emits a file node and section nodes from the heading breadcrumb, joined by contains', () => {
|
|
46
|
+
const guide = extract('docs/guide.md');
|
|
47
|
+
expect(guide.sourceRef).toBe(ref('docs/guide.md'));
|
|
48
|
+
expect(guide.sourceHash).toBe('h:docs/guide.md');
|
|
49
|
+
expect(guide.nodes.map((n) => `${n.kind}:${n.ref}`)).toEqual([
|
|
50
|
+
`file:${ref('docs/guide.md')}`,
|
|
51
|
+
`section:${ref('docs/guide.md')}#guide`,
|
|
52
|
+
`section:${ref('docs/guide.md')}#guide/setup`,
|
|
53
|
+
]);
|
|
54
|
+
expect(kinds(guide, 'contains')).toEqual([
|
|
55
|
+
`${ref('docs/guide.md')} -> ${ref('docs/guide.md')}#guide`,
|
|
56
|
+
`${ref('docs/guide.md')}#guide -> ${ref('docs/guide.md')}#guide/setup`,
|
|
57
|
+
]);
|
|
58
|
+
expect(guide.nodes.every((n) => n.origin === 'extracted')).toBe(true);
|
|
59
|
+
expect(guide.nodes[0]!.label).toBe('The Guide');
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('resolves relative and wiki links; skips external links and images', () => {
|
|
63
|
+
expect(kinds(extract('README.md'), 'links')).toEqual([
|
|
64
|
+
`${ref('README.md')}#home -> ${ref('docs/guide.md')}`,
|
|
65
|
+
`${ref('README.md')}#home -> ${ref('notes.md')}`,
|
|
66
|
+
]);
|
|
67
|
+
expect(kinds(extract('docs/guide.md'), 'links')).toEqual([`${ref('docs/guide.md')}#guide/setup -> ${ref('docs/nope.md')}`]);
|
|
68
|
+
expect(kinds(extract('docs/page.html'), 'links')).toEqual([`${ref('docs/page.html')} -> ${ref('docs/guide.md')}`]);
|
|
69
|
+
expect(kinds(extract('docs/plain.txt'), 'links')).toEqual([`${ref('docs/plain.txt')}#body -> ${ref('docs/guide.md')}`]);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it('mentions known keys by exact normalized match', () => {
|
|
73
|
+
expect(kinds(extract('docs/guide.md'), 'mentions')).toEqual([`${ref('docs/guide.md')}#guide -> ent:docs:lease`]);
|
|
74
|
+
expect(kinds(extract('notes.md'), 'mentions')).toEqual([]);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('keys carry the path, basename, heading slugs and front-matter values', () => {
|
|
78
|
+
const keys = extract('docs/guide.md').keys.map((k) => k.key);
|
|
79
|
+
expect(keys).toEqual(expect.arrayContaining(['docs/guide.md', 'guide', 'guide.md', 'the guide', 'alpha', 'beta', 'setup']));
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('is pure: the same source extracts to the same output', () => {
|
|
83
|
+
expect(JSON.stringify(extract('docs/guide.md'))).toBe(JSON.stringify(extract('docs/guide.md')));
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
describe('docs findings and cards', () => {
|
|
88
|
+
const g = buildGraph({ rootId: ROOT, extracted: all, facts: [], relations: [], packs: [DOCS_PACK] });
|
|
89
|
+
|
|
90
|
+
it('reports a broken link per linking file', () => {
|
|
91
|
+
const broken = g.findings.filter((f) => f.kind === 'broken-link');
|
|
92
|
+
expect(broken).toHaveLength(1);
|
|
93
|
+
expect(broken[0]!.nodes).toEqual([ref('docs/guide.md'), ref('docs/nope.md')]);
|
|
94
|
+
expect(broken[0]!.requestKey).toBe('knowledgeGraph.fix.brokenLink');
|
|
95
|
+
expect(g.nodes.find((n) => n.ref === ref('docs/nope.md'))?.kind).toBe('missing');
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('reports orphan documents (entry documents aside)', () => {
|
|
99
|
+
const orphan = g.findings.find((f) => f.kind === 'orphan');
|
|
100
|
+
expect(orphan?.nodes).toEqual([ref('docs/orphan.md')]);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it('places docs on top-level folder planes without verdicts', () => {
|
|
104
|
+
expect(g.nodes.find((n) => n.ref === ref('docs/guide.md'))?.partition).toBe('docs');
|
|
105
|
+
expect(g.nodes.find((n) => n.ref === ref('README.md'))?.partition).toBe('(root)');
|
|
106
|
+
expect(g.edges.some((e) => e.verdict)).toBe(false);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('cards: one per node, a section carries its breadcrumb and excerpt', () => {
|
|
110
|
+
const cards = toCards(g);
|
|
111
|
+
expect(cards.map((c) => c.id)).toEqual(g.nodes.map((n) => n.ref));
|
|
112
|
+
const setup = cards.find((c) => c.id === `${ref('docs/guide.md')}#guide/setup`)!;
|
|
113
|
+
expect(setup.text).toContain('Guide › Setup');
|
|
114
|
+
expect(setup.text).toContain('docs/nope.md');
|
|
115
|
+
expect(setup.keys).toEqual(expect.arrayContaining(['docs/guide.md', 'setup', 'alpha']));
|
|
116
|
+
expect(setup.hash).toMatch(/^[0-9a-f]{64}$/);
|
|
117
|
+
});
|
|
118
|
+
});
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* knowledge/findings.ts — what a person can act on (PRD 38 §3.5).
|
|
3
|
+
*
|
|
4
|
+
* Generic findings, for every pack: cycle, clone group, near-duplicate, partition
|
|
5
|
+
* violation, unpartitioned share, stale knowledge — plus `possible-same` entities. Packs
|
|
6
|
+
* add their own through {@link FindingRule}s. Every finding has a stable id (a hash of its
|
|
7
|
+
* kind and refs) and a `requestKey` under `knowledgeGraph.fix.*`: the fixed, prefilled
|
|
8
|
+
* Fix-with-agent request, never a model call.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { hashText } from '../ingest/hash.js';
|
|
12
|
+
import type { Finding, KEdge, KNode, KnowledgeGraph, KnowledgePack, NodeRef, PartitionRegistry } from './model.js';
|
|
13
|
+
import type { SimilarPair } from '../retrieval/knowledge-index/types.js';
|
|
14
|
+
import { possibleSameFindings } from './resolve.js';
|
|
15
|
+
import { compareStr, round6 } from './refs.js';
|
|
16
|
+
|
|
17
|
+
/** Node badge naming the clone group (e.g. a jscpd issue id) its `duplicate` edges belong to. */
|
|
18
|
+
export const CLONE_GROUP_BADGE = 'cloneGroup';
|
|
19
|
+
/** Refs listed on one grouped finding, at most (the count rides in params). */
|
|
20
|
+
export const MAX_FINDING_NODES = 50;
|
|
21
|
+
|
|
22
|
+
const SEVERITY_RANK = { error: 0, warn: 1, info: 2 } as const;
|
|
23
|
+
|
|
24
|
+
export function findingId(kind: string, refs: readonly string[]): string {
|
|
25
|
+
return `${kind}:${hashText([...refs].sort(compareStr).join('\u0000'))}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** `clone-group` → `knowledgeGraph.fix.cloneGroup`. */
|
|
29
|
+
export function requestKeyFor(kind: string): string {
|
|
30
|
+
return `knowledgeGraph.fix.${kind.replace(/-([a-z])/g, (_, c: string) => c.toUpperCase())}`;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function finding(kind: string, pack: string, nodes: NodeRef[], severity: Finding['severity'], params: Finding['params']): Finding {
|
|
34
|
+
const sorted = [...new Set(nodes)].sort(compareStr);
|
|
35
|
+
return {
|
|
36
|
+
id: findingId(kind, sorted), kind, pack, nodes: sorted.slice(0, MAX_FINDING_NODES), severity, params,
|
|
37
|
+
requestKey: requestKeyFor(kind),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Connected components over edges, each sorted; components sorted by first ref. */
|
|
42
|
+
export function components(edges: readonly KEdge[]): NodeRef[][] {
|
|
43
|
+
const parent = new Map<NodeRef, NodeRef>();
|
|
44
|
+
const find = (x: NodeRef): NodeRef => {
|
|
45
|
+
let r = x;
|
|
46
|
+
while (parent.get(r) !== r) r = parent.get(r)!;
|
|
47
|
+
parent.set(x, r);
|
|
48
|
+
return r;
|
|
49
|
+
};
|
|
50
|
+
for (const e of edges) {
|
|
51
|
+
for (const r of [e.from, e.to]) if (!parent.has(r)) parent.set(r, r);
|
|
52
|
+
const a = find(e.from), b = find(e.to);
|
|
53
|
+
if (a !== b) parent.set(compareStr(a, b) < 0 ? b : a, compareStr(a, b) < 0 ? a : b);
|
|
54
|
+
}
|
|
55
|
+
const groups = new Map<NodeRef, NodeRef[]>();
|
|
56
|
+
for (const r of [...parent.keys()].sort(compareStr)) {
|
|
57
|
+
const root = find(r);
|
|
58
|
+
let g = groups.get(root);
|
|
59
|
+
if (!g) groups.set(root, (g = []));
|
|
60
|
+
g.push(r);
|
|
61
|
+
}
|
|
62
|
+
return [...groups.values()].sort((a, b) => compareStr(a[0]!, b[0]!));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function genericFindings(g: KnowledgeGraph, registries: ReadonlyMap<string, PartitionRegistry>): Finding[] {
|
|
66
|
+
const byRef = new Map(g.nodes.map((n) => [n.ref, n] as const));
|
|
67
|
+
const packOf = (ref: NodeRef) => byRef.get(ref)?.pack ?? 'memory';
|
|
68
|
+
const out: Finding[] = [];
|
|
69
|
+
|
|
70
|
+
for (const refs of components(g.edges.filter((e) => e.cycle))) {
|
|
71
|
+
out.push(finding('cycle', packOf(refs[0]!), refs, 'warn', { nodes: refs.length }));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Clone groups: duplicate edges grouped by the clone-group badge, else by component.
|
|
75
|
+
const dup = g.edges.filter((e) => e.kind === 'duplicate');
|
|
76
|
+
const byBadge = new Map<string, KEdge[]>();
|
|
77
|
+
const unbadged: KEdge[] = [];
|
|
78
|
+
for (const e of dup) {
|
|
79
|
+
const badge = byRef.get(e.from)?.badges?.[CLONE_GROUP_BADGE] ?? byRef.get(e.to)?.badges?.[CLONE_GROUP_BADGE];
|
|
80
|
+
if (badge === undefined) { unbadged.push(e); continue; }
|
|
81
|
+
const list = byBadge.get(String(badge)) ?? [];
|
|
82
|
+
list.push(e);
|
|
83
|
+
byBadge.set(String(badge), list);
|
|
84
|
+
}
|
|
85
|
+
const groups: Array<{ group: string; refs: NodeRef[] }> = [
|
|
86
|
+
...[...byBadge].map(([group, es]) => ({ group, refs: es.flatMap((e) => [e.from, e.to]) })),
|
|
87
|
+
...components(unbadged).map((refs) => ({ group: '', refs })),
|
|
88
|
+
];
|
|
89
|
+
for (const { group, refs } of groups) {
|
|
90
|
+
const unique = [...new Set(refs)];
|
|
91
|
+
out.push(finding('clone-group', packOf(unique.sort(compareStr)[0]!), unique, 'warn', { group, members: unique.length }));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
for (const e of g.edges) {
|
|
95
|
+
if (e.kind === 'near-duplicate') {
|
|
96
|
+
out.push(finding('near-duplicate', packOf(e.from), [e.from, e.to], 'info', { similarity: round6(e.weight) }));
|
|
97
|
+
}
|
|
98
|
+
if (e.verdict === 'upward' || e.verdict === 'skips') {
|
|
99
|
+
const from = byRef.get(e.from), to = byRef.get(e.to);
|
|
100
|
+
out.push(finding('partition-violation', packOf(e.from), [e.from, e.to], e.verdict === 'upward' ? 'error' : 'warn', {
|
|
101
|
+
verdict: e.verdict, from: from?.partition ?? '', to: to?.partition ?? '', kind: e.kind,
|
|
102
|
+
}));
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
for (const [pack, reg] of [...registries].sort((a, b) => compareStr(a[0], b[0]))) {
|
|
107
|
+
if (!reg.allow) continue;
|
|
108
|
+
const files = g.nodes.filter((n) => n.pack === pack && n.kind === 'file');
|
|
109
|
+
const loose = files.filter((n) => n.partition === undefined).map((n) => n.ref);
|
|
110
|
+
if (loose.length === 0) continue;
|
|
111
|
+
out.push(finding('unpartitioned', pack, loose, 'info', {
|
|
112
|
+
count: loose.length, total: files.length, share: Math.round((loose.length / files.length) * 100),
|
|
113
|
+
}));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
out.push(...staleFindings(g, byRef));
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** One finding per source whose stored knowledge went stale (unanchored stale nodes: one each). */
|
|
121
|
+
function staleFindings(g: KnowledgeGraph, byRef: ReadonlyMap<NodeRef, KNode>): Finding[] {
|
|
122
|
+
const about = new Map<NodeRef, NodeRef>();
|
|
123
|
+
for (const e of g.edges) if (e.kind === 'about' && !about.has(e.from)) about.set(e.from, e.to);
|
|
124
|
+
const bySource = new Map<NodeRef, NodeRef[]>();
|
|
125
|
+
for (const n of g.nodes) {
|
|
126
|
+
if (!n.stale) continue;
|
|
127
|
+
const target = about.get(n.ref) ?? n.ref;
|
|
128
|
+
const list = bySource.get(target) ?? [];
|
|
129
|
+
list.push(n.ref);
|
|
130
|
+
bySource.set(target, list);
|
|
131
|
+
}
|
|
132
|
+
return [...bySource].map(([source, refs]) =>
|
|
133
|
+
finding('stale', byRef.get(source)?.pack ?? 'memory', [source, ...refs], 'warn', { count: refs.length }));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Generic + possible-same + every pack's rules; deduped by id; severity, then id. */
|
|
137
|
+
export function computeFindings(
|
|
138
|
+
g: KnowledgeGraph,
|
|
139
|
+
packs: readonly KnowledgePack[],
|
|
140
|
+
registries: ReadonlyMap<string, PartitionRegistry>,
|
|
141
|
+
nearDuplicates: readonly SimilarPair[],
|
|
142
|
+
): Finding[] {
|
|
143
|
+
const all = [...genericFindings(g, registries), ...possibleSameFindings(nearDuplicates)];
|
|
144
|
+
for (const pack of [...packs].sort((a, b) => compareStr(a.id, b.id))) {
|
|
145
|
+
for (const rule of pack.findings ?? []) {
|
|
146
|
+
for (const f of rule.find(g)) {
|
|
147
|
+
all.push({
|
|
148
|
+
...f,
|
|
149
|
+
id: f.id || findingId(f.kind, f.nodes),
|
|
150
|
+
nodes: [...new Set(f.nodes)].sort(compareStr),
|
|
151
|
+
requestKey: f.requestKey ?? requestKeyFor(f.kind),
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const byId = new Map<string, Finding>();
|
|
157
|
+
for (const f of all) if (!byId.has(f.id)) byId.set(f.id, f);
|
|
158
|
+
return [...byId.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || compareStr(a.id, b.id));
|
|
159
|
+
}
|