open-memex 0.9.0-alpha.3 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +27 -2
- package/CHANGELOG.md +5 -7
- package/README.md +43 -1
- package/README.zh-CN.md +26 -1
- package/dist/.build-stamp +1 -1
- package/dist/cli.js +167 -2
- package/dist/config.js +11 -0
- package/dist/graph/extract.js +234 -0
- package/dist/graph/lint.js +89 -0
- package/dist/graph/outline.js +146 -0
- package/dist/graph/queries.js +122 -0
- package/dist/plugin-core.js +4 -0
- package/dist/retrieve/inject.js +26 -2
- package/dist/review.js +9 -0
- package/dist/store/db.js +8 -2
- package/dist/store/graph.js +225 -0
- package/dist/store/sync.js +7 -3
- package/dist/tools/ops.js +10 -1
- package/docs/V2-DESIGN.md +2 -2
- package/docs/design/decisions.md +3 -0
- package/docs/design/knowledge-graph.md +1 -1
- package/package.json +1 -1
- package/scripts/smoke-pure.ts +203 -0
- package/scripts/test-full.ts +76 -4
- package/server.json +2 -2
- package/skills/open-memex/SKILL.md +17 -0
- package/src/cli.ts +181 -4
- package/src/config.ts +17 -0
- package/src/graph/extract.ts +276 -0
- package/src/graph/lint.ts +107 -0
- package/src/graph/outline.ts +167 -0
- package/src/graph/queries.ts +143 -0
- package/src/plugin-core.ts +4 -0
- package/src/retrieve/inject.ts +27 -2
- package/src/review.ts +9 -0
- package/src/store/db.ts +8 -2
- package/src/store/graph.ts +293 -0
- package/src/store/sync.ts +6 -2
- package/src/tools/ops.ts +11 -2
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §20.3 (D88): the graph layer — structural edges only, extracted by
|
|
3
|
+
* deterministic rules. No LLM, no network: everything here runs on the
|
|
4
|
+
* memory rows already on this machine.
|
|
5
|
+
*
|
|
6
|
+
* Borrowing Graphify's tree-sitter principle (§20.5): structure is
|
|
7
|
+
* *extracted*, not inferred. What the memory states explicitly in
|
|
8
|
+
* frontmatter (supersedes chain, derived_from) and in its body (a
|
|
9
|
+
* `[[wiki-link]]` around another memory's id) becomes an EXTRACTED edge —
|
|
10
|
+
* evidence the source itself wrote, confidence `high`. What the system
|
|
11
|
+
* merely notices (a bare id in prose, two active memories sharing a tag)
|
|
12
|
+
* is an INFERRED edge — a suggestion, never presented as a fact (§11's
|
|
13
|
+
* trust model). Inferred edges are always marked, so a graph hit can never
|
|
14
|
+
* quietly become "the source said so".
|
|
15
|
+
*
|
|
16
|
+
* Whether `related` enters the frontmatter is a v0.3 protocol decision
|
|
17
|
+
* (§20.8); until then edges live only in this derived layer. The layer is
|
|
18
|
+
* rebuildable: `graph build` re-derives everything from the Markdown
|
|
19
|
+
* source of truth (D1).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export type EdgeType = "supersedes" | "derived-from" | "relates-to";
|
|
23
|
+
export type Provenance = "EXTRACTED" | "INFERRED";
|
|
24
|
+
export type Confidence = "high" | "medium" | "low";
|
|
25
|
+
|
|
26
|
+
/** One memory row as the graph sees it — a projection of the index. */
|
|
27
|
+
export interface GraphNode {
|
|
28
|
+
id: string;
|
|
29
|
+
status: string;
|
|
30
|
+
type: string;
|
|
31
|
+
tags: string[];
|
|
32
|
+
body: string;
|
|
33
|
+
/** Frontmatter `supersedes` (on the NEW memory, points BACK at the old). */
|
|
34
|
+
supersedes?: string | null;
|
|
35
|
+
/** Frontmatter `derived_from` (propose-copies point at the personal original). */
|
|
36
|
+
derived_from?: string | null;
|
|
37
|
+
/** Optional pre-truncated one-liner for outline rendering. */
|
|
38
|
+
snippet?: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** One directed edge with provenance (§20.3). */
|
|
42
|
+
export interface GraphEdge {
|
|
43
|
+
from: string;
|
|
44
|
+
to: string;
|
|
45
|
+
type: EdgeType;
|
|
46
|
+
provenance: Provenance;
|
|
47
|
+
confidence: Confidence;
|
|
48
|
+
/** What produced the edge (e.g. `shared-tag:deploy,ci`), for query receipts. */
|
|
49
|
+
via: string;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The deterministic confidence ladder for shared-tag edges: one shared tag
|
|
54
|
+
* is the weakest signal, two starts to connect, three or more together are
|
|
55
|
+
* a strong one. Decision D91 keeps this fixed — an agent reading a graph
|
|
56
|
+
* hit gets the same meaning on every install.
|
|
57
|
+
*/
|
|
58
|
+
export function sharedTagConfidence(sharedCount: number): Confidence {
|
|
59
|
+
if (sharedCount >= 3) return "high";
|
|
60
|
+
if (sharedCount === 2) return "medium";
|
|
61
|
+
return "low";
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Crockford-base32 alphabet ULID (0-9, A-Z minus I, L, O, U) delimited by
|
|
66
|
+
* non-alphanumerics. The look-arounds keep a longer prefixed token from
|
|
67
|
+
* matching as a "mention": only a standalone id in the body counts as
|
|
68
|
+
* explicit evidence. The class must exclude L as well as I/O/U — a token
|
|
69
|
+
* like `FLATBATTERYWELLSXXQZABCDEF` is a word, not an id, and an EXTRACTED
|
|
70
|
+
* edge means the source said it explicitly.
|
|
71
|
+
*/
|
|
72
|
+
const ULID_BODY = "[0-9A-HJKMNP-TV-Z]{26}";
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* A `[[ULID]]` wiki-link is the writer's explicit statement that this is a
|
|
76
|
+
* reference — EXTRACTED. A bare ULID is an id typed in prose: the system
|
|
77
|
+
* noticed it, it was not marked as a link — INFERRED (§20.3, D91).
|
|
78
|
+
*/
|
|
79
|
+
const WIKI_LINK_RE = new RegExp(`\\[\\[(${ULID_BODY})\\]\\]`, "g");
|
|
80
|
+
const BARE_ULID_RE = new RegExp(`(?<![0-9A-Z])${ULID_BODY}(?![0-9A-Z])`, "g");
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Per-node cap on INFERRED shared-tag edges. A dense tag vocabulary would
|
|
84
|
+
* otherwise grow the stored layer quadratically; the cap keeps, per node,
|
|
85
|
+
* the top neighbours by (shared count desc, id asc) — deterministic — and
|
|
86
|
+
* never drops an EXTRACTED edge.
|
|
87
|
+
*/
|
|
88
|
+
export const MAX_SHARED_TAG_EDGES_PER_NODE = 12;
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The node count above which shared-tag inference is skipped entirely.
|
|
92
|
+
* At that scale the O(n²) pair scan stops being cheap; the LLM half of
|
|
93
|
+
* §20, not more low-confidence edges, is the answer there.
|
|
94
|
+
*/
|
|
95
|
+
export const MAX_SHARED_TAG_NODES = 2000;
|
|
96
|
+
|
|
97
|
+
/** One edge per (from, to, type). Used while collecting raw candidates. */
|
|
98
|
+
function dedupInsert(list: GraphEdge[], edge: GraphEdge): void {
|
|
99
|
+
for (const e of list) {
|
|
100
|
+
if (e.from === edge.from && e.to === edge.to && e.type === edge.type) return;
|
|
101
|
+
}
|
|
102
|
+
list.push(edge);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const pairKey = (a: string, b: string) => (a < b ? `${a}\u0000${b}` : `${b}\u0000${a}`);
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Extract all structural edges for a set of nodes (§20.3):
|
|
109
|
+
*
|
|
110
|
+
* - `supersedes` — frontmatter chain, new → old (EXTRACTED / high).
|
|
111
|
+
* - `derived-from` — frontmatter `derived_from`, copy → original (EXTRACTED / high).
|
|
112
|
+
* - `relates-to` — EXTRACTED when one body wraps another memory's ULID in
|
|
113
|
+
* `[[ ]]` (an explicit link); INFERRED/high when an id
|
|
114
|
+
* appears bare in prose; INFERRED at the shared-tag
|
|
115
|
+
* confidence ladder when two active memories share tags.
|
|
116
|
+
* One edge per unordered pair; an EXTRACTED body-mention
|
|
117
|
+
* beats the INFERRED links for the same pair.
|
|
118
|
+
*
|
|
119
|
+
* Edges are returned sorted (from, to, type) so two builds over the same
|
|
120
|
+
* data produce byte-identical output — the graph must behave like a
|
|
121
|
+
* derived index, not like a fickle cache.
|
|
122
|
+
*/
|
|
123
|
+
export function extractEdges(nodes: GraphNode[]): GraphEdge[] {
|
|
124
|
+
const raw: GraphEdge[] = [];
|
|
125
|
+
|
|
126
|
+
for (const n of nodes) {
|
|
127
|
+
if (n.supersedes && n.supersedes !== n.id) {
|
|
128
|
+
dedupInsert(raw, {
|
|
129
|
+
from: n.id,
|
|
130
|
+
to: n.supersedes,
|
|
131
|
+
type: "supersedes",
|
|
132
|
+
provenance: "EXTRACTED",
|
|
133
|
+
confidence: "high",
|
|
134
|
+
via: "supersede-chain",
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
if (n.derived_from && n.derived_from !== n.id) {
|
|
138
|
+
dedupInsert(raw, {
|
|
139
|
+
from: n.id,
|
|
140
|
+
to: n.derived_from,
|
|
141
|
+
type: "derived-from",
|
|
142
|
+
provenance: "EXTRACTED",
|
|
143
|
+
confidence: "high",
|
|
144
|
+
via: "derive",
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Body mentions are stored even when the mentioned id is dangling —
|
|
150
|
+
// lint enumerates it (graph/lint.ts); the query layer simply never
|
|
151
|
+
// traverses into a node that does not exist.
|
|
152
|
+
for (const n of nodes) {
|
|
153
|
+
const skip = new Set<string>(
|
|
154
|
+
[n.id, n.supersedes, n.derived_from].filter((x): x is string => Boolean(x)),
|
|
155
|
+
);
|
|
156
|
+
// Explicit wiki-links first so dedupInsert keeps the EXTRACTED edge
|
|
157
|
+
// when the same pair is also mentioned bare somewhere in the body.
|
|
158
|
+
for (const m of n.body.matchAll(WIKI_LINK_RE)) {
|
|
159
|
+
const id = m[1]!;
|
|
160
|
+
if (skip.has(id)) continue;
|
|
161
|
+
dedupInsert(raw, {
|
|
162
|
+
from: n.id,
|
|
163
|
+
to: id,
|
|
164
|
+
type: "relates-to",
|
|
165
|
+
provenance: "EXTRACTED",
|
|
166
|
+
confidence: "high",
|
|
167
|
+
via: "wiki-link",
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
// Bare ids: the system noticed an id in prose, the writer did not link
|
|
171
|
+
// it — INFERRED/high (a 26-char id is an unambiguous id, not a guess;
|
|
172
|
+
// the uncertainty is about the *relationship*, hence not EXTRACTED).
|
|
173
|
+
// Strip the wiki-links first so they are not counted twice.
|
|
174
|
+
const bare = n.body.replace(WIKI_LINK_RE, " ");
|
|
175
|
+
for (const m of bare.matchAll(BARE_ULID_RE)) {
|
|
176
|
+
const id = m[0];
|
|
177
|
+
if (skip.has(id)) continue;
|
|
178
|
+
dedupInsert(raw, {
|
|
179
|
+
from: n.id,
|
|
180
|
+
to: id,
|
|
181
|
+
type: "relates-to",
|
|
182
|
+
provenance: "INFERRED",
|
|
183
|
+
confidence: "high",
|
|
184
|
+
via: "body-mention",
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Shared-tag inference scans ACTIVE memories only (a superseded version
|
|
190
|
+
// of a fact would otherwise double every inference) and is canonicalized
|
|
191
|
+
// to the unordered pair so `relates-to` stays one edge per pair.
|
|
192
|
+
if (nodes.length > 0 && nodes.length <= MAX_SHARED_TAG_NODES) {
|
|
193
|
+
const active = nodes.filter((n) => n.status === "active");
|
|
194
|
+
const tagSets = new Map<string, Set<string>>();
|
|
195
|
+
for (const n of active) tagSets.set(n.id, new Set(n.tags));
|
|
196
|
+
|
|
197
|
+
const candidates = new Map<string, Array<{ to: string; shared: string[] }>>();
|
|
198
|
+
const pairShared = new Map<string, string[]>();
|
|
199
|
+
const listFor = (id: string): Array<{ to: string; shared: string[] }> => {
|
|
200
|
+
let list = candidates.get(id);
|
|
201
|
+
if (!list) {
|
|
202
|
+
list = [];
|
|
203
|
+
candidates.set(id, list);
|
|
204
|
+
}
|
|
205
|
+
return list;
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
for (let i = 0; i < active.length; i++) {
|
|
209
|
+
const a = active[i]!;
|
|
210
|
+
const aSet = tagSets.get(a.id)!;
|
|
211
|
+
for (let j = i + 1; j < active.length; j++) {
|
|
212
|
+
const b = active[j]!;
|
|
213
|
+
const bSet = tagSets.get(b.id)!;
|
|
214
|
+
const shared: string[] = [];
|
|
215
|
+
for (const t of aSet) if (bSet.has(t)) shared.push(t);
|
|
216
|
+
if (shared.length === 0) continue;
|
|
217
|
+
shared.sort();
|
|
218
|
+
pairShared.set(pairKey(a.id, b.id), shared);
|
|
219
|
+
listFor(a.id)!.push({ to: b.id, shared });
|
|
220
|
+
listFor(b.id)!.push({ to: a.id, shared });
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
for (const [id, list] of candidates) {
|
|
225
|
+
list.sort(
|
|
226
|
+
(x, y) => y.shared.length - x.shared.length || x.to.localeCompare(y.to),
|
|
227
|
+
);
|
|
228
|
+
for (const c of list.slice(0, MAX_SHARED_TAG_EDGES_PER_NODE)) {
|
|
229
|
+
// Canonical direction + skip self-order duplicates: emit each
|
|
230
|
+
// unordered pair exactly once, from the smaller id's own list.
|
|
231
|
+
if (id >= c.to) continue;
|
|
232
|
+
dedupInsert(raw, {
|
|
233
|
+
from: id,
|
|
234
|
+
to: c.to,
|
|
235
|
+
type: "relates-to",
|
|
236
|
+
provenance: "INFERRED",
|
|
237
|
+
confidence: sharedTagConfidence(pairShared.get(pairKey(id, c.to))!.length),
|
|
238
|
+
via: `shared-tag:${pairShared.get(pairKey(id, c.to))!.join(",")}`,
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Collapse relates-to to one edge per unordered pair, canonicalized to
|
|
245
|
+
// the smaller id as `from`; the EXTRACTED body-mention beats the
|
|
246
|
+
// INFERRED shared-tag edge for the same pair. The cap is per node, so
|
|
247
|
+
// the two sides of a pair can disagree on which of them kept the edge —
|
|
248
|
+
// canonicalizing here, after collection, is what makes the result
|
|
249
|
+
// independent of which candidate list the edge happened to survive in.
|
|
250
|
+
const relates = new Map<string, GraphEdge>();
|
|
251
|
+
const out: GraphEdge[] = [];
|
|
252
|
+
for (const e of raw) {
|
|
253
|
+
if (e.type !== "relates-to") {
|
|
254
|
+
out.push(e);
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
const from = e.from < e.to ? e.from : e.to;
|
|
258
|
+
const to = e.from < e.to ? e.to : e.from;
|
|
259
|
+
const k = pairKey(from, to);
|
|
260
|
+
const cur = relates.get(k);
|
|
261
|
+
if (!cur || (cur.provenance === "INFERRED" && e.provenance === "EXTRACTED")) {
|
|
262
|
+
relates.set(k, { ...e, from, to });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
for (const e of relates.values()) out.push(e);
|
|
266
|
+
|
|
267
|
+
out.sort(
|
|
268
|
+
(x, y) =>
|
|
269
|
+
x.from === y.from
|
|
270
|
+
? x.to === y.to
|
|
271
|
+
? x.type.localeCompare(y.type)
|
|
272
|
+
: x.to.localeCompare(y.to)
|
|
273
|
+
: x.from.localeCompare(y.from),
|
|
274
|
+
);
|
|
275
|
+
return out;
|
|
276
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §20.4 (D88): graph lint — a health check that only discovers and
|
|
3
|
+
* reports; it never auto-resolves (Principle 5). The findings are the
|
|
4
|
+
* input the promotion flow (§5) and D78's curate step consume later:
|
|
5
|
+
* "which memories contradict (LLM half), which are orphaned, which are
|
|
6
|
+
* stale". Contradiction detection needs the LLM half of §20 and is
|
|
7
|
+
* deliberately out of D91's deterministic subset.
|
|
8
|
+
*
|
|
9
|
+
* Deterministic checks in D91:
|
|
10
|
+
* - orphan — an active memory with zero structural edges: written
|
|
11
|
+
* but never linked, in no topic.
|
|
12
|
+
* - dangling-edge — an edge points at an id that does not exist in the
|
|
13
|
+
* index (a mention to a deleted/foreign memory). The
|
|
14
|
+
* chain-repair path (§3.3) self-heals frontmatter links
|
|
15
|
+
* with a warning; lint just enumerates the leftovers.
|
|
16
|
+
* - stale — an active memory whose body is a near-duplicate
|
|
17
|
+
* (Jaccard ≥ 0.8, the §3.4 threshold) of a superseded or
|
|
18
|
+
* deprecated memory: it looks like a replacement that
|
|
19
|
+
* should have entered the supersede chain but didn't.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { similarity, NEAR_DUP_THRESHOLD } from "../store/lifecycle.ts";
|
|
23
|
+
import type { GraphEdge, GraphNode } from "./extract.ts";
|
|
24
|
+
|
|
25
|
+
export type LintKind = "orphan" | "dangling-edge" | "stale";
|
|
26
|
+
|
|
27
|
+
export interface LintFinding {
|
|
28
|
+
kind: LintKind;
|
|
29
|
+
/** The memory the human should look at (the REPORTED side). */
|
|
30
|
+
id: string;
|
|
31
|
+
/** The other end for dangling-edge / stale findings. */
|
|
32
|
+
other?: string;
|
|
33
|
+
detail: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function lintGraph(
|
|
37
|
+
nodes: GraphNode[],
|
|
38
|
+
edges: GraphEdge[],
|
|
39
|
+
foreignIds: Iterable<string> = [],
|
|
40
|
+
): LintFinding[] {
|
|
41
|
+
const byId = new Map(nodes.map((n) => [n.id, n]));
|
|
42
|
+
// Ids that exist in the store but OUTSIDE the linted scope: a propose-copy
|
|
43
|
+
// legitimately derives from the personal original, which lives in another
|
|
44
|
+
// scope — that is a foreign link, not a dangling one (D91).
|
|
45
|
+
const foreign = new Set(foreignIds);
|
|
46
|
+
const findings: LintFinding[] = [];
|
|
47
|
+
|
|
48
|
+
// Orphans: active memories not touched by any edge at all.
|
|
49
|
+
const connected = new Set<string>();
|
|
50
|
+
for (const e of edges) {
|
|
51
|
+
if (byId.has(e.from)) connected.add(e.from);
|
|
52
|
+
if (byId.has(e.to)) connected.add(e.to);
|
|
53
|
+
}
|
|
54
|
+
for (const n of nodes) {
|
|
55
|
+
if (n.status !== "active") continue;
|
|
56
|
+
if (!connected.has(n.id)) {
|
|
57
|
+
findings.push({
|
|
58
|
+
kind: "orphan",
|
|
59
|
+
id: n.id,
|
|
60
|
+
detail: "active but in no topic and linked to nothing — no tag overlap with other memories, no references",
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Dangling edges: an endpoint missing from the index. An endpoint that
|
|
66
|
+
// exists in ANOTHER scope is expected (the derive edge from a propose-copy
|
|
67
|
+
// points at the personal original) — skipped, not reported.
|
|
68
|
+
for (const e of edges) {
|
|
69
|
+
const missingFrom = !byId.has(e.from);
|
|
70
|
+
const missingTo = !byId.has(e.to);
|
|
71
|
+
if (!missingFrom && !missingTo) continue;
|
|
72
|
+
if ((missingFrom && foreign.has(e.from)) || (missingTo && foreign.has(e.to))) continue;
|
|
73
|
+
const other = missingFrom ? e.from : e.to;
|
|
74
|
+
const id = other === e.from ? e.to : e.from;
|
|
75
|
+
findings.push({
|
|
76
|
+
kind: "dangling-edge",
|
|
77
|
+
id,
|
|
78
|
+
other,
|
|
79
|
+
detail: `${e.type} edge ${e.via} points at ${other}, which is not in this index (deleted, another scope, or a foreign id)`,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Stale: active content that nearly duplicates a superseded/deprecated
|
|
84
|
+
// memory — the newer copy and the old chain never met.
|
|
85
|
+
const retired = nodes.filter((n) => n.status === "superseded" || n.status === "deprecated");
|
|
86
|
+
for (const n of nodes) {
|
|
87
|
+
if (n.status !== "active") continue;
|
|
88
|
+
for (const r of retired) {
|
|
89
|
+
const score = similarity(n.body, r.body);
|
|
90
|
+
if (score < NEAR_DUP_THRESHOLD) continue;
|
|
91
|
+
findings.push({
|
|
92
|
+
kind: "stale",
|
|
93
|
+
id: n.id,
|
|
94
|
+
other: r.id,
|
|
95
|
+
detail: `active but ${score === 1 ? "identical" : `near-duplicate (${score.toFixed(2)})`} to ${r.id} (${r.status}) — one of them should be superseded by the other`,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
findings.sort(
|
|
101
|
+
(a, b) =>
|
|
102
|
+
a.kind.localeCompare(b.kind) ||
|
|
103
|
+
a.id.localeCompare(b.id) ||
|
|
104
|
+
(a.other ?? "").localeCompare(b.other ?? ""),
|
|
105
|
+
);
|
|
106
|
+
return findings;
|
|
107
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §20.4 / D88: the compiled outline — one compact topic map of what is in
|
|
3
|
+
* memory, derived from the structural graph and injected at session start.
|
|
4
|
+
*
|
|
5
|
+
* Injection-cost discipline (2026-10-19): "important but not every turn"
|
|
6
|
+
* stays on-demand, so this renderer exposes a line cap (D91: default 10)
|
|
7
|
+
* and nothing more. Topics are components of active memories joined ONLY
|
|
8
|
+
* by medium/high INFERRED shared-tag edges — a low-confidence edge (one
|
|
9
|
+
* shared tag) never contributes to a topic. That is the guard against the
|
|
10
|
+
* universal-tag collapse: one common tag across the whole store produces
|
|
11
|
+
* only low band edges, so the store cannot silently fuse into a single
|
|
12
|
+
* "topic" that says nothing. Body-mention / supersedes / derived-from
|
|
13
|
+
* edges are EXTRACTED evidence for queries and lint, not topic glue —
|
|
14
|
+
* two memories can point at each other in prose without belonging to
|
|
15
|
+
* the same cluster.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { GraphEdge, GraphNode } from "./extract.ts";
|
|
19
|
+
|
|
20
|
+
export interface OutlineTopic {
|
|
21
|
+
/** The most representative shared tag inside the topic. */
|
|
22
|
+
name: string;
|
|
23
|
+
ids: string[];
|
|
24
|
+
size: number;
|
|
25
|
+
/** Member ids whose one-liners actually appear in the rendered line. */
|
|
26
|
+
snippetIds: string[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface OutlineResult {
|
|
30
|
+
/** Rendered lines (topics first, disclosure last), capped by maxLines. */
|
|
31
|
+
lines: string[];
|
|
32
|
+
topics: OutlineTopic[];
|
|
33
|
+
/** Ids named in the lines — the outline is a retrieval surface (D77-L2). */
|
|
34
|
+
exposureIds: string[];
|
|
35
|
+
/** Rendered topics that did not fit the cap. */
|
|
36
|
+
droppedTopics: number;
|
|
37
|
+
/** Active memories not in any topic (disclosed, never listed). */
|
|
38
|
+
outsideCount: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function oneLine(s: string, max = 120): string {
|
|
42
|
+
const trimmed = s.replace(/\s+/g, " ").trim();
|
|
43
|
+
return trimmed.length > max ? trimmed.slice(0, max - 3) + "..." : trimmed;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Union-find over string ids. Both-sides threading keeps the same shape
|
|
47
|
+
* as the package's other pure modules (small, no deps). */
|
|
48
|
+
class UnionFind {
|
|
49
|
+
private parent = new Map<string, string>();
|
|
50
|
+
find(id: string): string {
|
|
51
|
+
if (!this.parent.has(id)) this.parent.set(id, id);
|
|
52
|
+
let root = id;
|
|
53
|
+
while (this.parent.get(root) !== root) root = this.parent.get(root)!;
|
|
54
|
+
let cur = id;
|
|
55
|
+
while (cur !== root) {
|
|
56
|
+
const next = this.parent.get(cur)!;
|
|
57
|
+
this.parent.set(cur, root);
|
|
58
|
+
cur = next;
|
|
59
|
+
}
|
|
60
|
+
return root;
|
|
61
|
+
}
|
|
62
|
+
union(a: string, b: string): void {
|
|
63
|
+
const ra = this.find(a);
|
|
64
|
+
const rb = this.find(b);
|
|
65
|
+
if (ra !== rb) this.parent.set(ra, rb);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Render the outline. `nodes` must be the visible set (active memories —
|
|
71
|
+
* the caller filters; superseded versions belong to history, not to the
|
|
72
|
+
* "what is in memory" map). `edges` must already be scoped to those nodes.
|
|
73
|
+
*/
|
|
74
|
+
export function renderOutline(
|
|
75
|
+
nodes: GraphNode[],
|
|
76
|
+
edges: GraphEdge[],
|
|
77
|
+
opts: { maxLines?: number } = {},
|
|
78
|
+
): OutlineResult {
|
|
79
|
+
// Floor 1, ceiling 30 (the same bound `config set graphOutlineLines`
|
|
80
|
+
// enforces) — a hand-edited config file must not be able to blow up the
|
|
81
|
+
// session-start injection.
|
|
82
|
+
const maxLines = Math.min(30, Math.max(1, opts.maxLines ?? 10));
|
|
83
|
+
const byId = new Map(nodes.map((n) => [n.id, n]));
|
|
84
|
+
|
|
85
|
+
const uf = new UnionFind();
|
|
86
|
+
for (const n of nodes) uf.find(n.id);
|
|
87
|
+
for (const e of edges) {
|
|
88
|
+
if (e.provenance !== "INFERRED") continue;
|
|
89
|
+
if (e.confidence !== "medium" && e.confidence !== "high") continue;
|
|
90
|
+
// Only shared-tag evidence creates topics. A supersede chain, a
|
|
91
|
+
// derive copy or a body mention (even an INFERRED bare-id mention) is
|
|
92
|
+
// a query/lint relation, not topic glue — otherwise two memories with
|
|
93
|
+
// no common subject could form a "related" topic.
|
|
94
|
+
if (!e.via.startsWith("shared-tag:")) continue;
|
|
95
|
+
if (!byId.has(e.from) || !byId.has(e.to)) continue;
|
|
96
|
+
uf.union(e.from, e.to);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const components = new Map<string, GraphNode[]>();
|
|
100
|
+
for (const n of nodes) {
|
|
101
|
+
const root = uf.find(n.id);
|
|
102
|
+
const list = components.get(root);
|
|
103
|
+
if (list) list.push(n);
|
|
104
|
+
else components.set(root, [n]);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const topics: OutlineTopic[] = [];
|
|
108
|
+
let outside = 0;
|
|
109
|
+
for (const members of components.values()) {
|
|
110
|
+
if (members.length < 2) {
|
|
111
|
+
outside++;
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
// Topic name: the tag with the most occurrences among members
|
|
115
|
+
// (counted per member), ties broken lexicographically. A universal
|
|
116
|
+
// tag can win here when a topic is real anyway — that is fine; what
|
|
117
|
+
// it can never do by itself is CREATE the component (only medium+
|
|
118
|
+
// edges join).
|
|
119
|
+
const counts = new Map<string, number>();
|
|
120
|
+
for (const m of members) for (const t of m.tags) counts.set(t, (counts.get(t) ?? 0) + 1);
|
|
121
|
+
let name = "";
|
|
122
|
+
let best = 0;
|
|
123
|
+
for (const [t, c] of counts) {
|
|
124
|
+
if (c > best || (c === best && (name === "" || t < name))) {
|
|
125
|
+
name = t;
|
|
126
|
+
best = c;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (!name) name = "related";
|
|
130
|
+
// Snippet members: largest-tag-bearing members first, stable by id.
|
|
131
|
+
const ordered = [...members].sort((x, y) => {
|
|
132
|
+
const cx = x.tags.includes(name) ? 1 : 0;
|
|
133
|
+
const cy = y.tags.includes(name) ? 1 : 0;
|
|
134
|
+
return cy - cx || x.id.localeCompare(y.id);
|
|
135
|
+
});
|
|
136
|
+
topics.push({
|
|
137
|
+
name,
|
|
138
|
+
size: members.length,
|
|
139
|
+
ids: members.map((m) => m.id),
|
|
140
|
+
snippetIds: ordered.slice(0, 2).map((m) => m.id),
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
topics.sort((a, b) => b.size - a.size || a.name.localeCompare(b.name) || a.ids[0]!.localeCompare(b.ids[0]!));
|
|
145
|
+
|
|
146
|
+
const shown = topics.slice(0, maxLines);
|
|
147
|
+
const dropped = topics.length - shown.length;
|
|
148
|
+
const lines: string[] = [];
|
|
149
|
+
const exposureIds: string[] = [];
|
|
150
|
+
for (const t of shown) {
|
|
151
|
+
const members = t.ids
|
|
152
|
+
.map((id) => byId.get(id)!)
|
|
153
|
+
.filter(Boolean);
|
|
154
|
+
const tagMembers = members.filter((m) => m.tags.includes(t.name));
|
|
155
|
+
const snippets = (tagMembers.length > 0 ? tagMembers : members)
|
|
156
|
+
.slice(0, 2)
|
|
157
|
+
.map((m) => oneLine(m.snippet ?? m.body));
|
|
158
|
+
lines.push(`- ${t.name} (${t.size}): ${snippets.join(" | ")}`);
|
|
159
|
+
for (const id of t.snippetIds) exposureIds.push(id);
|
|
160
|
+
}
|
|
161
|
+
if (dropped > 0) lines.push(`(+${dropped} more topic${dropped === 1 ? "" : "s"} not shown)`);
|
|
162
|
+
if (outside > 0) {
|
|
163
|
+
lines.push(`(+${outside} memor${outside === 1 ? "y" : "ies"} not in any topic)`);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return { lines, topics: shown, exposureIds, droppedTopics: dropped, outsideCount: outside };
|
|
167
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §20.4 (D88): graph traversal as a read path — a node's neighbours and
|
|
3
|
+
* the shortest path between two memories. Pure over the extracted edge
|
|
4
|
+
* list; the store layer (src/store/graph.ts) owns persistence and gets
|
|
5
|
+
* edges back from here untouched.
|
|
6
|
+
*
|
|
7
|
+
* Traversal is undirected: a `supersedes` edge says new → old in the
|
|
8
|
+
* frontmatter, but from the reader's side a chain link works both ways.
|
|
9
|
+
* Every hop carries its edge (type + provenance + confidence + via) so a
|
|
10
|
+
* query receipt can tell the reader which connection was travelled and
|
|
11
|
+
* whether it was INFERRED — §20.3's rule that an inferred link must never
|
|
12
|
+
* look like a fact that was in the source.
|
|
13
|
+
*
|
|
14
|
+
* Traversal never crosses into a node that is not in the node set: a
|
|
15
|
+
* dangling id in an edge is lint's finding to report, not a ghost node
|
|
16
|
+
* to walk through.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { GraphEdge, GraphNode } from "./extract.ts";
|
|
20
|
+
|
|
21
|
+
export interface NeighbourHit {
|
|
22
|
+
id: string;
|
|
23
|
+
/** BFS depth at which this neighbour was reached (1 = direct). */
|
|
24
|
+
depth: number;
|
|
25
|
+
/** The edge(s) that reached this node, one per discovery path's shortest layer. */
|
|
26
|
+
via: GraphEdge;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Undirected adjacency: id → outgoing + incoming edges. */
|
|
30
|
+
export function buildAdjacency(edges: GraphEdge[], nodeIds: Iterable<string>): Map<string, GraphEdge[]> {
|
|
31
|
+
const known = new Set(nodeIds);
|
|
32
|
+
const adj = new Map<string, GraphEdge[]>();
|
|
33
|
+
for (const id of known) adj.set(id, []);
|
|
34
|
+
for (const e of edges) {
|
|
35
|
+
if (!known.has(e.from) || !known.has(e.to)) continue;
|
|
36
|
+
adj.get(e.from)!.push(e);
|
|
37
|
+
const back: GraphEdge = { ...e, from: e.to, to: e.from };
|
|
38
|
+
adj.get(e.to)!.push(back);
|
|
39
|
+
}
|
|
40
|
+
return adj;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* BFS neighbours of `startId`, nearest first (depth ascending, id
|
|
45
|
+
* ascending within a depth — deterministic output). `maxDepth` caps the
|
|
46
|
+
* walk; `maxNodes` caps the result the same way a search `limit` does.
|
|
47
|
+
* The start node itself is never returned.
|
|
48
|
+
*/
|
|
49
|
+
export function graphNeighbours(
|
|
50
|
+
startId: string,
|
|
51
|
+
edges: GraphEdge[],
|
|
52
|
+
nodeIds: Iterable<string>,
|
|
53
|
+
opts: { maxDepth?: number; maxNodes?: number } = {},
|
|
54
|
+
): NeighbourHit[] {
|
|
55
|
+
const maxDepth = Math.max(1, Math.min(opts.maxDepth ?? 1, 6));
|
|
56
|
+
const maxNodes = Math.max(1, Math.min(opts.maxNodes ?? 20, 100));
|
|
57
|
+
const adj = buildAdjacency(edges, nodeIds);
|
|
58
|
+
if (!adj.has(startId)) return [];
|
|
59
|
+
|
|
60
|
+
const visited = new Set<string>([startId]);
|
|
61
|
+
const hits: NeighbourHit[] = [];
|
|
62
|
+
let frontier: Array<{ id: string; via: GraphEdge }> = [];
|
|
63
|
+
const enqueue = (edges: GraphEdge[]) => {
|
|
64
|
+
for (const e of [...edges].sort((a, b) => a.to.localeCompare(b.to))) {
|
|
65
|
+
if (visited.has(e.to)) continue;
|
|
66
|
+
visited.add(e.to);
|
|
67
|
+
frontier.push({ id: e.to, via: e });
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
enqueue(adj.get(startId)!);
|
|
71
|
+
let depth = 1;
|
|
72
|
+
for (; depth <= maxDepth && frontier.length > 0 && hits.length < maxNodes; depth++) {
|
|
73
|
+
const layer = [...frontier]
|
|
74
|
+
.sort((a, b) => a.id.localeCompare(b.id) || a.via.via.localeCompare(b.via.via))
|
|
75
|
+
.slice(0, maxNodes - hits.length);
|
|
76
|
+
for (const f of layer) hits.push({ id: f.id, depth, via: f.via });
|
|
77
|
+
const next: Array<{ id: string; via: GraphEdge }> = [];
|
|
78
|
+
for (const f of layer) {
|
|
79
|
+
if (next.length + hits.length >= maxNodes) break;
|
|
80
|
+
for (const e of adj.get(f.id) ?? []) {
|
|
81
|
+
if (visited.has(e.to)) continue;
|
|
82
|
+
visited.add(e.to);
|
|
83
|
+
next.push({ id: e.to, via: e });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
frontier = next;
|
|
87
|
+
}
|
|
88
|
+
return hits;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export interface PathResult {
|
|
92
|
+
found: boolean;
|
|
93
|
+
/** The edges travelled in order, from-first to to-last (traversal direction). */
|
|
94
|
+
hops: GraphEdge[];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Shortest path between two ids (BFS, undirected, `maxDepth` cap). Each
|
|
99
|
+
* hop keeps the edge object in travel direction — the receipt reads
|
|
100
|
+
* "A —[relates-to INFERRED/medium (shared-tag:x)]—> B", whether or not
|
|
101
|
+
* that was the stored direction.
|
|
102
|
+
*/
|
|
103
|
+
export function shortestPath(
|
|
104
|
+
fromId: string,
|
|
105
|
+
toId: string,
|
|
106
|
+
edges: GraphEdge[],
|
|
107
|
+
nodeIds: Iterable<string>,
|
|
108
|
+
opts: { maxDepth?: number } = {},
|
|
109
|
+
): PathResult {
|
|
110
|
+
const maxDepth = Math.max(1, Math.min(opts.maxDepth ?? 4, 6));
|
|
111
|
+
if (fromId === toId) return { found: true, hops: [] };
|
|
112
|
+
const adj = buildAdjacency(edges, nodeIds);
|
|
113
|
+
if (!adj.has(fromId) || !adj.has(toId)) return { found: false, hops: [] };
|
|
114
|
+
|
|
115
|
+
const prev = new Map<string, GraphEdge>();
|
|
116
|
+
const visited = new Set<string>([fromId]);
|
|
117
|
+
let frontier = [fromId];
|
|
118
|
+
for (let depth = 1; depth <= maxDepth; depth++) {
|
|
119
|
+
if (frontier.length === 0) break;
|
|
120
|
+
const next: string[] = [];
|
|
121
|
+
for (const cur of frontier.sort((a, b) => a.localeCompare(b))) {
|
|
122
|
+
for (const e of [...(adj.get(cur) ?? [])].sort((a, b) => a.to.localeCompare(b.to))) {
|
|
123
|
+
if (visited.has(e.to)) continue;
|
|
124
|
+
visited.add(e.to);
|
|
125
|
+
prev.set(e.to, e);
|
|
126
|
+
if (e.to === toId) {
|
|
127
|
+
// Walk back, then reverse into travel order.
|
|
128
|
+
const hops: GraphEdge[] = [];
|
|
129
|
+
let at = toId;
|
|
130
|
+
while (at !== fromId) {
|
|
131
|
+
const edge = prev.get(at)!;
|
|
132
|
+
hops.unshift(edge);
|
|
133
|
+
at = edge.from;
|
|
134
|
+
}
|
|
135
|
+
return { found: true, hops };
|
|
136
|
+
}
|
|
137
|
+
next.push(e.to);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
frontier = next;
|
|
141
|
+
}
|
|
142
|
+
return { found: false, hops: [] };
|
|
143
|
+
}
|