@hraness/kb 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/DISCLOSURE +43 -0
  2. package/LICENSE +21 -0
  3. package/README.md +545 -0
  4. package/dist/agent-context.js +35 -0
  5. package/dist/agent-guide-audit.js +18 -0
  6. package/dist/attachments.js +19 -0
  7. package/dist/authoring.js +33 -0
  8. package/dist/benchmark.js +19 -0
  9. package/dist/capture.js +49 -0
  10. package/dist/cli.js +3646 -0
  11. package/dist/clip/acquire.js +57 -0
  12. package/dist/clip/args.js +21 -0
  13. package/dist/clip/bounded-byte-buffer.js +9 -0
  14. package/dist/clip/browser-profiles.js +17 -0
  15. package/dist/clip/bundle-reader.js +11 -0
  16. package/dist/clip/cli.js +27 -0
  17. package/dist/clip/cookies.js +21 -0
  18. package/dist/clip/defuddle-worker.js +84 -0
  19. package/dist/clip/doctor.js +32 -0
  20. package/dist/clip/jobs.js +784 -0
  21. package/dist/clip/network-proxy.js +9 -0
  22. package/dist/clip/network.js +28 -0
  23. package/dist/clip/persist.js +30 -0
  24. package/dist/clip/refresh.js +11 -0
  25. package/dist/clip/terminal.js +11 -0
  26. package/dist/evaluation-builder.js +12238 -0
  27. package/dist/evaluation-kb.js +26 -0
  28. package/dist/evaluation.js +35 -0
  29. package/dist/git.js +39 -0
  30. package/dist/graph.js +39 -0
  31. package/dist/index-01jj6rbv.js +1103 -0
  32. package/dist/index-06c9ctr6.js +738 -0
  33. package/dist/index-0kavxzqj.js +2480 -0
  34. package/dist/index-1gwbassd.js +743 -0
  35. package/dist/index-1n418kb9.js +827 -0
  36. package/dist/index-1vrd1rmn.js +56 -0
  37. package/dist/index-1xxnjn0d.js +171 -0
  38. package/dist/index-3rm7cz6h.js +373 -0
  39. package/dist/index-3v2z4f0q.js +328 -0
  40. package/dist/index-48pz4jpc.js +392 -0
  41. package/dist/index-4j3tt0c3.js +234 -0
  42. package/dist/index-5n05se68.js +307 -0
  43. package/dist/index-5vwpzb5a.js +764 -0
  44. package/dist/index-6g2pv9d2.js +370 -0
  45. package/dist/index-7fzc32gf.js +712 -0
  46. package/dist/index-84x0vjjp.js +432 -0
  47. package/dist/index-adx6khj5.js +488 -0
  48. package/dist/index-b88v3vtm.js +805 -0
  49. package/dist/index-cv6fh7z5.js +462 -0
  50. package/dist/index-cxfrakt7.js +1067 -0
  51. package/dist/index-d13v9ckt.js +150 -0
  52. package/dist/index-dycjbhw2.js +1943 -0
  53. package/dist/index-dyqwejk5.js +531 -0
  54. package/dist/index-e5fbsywq.js +658 -0
  55. package/dist/index-f984hw45.js +927 -0
  56. package/dist/index-gh719d91.js +63 -0
  57. package/dist/index-hgve9rh2.js +2046 -0
  58. package/dist/index-hya40gb2.js +474 -0
  59. package/dist/index-j4zgmzjr.js +152 -0
  60. package/dist/index-jsmvyyvf.js +1166 -0
  61. package/dist/index-kh0pwvdd.js +1974 -0
  62. package/dist/index-mqx4nd6v.js +119 -0
  63. package/dist/index-mxxxytys.js +1254 -0
  64. package/dist/index-n5dd7r0v.js +711 -0
  65. package/dist/index-npg9z1a4.js +431 -0
  66. package/dist/index-pj501bh1.js +140 -0
  67. package/dist/index-qry4vhxk.js +1041 -0
  68. package/dist/index-s2gw5aw9.js +199 -0
  69. package/dist/index-vvdj6kca.js +70 -0
  70. package/dist/index-w2zc0vwa.js +571 -0
  71. package/dist/index-x3fthpsc.js +585 -0
  72. package/dist/index-zr53sf63.js +44 -0
  73. package/dist/index-zxdy5pby.js +2728 -0
  74. package/dist/index-zzhgcwyt.js +966 -0
  75. package/dist/index.js +453 -0
  76. package/dist/navigation.js +13 -0
  77. package/dist/pdf.js +85 -0
  78. package/dist/percolate.js +24 -0
  79. package/dist/portfolio.js +77 -0
  80. package/dist/query.js +29 -0
  81. package/dist/repository-memory.js +60 -0
  82. package/dist/sdk.js +36 -0
  83. package/dist/search-rules.js +34 -0
  84. package/dist/search.js +22 -0
  85. package/dist/semantic.js +39 -0
  86. package/dist/source-inbox.js +17 -0
  87. package/dist/untrusted-content.js +32 -0
  88. package/dist/url-intelligence.js +74 -0
  89. package/dist/workflow.js +23 -0
  90. package/dist/workflows/decision-context.js +20 -0
  91. package/dist/workflows/explain-change.js +8 -0
  92. package/dist/workflows/index.js +28 -0
  93. package/dist/workflows/plan-radar.js +10 -0
  94. package/package.json +409 -0
  95. package/skills/kb/AGENTS.md +19 -0
  96. package/skills/kb/SKILL.md +82 -0
  97. package/skills/kb/agents/openai.yaml +4 -0
  98. package/skills/kb/references/pdf-review.md +51 -0
  99. package/skills/kb/references/percolate.md +144 -0
  100. package/skills/kb/references/plan-structure.md +105 -0
  101. package/skills/kb/references/plan.md +115 -0
  102. package/skills/kb/references/query.md +231 -0
  103. package/skills/kb/references/refresh.md +133 -0
  104. package/skills/kb/references/save-pdf.md +125 -0
  105. package/skills/kb/references/save-url.md +178 -0
  106. package/skills/kb/references/url-authentication.md +75 -0
  107. package/skills/kb/references/url-platforms.md +25 -0
  108. package/src/agent-context.ts +1241 -0
  109. package/src/agent-guide-audit.ts +750 -0
  110. package/src/attachments.ts +723 -0
  111. package/src/authoring.ts +1725 -0
  112. package/src/benchmark.ts +362 -0
  113. package/src/capture.ts +68 -0
  114. package/src/cli.ts +3660 -0
  115. package/src/clip/acquire.ts +1273 -0
  116. package/src/clip/archive-today.ts +402 -0
  117. package/src/clip/args.ts +423 -0
  118. package/src/clip/assets.ts +262 -0
  119. package/src/clip/bounded-byte-buffer.ts +75 -0
  120. package/src/clip/browser-profiles.ts +372 -0
  121. package/src/clip/bundle-reader.ts +608 -0
  122. package/src/clip/capture.ts +1054 -0
  123. package/src/clip/cli.ts +181 -0
  124. package/src/clip/cookies.ts +512 -0
  125. package/src/clip/doctor.ts +1110 -0
  126. package/src/clip/extract.ts +868 -0
  127. package/src/clip/jobs.ts +959 -0
  128. package/src/clip/lib.ts +861 -0
  129. package/src/clip/media.ts +1474 -0
  130. package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
  131. package/src/clip/metadata-search-tool/Cargo.toml +18 -0
  132. package/src/clip/metadata-search-tool/runner.ts +685 -0
  133. package/src/clip/metadata-search-tool/src/main.rs +389 -0
  134. package/src/clip/metadata-search.ts +687 -0
  135. package/src/clip/network-proxy.ts +627 -0
  136. package/src/clip/network.ts +950 -0
  137. package/src/clip/package-root.ts +54 -0
  138. package/src/clip/persist.ts +1472 -0
  139. package/src/clip/platforms.ts +1824 -0
  140. package/src/clip/refresh.ts +197 -0
  141. package/src/clip/structured.ts +432 -0
  142. package/src/clip/terminal.ts +197 -0
  143. package/src/clip/url-intelligence.ts +581 -0
  144. package/src/clip/url-metadata-backfill.ts +616 -0
  145. package/src/clip/url-metadata-cli.ts +251 -0
  146. package/src/clip/url-metadata.ts +1397 -0
  147. package/src/evaluation-analysis-v2.ts +1440 -0
  148. package/src/evaluation-builder.ts +2539 -0
  149. package/src/evaluation-corpus-authoring.ts +2052 -0
  150. package/src/evaluation-evidence.ts +1147 -0
  151. package/src/evaluation-execution-v2.ts +426 -0
  152. package/src/evaluation-implementation.ts +131 -0
  153. package/src/evaluation-kb-closure.ts +1430 -0
  154. package/src/evaluation-kb-runner-v2.ts +735 -0
  155. package/src/evaluation-kb-v2.ts +2567 -0
  156. package/src/evaluation-kb.ts +1018 -0
  157. package/src/evaluation-measurement-v2.ts +1462 -0
  158. package/src/evaluation-packing-v2.ts +452 -0
  159. package/src/evaluation-redaction.ts +28 -0
  160. package/src/evaluation-v2.ts +3857 -0
  161. package/src/evaluation.ts +1220 -0
  162. package/src/git.ts +1276 -0
  163. package/src/graph.ts +1487 -0
  164. package/src/index.ts +20 -0
  165. package/src/init.ts +123 -0
  166. package/src/navigation.ts +277 -0
  167. package/src/note-lock.ts +529 -0
  168. package/src/pdf/args.ts +202 -0
  169. package/src/pdf/capture.ts +298 -0
  170. package/src/pdf/cli.ts +159 -0
  171. package/src/pdf/extract.ts +303 -0
  172. package/src/pdf/layout.ts +618 -0
  173. package/src/pdf/markdown.ts +154 -0
  174. package/src/pdf/model.ts +234 -0
  175. package/src/pdf/ocr.ts +190 -0
  176. package/src/pdf/persist.ts +314 -0
  177. package/src/pdf/source.ts +135 -0
  178. package/src/pdf/tools.ts +146 -0
  179. package/src/pdf.ts +92 -0
  180. package/src/percolate.ts +877 -0
  181. package/src/portfolio-audit.ts +574 -0
  182. package/src/portfolio-identity.ts +162 -0
  183. package/src/portfolio-registry.ts +473 -0
  184. package/src/portfolio.ts +765 -0
  185. package/src/query.ts +646 -0
  186. package/src/repository-memory.ts +1150 -0
  187. package/src/sdk.ts +1452 -0
  188. package/src/search-rules.ts +712 -0
  189. package/src/search.ts +710 -0
  190. package/src/semantic-runtime.ts +1139 -0
  191. package/src/semantic.ts +2480 -0
  192. package/src/source-inbox.ts +232 -0
  193. package/src/untrusted-content.ts +401 -0
  194. package/src/url-intelligence.ts +104 -0
  195. package/src/vault.ts +636 -0
  196. package/src/workflow.ts +653 -0
  197. package/src/workflows/decision-context.ts +81 -0
  198. package/src/workflows/explain-change.ts +73 -0
  199. package/src/workflows/index.ts +3 -0
  200. package/src/workflows/plan-radar.ts +101 -0
@@ -0,0 +1,877 @@
1
+ import { createHash } from "node:crypto";
2
+ import { posix } from "node:path";
3
+
4
+ import {
5
+ lookupNote,
6
+ MAX_ANALYZED_NOTES,
7
+ MAX_MENTIONS,
8
+ type Note,
9
+ type VaultAnalysis,
10
+ } from "./graph.js";
11
+
12
+ export const DEFAULT_PERCOLATION_LIMIT = 100;
13
+ export const MAX_PERCOLATION_LIMIT = 1_000;
14
+ export const DEFAULT_PERCOLATION_MIN_SUPPORT = 2;
15
+ export const MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE = 100;
16
+ export const MAX_PERCOLATION_NOTES = MAX_ANALYZED_NOTES;
17
+ export const MAX_PERCOLATION_MENTION_PAIRS = 250_000;
18
+ export const MAX_PERCOLATION_MENTIONS = MAX_MENTIONS;
19
+ export const MAX_SCOPED_PERCOLATION_MENTION_PAIRS =
20
+ MAX_PERCOLATION_NOTES * 2;
21
+
22
+ const MAX_PERCOLATION_EVIDENCE = 250_000;
23
+ const MAX_PERCOLATION_PAIR_OBSERVATIONS = MAX_PERCOLATION_MENTION_PAIRS;
24
+
25
+ export type MissingConceptEvidence = {
26
+ readonly kind: "tag";
27
+ readonly note: string;
28
+ readonly path: string;
29
+ readonly tag: string;
30
+ };
31
+
32
+ export type SharedTagEvidence = {
33
+ readonly kind: "shared-tag";
34
+ readonly note: string;
35
+ readonly path: string;
36
+ readonly tag: string;
37
+ };
38
+
39
+ export type SharedConceptEvidence = {
40
+ readonly kind: "shared-concept";
41
+ readonly note: string;
42
+ readonly path: string;
43
+ readonly concept: string;
44
+ readonly conceptPath: string;
45
+ };
46
+
47
+ export type MentionEvidence = {
48
+ readonly kind: "mention";
49
+ readonly source: string;
50
+ readonly target: string;
51
+ readonly line: number;
52
+ readonly phrase: string;
53
+ };
54
+
55
+ export type RelationEvidence = {
56
+ readonly kind: "relation";
57
+ readonly source: string;
58
+ readonly target: string;
59
+ readonly predicate: string;
60
+ readonly line: number;
61
+ readonly authoredTarget: string;
62
+ };
63
+
64
+ export type RelationIssueEvidence = {
65
+ readonly kind: "relation-issue";
66
+ readonly issue: "malformed" | "broken" | "ambiguous";
67
+ readonly source: string;
68
+ readonly line: number;
69
+ readonly predicate: string | null;
70
+ readonly target: string | null;
71
+ readonly candidates: readonly string[];
72
+ readonly candidatesTruncated: boolean;
73
+ readonly message: string;
74
+ };
75
+
76
+ export type MissingConceptCandidate = {
77
+ readonly kind: "missing-concept";
78
+ readonly tag: string;
79
+ readonly suggestedId: string;
80
+ /** Existing non-concept occupying the natural ID, when one was found. */
81
+ readonly collidesWith: string | null;
82
+ readonly support: number;
83
+ readonly evidenceTruncated: boolean;
84
+ readonly evidence: readonly MissingConceptEvidence[];
85
+ };
86
+
87
+ export type MissingRelationCandidate = {
88
+ readonly kind: "missing-relation";
89
+ readonly source: string;
90
+ readonly target: string;
91
+ readonly suggestedPredicate: "related-to";
92
+ readonly support: number;
93
+ readonly evidenceTruncated: boolean;
94
+ readonly evidence: readonly (SharedTagEvidence | SharedConceptEvidence)[];
95
+ };
96
+
97
+ export type UnlinkedMentionCandidate = {
98
+ readonly kind: "unlinked-mention";
99
+ readonly source: string;
100
+ readonly target: string;
101
+ readonly support: number;
102
+ readonly evidenceTruncated: boolean;
103
+ readonly evidence: readonly MentionEvidence[];
104
+ };
105
+
106
+ export type RelationHygieneProblem =
107
+ | "self-relation"
108
+ | "reciprocal-relation"
109
+ | "malformed-relation"
110
+ | "broken-relation"
111
+ | "ambiguous-relation";
112
+
113
+ export type RelationHygieneCandidate = {
114
+ readonly kind: "relation-hygiene";
115
+ readonly problem: RelationHygieneProblem;
116
+ readonly source: string;
117
+ readonly target: string | null;
118
+ readonly predicate: string | null;
119
+ readonly message: string;
120
+ readonly support: number;
121
+ readonly evidenceTruncated: boolean;
122
+ readonly evidence: readonly (RelationEvidence | RelationIssueEvidence)[];
123
+ };
124
+
125
+ export type PercolationCandidate =
126
+ | MissingConceptCandidate
127
+ | MissingRelationCandidate
128
+ | UnlinkedMentionCandidate
129
+ | RelationHygieneCandidate;
130
+
131
+ export type PercolateOptions = {
132
+ /** Limit candidates to evidence involving this resolvable note. */
133
+ readonly note?: string;
134
+ /** Minimum evidence count for recurrence-based candidates. */
135
+ readonly minSupport?: number;
136
+ readonly limit?: number;
137
+ };
138
+
139
+ export type PercolationResult = {
140
+ readonly candidates: readonly PercolationCandidate[];
141
+ readonly truncated: boolean;
142
+ };
143
+
144
+ type AuthoredRelationLike = {
145
+ readonly source: string;
146
+ readonly target: string;
147
+ readonly predicate: string;
148
+ readonly provenance: {
149
+ readonly kind: "frontmatter";
150
+ readonly source: string;
151
+ readonly line: number;
152
+ readonly authoredTarget: string;
153
+ };
154
+ };
155
+
156
+ type RelationIssueLike =
157
+ | {
158
+ readonly kind: "malformed";
159
+ readonly source: string;
160
+ readonly line: number;
161
+ readonly predicate?: string;
162
+ readonly target?: string;
163
+ readonly message: string;
164
+ }
165
+ | {
166
+ readonly kind: "broken";
167
+ readonly source: string;
168
+ readonly line: number;
169
+ readonly predicate: string;
170
+ readonly target: string;
171
+ }
172
+ | {
173
+ readonly kind: "ambiguous";
174
+ readonly source: string;
175
+ readonly line: number;
176
+ readonly predicate: string;
177
+ readonly target: string;
178
+ readonly candidates: readonly string[];
179
+ };
180
+
181
+ type AnalysisWithRelations = VaultAnalysis & {
182
+ readonly authoredRelations?: readonly AuthoredRelationLike[];
183
+ readonly relationIssues?: readonly RelationIssueLike[];
184
+ };
185
+
186
+ type SharedEvidence = SharedTagEvidence | SharedConceptEvidence;
187
+
188
+ type SharedAccumulation = {
189
+ support: number;
190
+ evidenceCount: number;
191
+ readonly evidence: SharedEvidence[];
192
+ };
193
+
194
+ type MentionAccumulation = {
195
+ support: number;
196
+ readonly evidence: MentionEvidence[];
197
+ };
198
+
199
+ function compareText(left: string, right: string): number {
200
+ return left < right ? -1 : left > right ? 1 : 0;
201
+ }
202
+
203
+ function pairKey(left: string, right: string): string {
204
+ return compareText(left, right) <= 0
205
+ ? `${left}\u0000${right}`
206
+ : `${right}\u0000${left}`;
207
+ }
208
+
209
+ function directedKey(source: string, target: string): string {
210
+ return `${source}\u0000${target}`;
211
+ }
212
+
213
+ function relationKey(
214
+ source: string,
215
+ predicate: string,
216
+ target: string,
217
+ ): string {
218
+ return `${source}\u0000${predicate}\u0000${target}`;
219
+ }
220
+
221
+ function checkedLine(line: number, context: string): number {
222
+ if (!Number.isSafeInteger(line) || line < 1) {
223
+ throw new TypeError(`${context} has an invalid evidence line.`);
224
+ }
225
+ return line;
226
+ }
227
+
228
+ function checkedOptions(
229
+ options: PercolateOptions,
230
+ ): { readonly limit: number; readonly minSupport: number } {
231
+ const limit = options.limit ?? DEFAULT_PERCOLATION_LIMIT;
232
+ if (
233
+ !Number.isSafeInteger(limit)
234
+ || limit < 0
235
+ || limit > MAX_PERCOLATION_LIMIT
236
+ ) {
237
+ throw new RangeError(
238
+ `Percolation limit must be a safe integer from 0 to ${MAX_PERCOLATION_LIMIT}.`,
239
+ );
240
+ }
241
+ const minSupport = options.minSupport ?? DEFAULT_PERCOLATION_MIN_SUPPORT;
242
+ if (
243
+ !Number.isSafeInteger(minSupport)
244
+ || minSupport < 1
245
+ || minSupport > MAX_PERCOLATION_EVIDENCE
246
+ ) {
247
+ throw new RangeError(
248
+ `Percolation minimum support must be a safe integer from 1 to ${MAX_PERCOLATION_EVIDENCE}.`,
249
+ );
250
+ }
251
+ return { limit, minSupport };
252
+ }
253
+
254
+ function indexedContentNotes(
255
+ notes: readonly Note[],
256
+ analysis: VaultAnalysis,
257
+ ): {
258
+ readonly notes: readonly Note[];
259
+ readonly byId: ReadonlyMap<string, Note>;
260
+ readonly byPath: ReadonlyMap<string, Note>;
261
+ } {
262
+ if (analysis.noteConnections.length > MAX_PERCOLATION_NOTES) {
263
+ throw new RangeError(
264
+ `Percolation exceeds the ${MAX_PERCOLATION_NOTES} note limit.`,
265
+ );
266
+ }
267
+ const allById = new Map<string, Note>();
268
+ for (const note of notes) {
269
+ if (allById.has(note.id)) {
270
+ throw new Error(`Duplicate note identity in percolation: ${note.id}.`);
271
+ }
272
+ allById.set(note.id, note);
273
+ }
274
+
275
+ const byId = new Map<string, Note>();
276
+ const byPath = new Map<string, Note>();
277
+ for (const connection of analysis.noteConnections) {
278
+ if (byId.has(connection.id)) {
279
+ throw new Error(`Duplicate analyzed note identity: ${connection.id}.`);
280
+ }
281
+ const note = allById.get(connection.id);
282
+ if (note === undefined) {
283
+ throw new Error(`Analysis references missing note identity: ${connection.id}.`);
284
+ }
285
+ if (byPath.has(note.path)) {
286
+ throw new Error(`Duplicate note path in percolation: ${note.path}.`);
287
+ }
288
+ byId.set(note.id, note);
289
+ byPath.set(note.path, note);
290
+ }
291
+ return {
292
+ notes: [...byId.values()].toSorted((left, right) =>
293
+ compareText(left.id, right.id)),
294
+ byId,
295
+ byPath,
296
+ };
297
+ }
298
+
299
+ function conceptKey(value: string): string {
300
+ return value
301
+ .normalize("NFKC")
302
+ .toLocaleLowerCase("en-US")
303
+ .replace(/^#+/u, "")
304
+ .replace(/[^\p{Letter}\p{Number}]+/gu, "-")
305
+ .replace(/^-+|-+$/gu, "");
306
+ }
307
+
308
+ function isConcept(note: Note): boolean {
309
+ return note.properties.type
310
+ ?.normalize("NFC")
311
+ .toLocaleLowerCase("en-US") === "concept";
312
+ }
313
+
314
+ function conceptLabels(note: Note): readonly string[] {
315
+ return [
316
+ note.title,
317
+ ...note.aliases,
318
+ posix.basename(note.id),
319
+ ].map(conceptKey).filter((value) => value !== "");
320
+ }
321
+
322
+ function naturalConceptId(tag: string): string {
323
+ const slug = conceptKey(tag);
324
+ const digest = createHash("sha256").update(tag.normalize("NFC")).digest("hex");
325
+ if (slug === "") return `notes/concept-${digest.slice(0, 16)}`;
326
+ if (slug.length <= 160) return `notes/${slug}`;
327
+ let prefix = "";
328
+ let count = 0;
329
+ for (const character of slug) {
330
+ if (count >= 144) break;
331
+ prefix += character;
332
+ count += 1;
333
+ }
334
+ return `notes/${prefix.replace(/-+$/u, "")}-${digest.slice(0, 12)}`;
335
+ }
336
+
337
+ function suggestedConceptId(
338
+ tag: string,
339
+ occupiedIds: ReadonlyMap<string, string>,
340
+ ): { readonly id: string; readonly collidesWith: string | null } {
341
+ const natural = naturalConceptId(tag);
342
+ const foldedNatural = natural.toLocaleLowerCase("en-US");
343
+ const collidesWith = occupiedIds.get(foldedNatural) ?? null;
344
+ if (collidesWith === null) return { id: natural, collidesWith: null };
345
+
346
+ const suffixed = `${natural}-concept`;
347
+ if (!occupiedIds.has(suffixed.toLocaleLowerCase("en-US"))) {
348
+ return { id: suffixed, collidesWith };
349
+ }
350
+ for (let suffix = 2; suffix <= MAX_PERCOLATION_NOTES + 2; suffix += 1) {
351
+ const candidate = `${suffixed}-${suffix}`;
352
+ if (!occupiedIds.has(candidate.toLocaleLowerCase("en-US"))) {
353
+ return { id: candidate, collidesWith };
354
+ }
355
+ }
356
+ throw new RangeError("Percolation could not choose an unoccupied concept ID.");
357
+ }
358
+
359
+ function resolvedNoteFilter(
360
+ notes: readonly Note[],
361
+ query: string | undefined,
362
+ ): string | null {
363
+ if (query === undefined) return null;
364
+ const result = lookupNote(notes, query);
365
+ if (result.kind === "found") return result.note.id;
366
+ if (result.kind === "ambiguous") {
367
+ throw new Error(
368
+ `Percolation note is ambiguous: ${result.candidates.map((note) => note.id).join(", ")}.`,
369
+ );
370
+ }
371
+ throw new Error(`Percolation note does not exist: ${query}.`);
372
+ }
373
+
374
+ function candidateInvolvesNote(
375
+ candidate: PercolationCandidate,
376
+ note: string | null,
377
+ ): boolean {
378
+ if (note === null) return true;
379
+ if (candidate.kind === "missing-concept") {
380
+ return candidate.evidence.some((evidence) => evidence.note === note);
381
+ }
382
+ return candidate.source === note || candidate.target === note;
383
+ }
384
+
385
+ function compareSharedEvidence(
386
+ left: SharedEvidence,
387
+ right: SharedEvidence,
388
+ ): number {
389
+ return compareText(left.kind, right.kind)
390
+ || compareText(left.kind === "shared-tag" ? left.tag : left.concept,
391
+ right.kind === "shared-tag" ? right.tag : right.concept)
392
+ || compareText(left.note, right.note);
393
+ }
394
+
395
+ function candidateIdentity(candidate: PercolationCandidate): string {
396
+ switch (candidate.kind) {
397
+ case "missing-concept":
398
+ return candidate.tag;
399
+ case "missing-relation":
400
+ return `${candidate.source}\u0000${candidate.target}`;
401
+ case "unlinked-mention":
402
+ return `${candidate.source}\u0000${candidate.target}`;
403
+ case "relation-hygiene":
404
+ return [
405
+ candidate.problem,
406
+ candidate.source,
407
+ candidate.predicate ?? "",
408
+ candidate.target ?? "",
409
+ ].join("\u0000");
410
+ }
411
+ }
412
+
413
+ const candidateKindRank: Readonly<Record<PercolationCandidate["kind"], number>> = {
414
+ "relation-hygiene": 0,
415
+ "unlinked-mention": 1,
416
+ "missing-relation": 2,
417
+ "missing-concept": 3,
418
+ };
419
+
420
+ function compareCandidates(
421
+ left: PercolationCandidate,
422
+ right: PercolationCandidate,
423
+ ): number {
424
+ return right.support - left.support
425
+ || candidateKindRank[left.kind] - candidateKindRank[right.kind]
426
+ || compareText(candidateIdentity(left), candidateIdentity(right));
427
+ }
428
+
429
+ function relationEvidence(relation: AuthoredRelationLike): RelationEvidence {
430
+ return {
431
+ kind: "relation",
432
+ source: relation.source,
433
+ target: relation.target,
434
+ predicate: relation.predicate,
435
+ line: checkedLine(
436
+ relation.provenance.line,
437
+ `Authored relation ${relation.source} -> ${relation.target}`,
438
+ ),
439
+ authoredTarget: relation.provenance.authoredTarget,
440
+ };
441
+ }
442
+
443
+ function issueEvidence(
444
+ issue: RelationIssueLike,
445
+ source: string,
446
+ ): RelationIssueEvidence {
447
+ const line = checkedLine(issue.line, `Relation issue in ${issue.source}`);
448
+ if (issue.kind === "malformed") {
449
+ return {
450
+ kind: "relation-issue",
451
+ issue: issue.kind,
452
+ source,
453
+ line,
454
+ predicate: issue.predicate ?? null,
455
+ target: issue.target ?? null,
456
+ candidates: [],
457
+ candidatesTruncated: false,
458
+ message: issue.message,
459
+ };
460
+ }
461
+ if (issue.kind === "broken") {
462
+ return {
463
+ kind: "relation-issue",
464
+ issue: issue.kind,
465
+ source,
466
+ line,
467
+ predicate: issue.predicate,
468
+ target: issue.target,
469
+ candidates: [],
470
+ candidatesTruncated: false,
471
+ message: `Relationship target does not exist: ${issue.target}.`,
472
+ };
473
+ }
474
+ return {
475
+ kind: "relation-issue",
476
+ issue: issue.kind,
477
+ source,
478
+ line,
479
+ predicate: issue.predicate,
480
+ target: issue.target,
481
+ candidates: [...issue.candidates]
482
+ .toSorted(compareText)
483
+ .slice(0, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE),
484
+ candidatesTruncated:
485
+ issue.candidates.length > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
486
+ message: `Relationship target is ambiguous: ${issue.target}.`,
487
+ };
488
+ }
489
+
490
+ /**
491
+ * Derive bounded, evidence-backed graph maintenance candidates without
492
+ * changing notes, analysis, or any Markdown source.
493
+ */
494
+ export function percolateVault(
495
+ notes: readonly Note[],
496
+ analysis: VaultAnalysis,
497
+ options: PercolateOptions = {},
498
+ ): PercolationResult {
499
+ const { limit, minSupport } = checkedOptions(options);
500
+ const indexed = indexedContentNotes(notes, analysis);
501
+ const noteFilter = resolvedNoteFilter(indexed.notes, options.note);
502
+ const relations =
503
+ (analysis as AnalysisWithRelations).authoredRelations ?? [];
504
+ const relationIssues =
505
+ (analysis as AnalysisWithRelations).relationIssues ?? [];
506
+ let evidenceObservations =
507
+ analysis.mentions.length + relations.length + relationIssues.length;
508
+ for (const issue of relationIssues) {
509
+ if (issue.kind !== "ambiguous") continue;
510
+ evidenceObservations += issue.candidates.length;
511
+ if (evidenceObservations > MAX_PERCOLATION_EVIDENCE) break;
512
+ }
513
+ if (evidenceObservations > MAX_PERCOLATION_EVIDENCE) {
514
+ throw new RangeError(
515
+ `Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} evidence limit.`,
516
+ );
517
+ }
518
+
519
+ const conceptIds = new Set(
520
+ indexed.notes.filter(isConcept).map((note) => note.id),
521
+ );
522
+ const occupiedIds = new Map(indexed.notes.map((note) => [
523
+ note.id.toLocaleLowerCase("en-US"),
524
+ note.id,
525
+ ]));
526
+ const nonConceptNotes = indexed.notes.filter((note) => !conceptIds.has(note.id));
527
+ const conceptLabelKeys = new Set(
528
+ indexed.notes
529
+ .filter((note) => conceptIds.has(note.id))
530
+ .flatMap(conceptLabels),
531
+ );
532
+
533
+ const candidates: PercolationCandidate[] = [];
534
+ const notesByTag = new Map<string, Note[]>();
535
+ let tagEvidenceCount = 0;
536
+ for (const note of nonConceptNotes) {
537
+ for (const tag of new Set(note.tags)) {
538
+ tagEvidenceCount += 1;
539
+ if (tagEvidenceCount > MAX_PERCOLATION_EVIDENCE) {
540
+ throw new RangeError(
541
+ `Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} tag evidence limit.`,
542
+ );
543
+ }
544
+ const matches = notesByTag.get(tag) ?? [];
545
+ matches.push(note);
546
+ notesByTag.set(tag, matches);
547
+ }
548
+ }
549
+
550
+ for (const [tag, matchingNotes] of [...notesByTag].toSorted(([left], [right]) =>
551
+ compareText(left, right))) {
552
+ const sortedMatches = matchingNotes.toSorted((left, right) =>
553
+ (left.id === noteFilter ? -1 : 0)
554
+ - (right.id === noteFilter ? -1 : 0)
555
+ || compareText(left.id, right.id));
556
+ const evidence = sortedMatches
557
+ .slice(0, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE)
558
+ .map((note): MissingConceptEvidence => ({
559
+ kind: "tag",
560
+ note: note.id,
561
+ path: note.path,
562
+ tag,
563
+ }));
564
+ if (
565
+ matchingNotes.length >= Math.max(2, minSupport)
566
+ && !conceptLabelKeys.has(conceptKey(tag))
567
+ && (noteFilter === null || matchingNotes.some((note) => note.id === noteFilter))
568
+ ) {
569
+ const suggestion = suggestedConceptId(tag, occupiedIds);
570
+ candidates.push({
571
+ kind: "missing-concept",
572
+ tag,
573
+ suggestedId: suggestion.id,
574
+ collidesWith: suggestion.collidesWith,
575
+ support: matchingNotes.length,
576
+ evidenceTruncated:
577
+ matchingNotes.length > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
578
+ evidence,
579
+ });
580
+ }
581
+ }
582
+
583
+ const explicitPairs = new Set<string>();
584
+ const noteConcepts = new Map<string, Set<string>>();
585
+ const addConceptConnection = (noteId: string, conceptId: string): void => {
586
+ if (conceptIds.has(noteId) || !conceptIds.has(conceptId)) return;
587
+ const concepts = noteConcepts.get(noteId) ?? new Set<string>();
588
+ concepts.add(conceptId);
589
+ noteConcepts.set(noteId, concepts);
590
+ };
591
+ for (const link of analysis.contextualLinks) {
592
+ const source = indexed.byPath.get(link.source);
593
+ const target = indexed.byPath.get(link.target);
594
+ if (source === undefined || target === undefined) {
595
+ throw new Error(
596
+ `Contextual link references an unknown percolation note: ${link.source} -> ${link.target}.`,
597
+ );
598
+ }
599
+ explicitPairs.add(pairKey(source.id, target.id));
600
+ addConceptConnection(source.id, target.id);
601
+ addConceptConnection(target.id, source.id);
602
+ }
603
+ for (const relation of relations) {
604
+ if (!indexed.byId.has(relation.source) || !indexed.byId.has(relation.target)) {
605
+ throw new Error(
606
+ `Authored relation references an unknown percolation note: ${relation.source} -> ${relation.target}.`,
607
+ );
608
+ }
609
+ explicitPairs.add(pairKey(relation.source, relation.target));
610
+ addConceptConnection(relation.source, relation.target);
611
+ addConceptConnection(relation.target, relation.source);
612
+ }
613
+
614
+ const pairEvidence = new Map<string, SharedAccumulation>();
615
+ let pairObservations = 0;
616
+ const addPairEvidence = (
617
+ left: Note,
618
+ right: Note,
619
+ evidence: readonly SharedEvidence[],
620
+ ): void => {
621
+ pairObservations += 1;
622
+ if (pairObservations > MAX_PERCOLATION_PAIR_OBSERVATIONS) {
623
+ throw new RangeError(
624
+ `Percolation exceeds the ${MAX_PERCOLATION_PAIR_OBSERVATIONS} pair observation limit.`,
625
+ );
626
+ }
627
+ const key = pairKey(left.id, right.id);
628
+ if (explicitPairs.has(key)) return;
629
+ const accumulated = pairEvidence.get(key) ?? {
630
+ support: 0,
631
+ evidenceCount: 0,
632
+ evidence: [],
633
+ };
634
+ accumulated.support += 1;
635
+ accumulated.evidenceCount += evidence.length;
636
+ for (const item of evidence) {
637
+ if (
638
+ accumulated.evidence.length
639
+ < MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE
640
+ ) {
641
+ accumulated.evidence.push(item);
642
+ }
643
+ }
644
+ pairEvidence.set(key, accumulated);
645
+ };
646
+
647
+ const addPairsForGroup = (
648
+ matchingNotes: readonly Note[],
649
+ evidenceFor: (left: Note, right: Note) => readonly SharedEvidence[],
650
+ ): void => {
651
+ const sortedNotes = matchingNotes.toSorted((left, right) =>
652
+ compareText(left.id, right.id));
653
+ if (noteFilter !== null) {
654
+ const scoped = sortedNotes.find((note) => note.id === noteFilter);
655
+ if (scoped === undefined) return;
656
+ for (const other of sortedNotes) {
657
+ if (other.id === scoped.id) continue;
658
+ const [left, right] = compareText(scoped.id, other.id) < 0
659
+ ? [scoped, other]
660
+ : [other, scoped];
661
+ addPairEvidence(left, right, evidenceFor(left, right));
662
+ }
663
+ return;
664
+ }
665
+ for (let leftIndex = 0; leftIndex < sortedNotes.length; leftIndex += 1) {
666
+ for (
667
+ let rightIndex = leftIndex + 1;
668
+ rightIndex < sortedNotes.length;
669
+ rightIndex += 1
670
+ ) {
671
+ const left = sortedNotes[leftIndex];
672
+ const right = sortedNotes[rightIndex];
673
+ if (left === undefined || right === undefined) continue;
674
+ addPairEvidence(left, right, evidenceFor(left, right));
675
+ }
676
+ }
677
+ };
678
+
679
+ for (const [tag, matchingNotes] of [...notesByTag].toSorted(
680
+ ([left], [right]) => compareText(left, right),
681
+ )) {
682
+ addPairsForGroup(matchingNotes, (left, right) => [
683
+ { kind: "shared-tag", note: left.id, path: left.path, tag },
684
+ { kind: "shared-tag", note: right.id, path: right.path, tag },
685
+ ]);
686
+ }
687
+
688
+ const notesByConcept = new Map<string, Note[]>();
689
+ for (const [noteId, connectedConcepts] of noteConcepts) {
690
+ const note = indexed.byId.get(noteId);
691
+ if (note === undefined) continue;
692
+ for (const concept of connectedConcepts) {
693
+ const matches = notesByConcept.get(concept) ?? [];
694
+ matches.push(note);
695
+ notesByConcept.set(concept, matches);
696
+ }
697
+ }
698
+ for (const [concept, matchingNotes] of [...notesByConcept].toSorted(
699
+ ([left], [right]) => compareText(left, right),
700
+ )) {
701
+ const conceptNote = indexed.byId.get(concept);
702
+ if (conceptNote === undefined) continue;
703
+ addPairsForGroup(matchingNotes, (left, right) => [
704
+ {
705
+ kind: "shared-concept",
706
+ note: left.id,
707
+ path: left.path,
708
+ concept,
709
+ conceptPath: conceptNote.path,
710
+ },
711
+ {
712
+ kind: "shared-concept",
713
+ note: right.id,
714
+ path: right.path,
715
+ concept,
716
+ conceptPath: conceptNote.path,
717
+ },
718
+ ]);
719
+ }
720
+
721
+ for (const [key, accumulated] of pairEvidence) {
722
+ const separator = key.indexOf("\u0000");
723
+ const source = key.slice(0, separator);
724
+ const target = key.slice(separator + 1);
725
+ const evidence = accumulated.evidence.toSorted(compareSharedEvidence);
726
+ if (accumulated.support < minSupport) continue;
727
+ candidates.push({
728
+ kind: "missing-relation",
729
+ source,
730
+ target,
731
+ suggestedPredicate: "related-to",
732
+ support: accumulated.support,
733
+ evidenceTruncated:
734
+ accumulated.evidenceCount > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
735
+ evidence,
736
+ });
737
+ }
738
+
739
+ const mentionEvidenceByPair = new Map<string, MentionAccumulation>();
740
+ for (const mention of analysis.mentions) {
741
+ const source = indexed.byPath.get(mention.source);
742
+ const target = indexed.byPath.get(mention.target);
743
+ if (source === undefined || target === undefined) {
744
+ throw new Error(
745
+ `Mention references an unknown percolation note: ${mention.source} -> ${mention.target}.`,
746
+ );
747
+ }
748
+ if (
749
+ noteFilter !== null
750
+ && source.id !== noteFilter
751
+ && target.id !== noteFilter
752
+ ) continue;
753
+ if (explicitPairs.has(pairKey(source.id, target.id))) continue;
754
+ const key = directedKey(source.id, target.id);
755
+ const accumulated = mentionEvidenceByPair.get(key) ?? {
756
+ support: 0,
757
+ evidence: [],
758
+ };
759
+ accumulated.support += 1;
760
+ if (
761
+ accumulated.evidence.length
762
+ < MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE
763
+ ) accumulated.evidence.push({
764
+ kind: "mention",
765
+ source: source.id,
766
+ target: target.id,
767
+ line: checkedLine(
768
+ mention.line,
769
+ `Mention ${mention.source} -> ${mention.target}`,
770
+ ),
771
+ phrase: mention.phrase,
772
+ });
773
+ mentionEvidenceByPair.set(key, accumulated);
774
+ }
775
+ for (const [key, accumulated] of mentionEvidenceByPair) {
776
+ const separator = key.indexOf("\u0000");
777
+ const source = key.slice(0, separator);
778
+ const target = key.slice(separator + 1);
779
+ const sortedEvidence = accumulated.evidence.toSorted((left, right) =>
780
+ left.line - right.line || compareText(left.phrase, right.phrase));
781
+ candidates.push({
782
+ kind: "unlinked-mention",
783
+ source,
784
+ target,
785
+ support: accumulated.support,
786
+ evidenceTruncated:
787
+ accumulated.support > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
788
+ evidence: sortedEvidence,
789
+ });
790
+ }
791
+
792
+ const relationsByKey = new Map(
793
+ relations.map((relation) => [
794
+ relationKey(relation.source, relation.predicate, relation.target),
795
+ relation,
796
+ ]),
797
+ );
798
+ for (const relation of relations) {
799
+ if (
800
+ noteFilter !== null
801
+ && relation.source !== noteFilter
802
+ && relation.target !== noteFilter
803
+ ) continue;
804
+ if (relation.source === relation.target) {
805
+ candidates.push({
806
+ kind: "relation-hygiene",
807
+ problem: "self-relation",
808
+ source: relation.source,
809
+ target: relation.target,
810
+ predicate: relation.predicate,
811
+ message: "Review an authored relationship whose source and target are the same note.",
812
+ support: 1,
813
+ evidenceTruncated: false,
814
+ evidence: [relationEvidence(relation)],
815
+ });
816
+ continue;
817
+ }
818
+ if (compareText(relation.source, relation.target) >= 0) continue;
819
+ const reciprocal = relationsByKey.get(
820
+ relationKey(relation.target, relation.predicate, relation.source),
821
+ );
822
+ if (reciprocal === undefined) continue;
823
+ const evidence = [
824
+ relationEvidence(relation),
825
+ relationEvidence(reciprocal),
826
+ ].toSorted((left, right) =>
827
+ compareText(left.source, right.source)
828
+ || compareText(left.target, right.target));
829
+ candidates.push({
830
+ kind: "relation-hygiene",
831
+ problem: "reciprocal-relation",
832
+ source: relation.source,
833
+ target: relation.target,
834
+ predicate: relation.predicate,
835
+ message: "Review reciprocal assertions of the same directional predicate.",
836
+ support: evidence.length,
837
+ evidenceTruncated: false,
838
+ evidence,
839
+ });
840
+ }
841
+
842
+ for (const issue of relationIssues) {
843
+ const sourceNote = indexed.byPath.get(issue.source);
844
+ if (sourceNote === undefined) continue;
845
+ if (noteFilter !== null && sourceNote.id !== noteFilter) continue;
846
+ const evidence = issueEvidence(issue, sourceNote.id);
847
+ const problem: RelationHygieneProblem = issue.kind === "malformed"
848
+ ? "malformed-relation"
849
+ : issue.kind === "broken"
850
+ ? "broken-relation"
851
+ : "ambiguous-relation";
852
+ candidates.push({
853
+ kind: "relation-hygiene",
854
+ problem,
855
+ source: sourceNote.id,
856
+ target: evidence.target,
857
+ predicate: evidence.predicate,
858
+ message: evidence.message,
859
+ support: 1,
860
+ evidenceTruncated: evidence.candidatesTruncated,
861
+ evidence: [evidence],
862
+ });
863
+ }
864
+
865
+ if (candidates.length > MAX_PERCOLATION_EVIDENCE) {
866
+ throw new RangeError(
867
+ `Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} candidate limit.`,
868
+ );
869
+ }
870
+ const sorted = candidates
871
+ .filter((candidate) => candidateInvolvesNote(candidate, noteFilter))
872
+ .toSorted(compareCandidates);
873
+ return {
874
+ candidates: sorted.slice(0, limit),
875
+ truncated: sorted.length > limit,
876
+ };
877
+ }