@hraness/wordcell 0.20.0-bootstrap.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +749 -0
- package/dist/agent-context.js +35 -0
- package/dist/agent-guide-audit.js +18 -0
- package/dist/attachments.js +19 -0
- package/dist/authoring.js +33 -0
- package/dist/benchmark.js +19 -0
- package/dist/capture.js +49 -0
- package/dist/cli.js +57 -0
- package/dist/clip/acquire.js +57 -0
- package/dist/clip/args.js +21 -0
- package/dist/clip/bounded-byte-buffer.js +9 -0
- package/dist/clip/browser-profiles.js +17 -0
- package/dist/clip/bundle-reader.js +11 -0
- package/dist/clip/cli.js +27 -0
- package/dist/clip/cookies.js +21 -0
- package/dist/clip/defuddle-worker.js +84 -0
- package/dist/clip/doctor.js +32 -0
- package/dist/clip/jobs.js +784 -0
- package/dist/clip/network-proxy.js +9 -0
- package/dist/clip/network.js +28 -0
- package/dist/clip/persist.js +30 -0
- package/dist/clip/refresh.js +11 -0
- package/dist/clip/terminal.js +11 -0
- package/dist/evaluation-builder.js +12238 -0
- package/dist/evaluation-kb.js +26 -0
- package/dist/evaluation.js +35 -0
- package/dist/git.js +39 -0
- package/dist/graph.js +41 -0
- package/dist/index-054mb7d3.js +1974 -0
- package/dist/index-06c9ctr6.js +738 -0
- package/dist/index-180zbkkt.js +2480 -0
- package/dist/index-1gwbassd.js +743 -0
- package/dist/index-1xxnjn0d.js +171 -0
- package/dist/index-23z4zxgg.js +119 -0
- package/dist/index-2gv8y733.js +432 -0
- package/dist/index-3ehtt59c.js +199 -0
- package/dist/index-3rm7cz6h.js +373 -0
- package/dist/index-48pz4jpc.js +392 -0
- package/dist/index-4j3tt0c3.js +234 -0
- package/dist/index-5n05se68.js +307 -0
- package/dist/index-5vwpzb5a.js +764 -0
- package/dist/index-6jcz0m1c.js +712 -0
- package/dist/index-6sw24nvv.js +1189 -0
- package/dist/index-8a08v6x8.js +56 -0
- package/dist/index-adx6khj5.js +488 -0
- package/dist/index-b88v3vtm.js +805 -0
- package/dist/index-bcknqxrq.js +1943 -0
- package/dist/index-bs0yf4pp.js +711 -0
- package/dist/index-by0pbant.js +827 -0
- package/dist/index-d13v9ckt.js +150 -0
- package/dist/index-dfag79p7.js +370 -0
- package/dist/index-e4fagfjz.js +44 -0
- package/dist/index-e5fbsywq.js +658 -0
- package/dist/index-ekpwvbra.js +1070 -0
- package/dist/index-f984hw45.js +927 -0
- package/dist/index-gh719d91.js +63 -0
- package/dist/index-h170byqw.js +371 -0
- package/dist/index-hgve9rh2.js +2046 -0
- package/dist/index-hya40gb2.js +474 -0
- package/dist/index-j4zgmzjr.js +152 -0
- package/dist/index-m7et4rz4.js +966 -0
- package/dist/index-mxxxytys.js +1254 -0
- package/dist/index-n05s3wsb.js +2728 -0
- package/dist/index-nd6nynv2.js +1162 -0
- package/dist/index-npg9z1a4.js +431 -0
- package/dist/index-pj501bh1.js +140 -0
- package/dist/index-rdd5f7jj.js +70 -0
- package/dist/index-sy0zety7.js +3630 -0
- package/dist/index-tkpxp48t.js +1166 -0
- package/dist/index-w2zc0vwa.js +571 -0
- package/dist/index-why54q5p.js +462 -0
- package/dist/index-x3fthpsc.js +585 -0
- package/dist/index-x6m57nfc.js +1041 -0
- package/dist/index.js +784 -0
- package/dist/kb-alias.js +55 -0
- package/dist/navigation.js +13 -0
- package/dist/pdf.js +85 -0
- package/dist/percolate.js +44 -0
- package/dist/portfolio.js +77 -0
- package/dist/query.js +29 -0
- package/dist/repository-memory.js +60 -0
- package/dist/sdk.js +36 -0
- package/dist/search-rules.js +34 -0
- package/dist/search.js +22 -0
- package/dist/semantic.js +39 -0
- package/dist/source-inbox.js +17 -0
- package/dist/untrusted-content.js +32 -0
- package/dist/url-intelligence.js +74 -0
- package/dist/workflow.js +23 -0
- package/dist/workflows/decision-context.js +20 -0
- package/dist/workflows/explain-change.js +8 -0
- package/dist/workflows/index.js +28 -0
- package/dist/workflows/plan-radar.js +10 -0
- package/package.json +421 -0
- package/skills/wordcell/AGENTS.md +22 -0
- package/skills/wordcell/SKILL.md +91 -0
- package/skills/wordcell/agents/openai.yaml +4 -0
- package/skills/wordcell/references/companion-skills.md +96 -0
- package/skills/wordcell/references/customize.md +123 -0
- package/skills/wordcell/references/pdf-review.md +51 -0
- package/skills/wordcell/references/percolate.md +176 -0
- package/skills/wordcell/references/plan-structure.md +105 -0
- package/skills/wordcell/references/plan.md +115 -0
- package/skills/wordcell/references/query.md +252 -0
- package/skills/wordcell/references/refresh.md +133 -0
- package/skills/wordcell/references/save-pdf.md +125 -0
- package/skills/wordcell/references/save-url.md +178 -0
- package/skills/wordcell/references/url-authentication.md +75 -0
- package/skills/wordcell/references/url-platforms.md +25 -0
- package/skills/wordcell/templates/companion-skill.template.md +57 -0
- package/src/agent-context.ts +1241 -0
- package/src/agent-guide-audit.ts +750 -0
- package/src/attachments.ts +723 -0
- package/src/authoring-model.ts +798 -0
- package/src/authoring-platform.ts +549 -0
- package/src/authoring-program.ts +278 -0
- package/src/authoring-runtime.ts +54 -0
- package/src/authoring.ts +75 -0
- package/src/benchmark.ts +362 -0
- package/src/capture.ts +68 -0
- package/src/cli-program.ts +3662 -0
- package/src/cli.ts +6 -0
- package/src/clip/acquire.ts +1273 -0
- package/src/clip/archive-today.ts +402 -0
- package/src/clip/args.ts +423 -0
- package/src/clip/assets.ts +262 -0
- package/src/clip/bounded-byte-buffer.ts +75 -0
- package/src/clip/browser-profiles.ts +372 -0
- package/src/clip/bundle-reader.ts +608 -0
- package/src/clip/capture.ts +1054 -0
- package/src/clip/cli.ts +181 -0
- package/src/clip/cookies.ts +512 -0
- package/src/clip/doctor.ts +1110 -0
- package/src/clip/extract.ts +868 -0
- package/src/clip/jobs.ts +959 -0
- package/src/clip/lib.ts +861 -0
- package/src/clip/media.ts +1474 -0
- package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
- package/src/clip/metadata-search-tool/Cargo.toml +18 -0
- package/src/clip/metadata-search-tool/runner.ts +685 -0
- package/src/clip/metadata-search-tool/src/main.rs +389 -0
- package/src/clip/metadata-search.ts +687 -0
- package/src/clip/network-proxy.ts +627 -0
- package/src/clip/network.ts +950 -0
- package/src/clip/package-root.ts +54 -0
- package/src/clip/persist.ts +1472 -0
- package/src/clip/platforms.ts +1824 -0
- package/src/clip/refresh.ts +197 -0
- package/src/clip/structured.ts +432 -0
- package/src/clip/terminal.ts +197 -0
- package/src/clip/url-intelligence.ts +581 -0
- package/src/clip/url-metadata-backfill.ts +616 -0
- package/src/clip/url-metadata-cli.ts +251 -0
- package/src/clip/url-metadata.ts +1397 -0
- package/src/evaluation-analysis-v2.ts +1440 -0
- package/src/evaluation-builder.ts +2539 -0
- package/src/evaluation-corpus-authoring.ts +2052 -0
- package/src/evaluation-evidence.ts +1147 -0
- package/src/evaluation-execution-v2.ts +426 -0
- package/src/evaluation-implementation.ts +131 -0
- package/src/evaluation-kb-closure.ts +1430 -0
- package/src/evaluation-kb-runner-v2.ts +735 -0
- package/src/evaluation-kb-v2.ts +2567 -0
- package/src/evaluation-kb.ts +1018 -0
- package/src/evaluation-measurement-v2.ts +1462 -0
- package/src/evaluation-packing-v2.ts +452 -0
- package/src/evaluation-redaction.ts +28 -0
- package/src/evaluation-v2.ts +3857 -0
- package/src/evaluation.ts +1220 -0
- package/src/git.ts +1276 -0
- package/src/graph.ts +1494 -0
- package/src/index.ts +21 -0
- package/src/init.ts +123 -0
- package/src/kb-alias.ts +14 -0
- package/src/navigation.ts +277 -0
- package/src/note-lock.ts +529 -0
- package/src/oh-adoption.ts +444 -0
- package/src/pdf/args.ts +202 -0
- package/src/pdf/capture.ts +298 -0
- package/src/pdf/cli.ts +159 -0
- package/src/pdf/extract.ts +303 -0
- package/src/pdf/layout.ts +618 -0
- package/src/pdf/markdown.ts +154 -0
- package/src/pdf/model.ts +234 -0
- package/src/pdf/ocr.ts +190 -0
- package/src/pdf/persist.ts +314 -0
- package/src/pdf/source.ts +135 -0
- package/src/pdf/tools.ts +146 -0
- package/src/pdf.ts +92 -0
- package/src/percolate.ts +1948 -0
- package/src/portfolio-audit.ts +574 -0
- package/src/portfolio-identity.ts +162 -0
- package/src/portfolio-registry.ts +473 -0
- package/src/portfolio.ts +765 -0
- package/src/query.ts +646 -0
- package/src/repository-memory.ts +1150 -0
- package/src/sdk.ts +1452 -0
- package/src/search-rules.ts +712 -0
- package/src/search.ts +710 -0
- package/src/semantic-runtime.ts +1139 -0
- package/src/semantic.ts +2480 -0
- package/src/source-inbox.ts +232 -0
- package/src/untrusted-content.ts +401 -0
- package/src/url-intelligence.ts +104 -0
- package/src/vault.ts +636 -0
- package/src/workflow-model.ts +444 -0
- package/src/workflow-platform.ts +74 -0
- package/src/workflow-program.ts +183 -0
- package/src/workflow-runtime.ts +51 -0
- package/src/workflow.ts +23 -0
- package/src/workflows/decision-context.ts +81 -0
- package/src/workflows/explain-change.ts +73 -0
- package/src/workflows/index.ts +3 -0
- package/src/workflows/plan-radar.ts +101 -0
package/src/percolate.ts
ADDED
|
@@ -0,0 +1,1948 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { posix } from "node:path";
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
isCanonicalNoteId,
|
|
6
|
+
isCanonicalRelationPredicate,
|
|
7
|
+
lookupNote,
|
|
8
|
+
MAX_ANALYZED_NOTES,
|
|
9
|
+
MAX_MENTIONS,
|
|
10
|
+
type Note,
|
|
11
|
+
type VaultAnalysis,
|
|
12
|
+
} from "./graph.js";
|
|
13
|
+
|
|
14
|
+
export const DEFAULT_PERCOLATION_LIMIT = 100;
|
|
15
|
+
export const MAX_PERCOLATION_LIMIT = 1_000;
|
|
16
|
+
export const DEFAULT_PERCOLATION_MIN_SUPPORT = 2;
|
|
17
|
+
export const MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE = 100;
|
|
18
|
+
export const MAX_PERCOLATION_NOTES = MAX_ANALYZED_NOTES;
|
|
19
|
+
export const MAX_PERCOLATION_MENTION_PAIRS = 250_000;
|
|
20
|
+
export const MAX_PERCOLATION_MENTIONS = MAX_MENTIONS;
|
|
21
|
+
export const MAX_SCOPED_PERCOLATION_MENTION_PAIRS =
|
|
22
|
+
MAX_PERCOLATION_NOTES * 2;
|
|
23
|
+
export const PERCOLATION_RESULT_SCHEMA_VERSION = 2 as const;
|
|
24
|
+
export const MAX_PERCOLATION_RESULT_NODES = 250_000;
|
|
25
|
+
export const MAX_PERCOLATION_RESULT_UTF8_BYTES = 16 * 1024 * 1024;
|
|
26
|
+
export const MAX_PERCOLATION_TEXT_UTF8_BYTES = 64 * 1024;
|
|
27
|
+
|
|
28
|
+
const MAX_PERCOLATION_EVIDENCE = 250_000;
|
|
29
|
+
const MAX_PERCOLATION_PAIR_OBSERVATIONS = MAX_PERCOLATION_MENTION_PAIRS;
|
|
30
|
+
|
|
31
|
+
export type MissingConceptEvidence = {
|
|
32
|
+
readonly kind: "tag";
|
|
33
|
+
readonly note: string;
|
|
34
|
+
readonly path: string;
|
|
35
|
+
readonly tag: string;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export type SharedTagEvidence = {
|
|
39
|
+
readonly kind: "shared-tag";
|
|
40
|
+
readonly note: string;
|
|
41
|
+
readonly path: string;
|
|
42
|
+
readonly tag: string;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export type SharedConceptEvidence = {
|
|
46
|
+
readonly kind: "shared-concept";
|
|
47
|
+
readonly note: string;
|
|
48
|
+
readonly path: string;
|
|
49
|
+
readonly concept: string;
|
|
50
|
+
readonly conceptPath: string;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
export type MentionEvidence = {
|
|
54
|
+
readonly kind: "mention";
|
|
55
|
+
readonly source: string;
|
|
56
|
+
readonly target: string;
|
|
57
|
+
readonly line: number;
|
|
58
|
+
readonly phrase: string;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
export type RelationEvidence = {
|
|
62
|
+
readonly kind: "relation";
|
|
63
|
+
readonly source: string;
|
|
64
|
+
readonly target: string;
|
|
65
|
+
readonly predicate: string;
|
|
66
|
+
readonly line: number;
|
|
67
|
+
readonly authoredTarget: string;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
export type RelationIssueEvidence = {
|
|
71
|
+
readonly kind: "relation-issue";
|
|
72
|
+
readonly issue: "malformed" | "broken" | "ambiguous";
|
|
73
|
+
readonly source: string;
|
|
74
|
+
readonly line: number;
|
|
75
|
+
readonly predicate: string | null;
|
|
76
|
+
readonly target: string | null;
|
|
77
|
+
readonly candidates: readonly string[];
|
|
78
|
+
readonly candidatesTruncated: boolean;
|
|
79
|
+
readonly message: string;
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
export type MissingConceptCandidate = {
|
|
83
|
+
readonly kind: "missing-concept";
|
|
84
|
+
readonly tag: string;
|
|
85
|
+
readonly suggestedId: string;
|
|
86
|
+
/** Existing non-concept occupying the natural ID, when one was found. */
|
|
87
|
+
readonly collidesWith: string | null;
|
|
88
|
+
readonly support: number;
|
|
89
|
+
readonly evidenceTruncated: boolean;
|
|
90
|
+
readonly evidence: readonly MissingConceptEvidence[];
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
export type PredicateDisposition = { readonly kind: "required" };
|
|
94
|
+
|
|
95
|
+
/** @deprecated Archival V1 shape; retained through the 0.19.x compatibility cycle. */
|
|
96
|
+
export type MissingRelationCandidateV1 = {
|
|
97
|
+
readonly kind: "missing-relation";
|
|
98
|
+
readonly source: string;
|
|
99
|
+
readonly target: string;
|
|
100
|
+
readonly suggestedPredicate: "related-to";
|
|
101
|
+
readonly support: number;
|
|
102
|
+
readonly evidenceTruncated: boolean;
|
|
103
|
+
readonly evidence: readonly (SharedTagEvidence | SharedConceptEvidence)[];
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
export type MissingRelationCandidateV2 = {
|
|
107
|
+
readonly kind: "missing-relation";
|
|
108
|
+
/** Lexicographically ordered endpoint; this is not semantic direction. */
|
|
109
|
+
readonly source: string;
|
|
110
|
+
/** Lexicographically ordered endpoint; this is not semantic direction. */
|
|
111
|
+
readonly target: string;
|
|
112
|
+
readonly predicate: PredicateDisposition;
|
|
113
|
+
readonly support: number;
|
|
114
|
+
readonly evidenceTruncated: boolean;
|
|
115
|
+
readonly evidence: readonly (SharedTagEvidence | SharedConceptEvidence)[];
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
export type MissingRelationCandidate = MissingRelationCandidateV2;
|
|
119
|
+
|
|
120
|
+
export type UnlinkedMentionCandidate = {
|
|
121
|
+
readonly kind: "unlinked-mention";
|
|
122
|
+
readonly source: string;
|
|
123
|
+
readonly target: string;
|
|
124
|
+
readonly support: number;
|
|
125
|
+
readonly evidenceTruncated: boolean;
|
|
126
|
+
readonly evidence: readonly MentionEvidence[];
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
export type RelationHygieneProblem =
|
|
130
|
+
| "self-relation"
|
|
131
|
+
| "reciprocal-relation"
|
|
132
|
+
| "malformed-relation"
|
|
133
|
+
| "broken-relation"
|
|
134
|
+
| "ambiguous-relation";
|
|
135
|
+
|
|
136
|
+
export type RelationHygieneCandidate = {
|
|
137
|
+
readonly kind: "relation-hygiene";
|
|
138
|
+
readonly problem: RelationHygieneProblem;
|
|
139
|
+
readonly source: string;
|
|
140
|
+
readonly target: string | null;
|
|
141
|
+
readonly predicate: string | null;
|
|
142
|
+
readonly message: string;
|
|
143
|
+
readonly support: number;
|
|
144
|
+
readonly evidenceTruncated: boolean;
|
|
145
|
+
readonly evidence: readonly (RelationEvidence | RelationIssueEvidence)[];
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
export type PercolationCandidateV1 =
|
|
149
|
+
| MissingConceptCandidate
|
|
150
|
+
| MissingRelationCandidateV1
|
|
151
|
+
| UnlinkedMentionCandidate
|
|
152
|
+
| RelationHygieneCandidate;
|
|
153
|
+
|
|
154
|
+
export type PercolationCandidateV2 =
|
|
155
|
+
| MissingConceptCandidate
|
|
156
|
+
| MissingRelationCandidateV2
|
|
157
|
+
| UnlinkedMentionCandidate
|
|
158
|
+
| RelationHygieneCandidate;
|
|
159
|
+
|
|
160
|
+
export type PercolationCandidate = PercolationCandidateV2;
|
|
161
|
+
|
|
162
|
+
export type PercolateOptions = {
|
|
163
|
+
/** Limit candidates to evidence involving this resolvable note. */
|
|
164
|
+
readonly note?: string;
|
|
165
|
+
/** Minimum evidence count for recurrence-based candidates. */
|
|
166
|
+
readonly minSupport?: number;
|
|
167
|
+
readonly limit?: number;
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* @deprecated Historical unversioned result retained for explicit archival
|
|
172
|
+
* parsing through 0.19.x; it may be removed no earlier than 0.20.0.
|
|
173
|
+
*/
|
|
174
|
+
export type PercolationResultV1 = {
|
|
175
|
+
readonly candidates: readonly PercolationCandidateV1[];
|
|
176
|
+
readonly truncated: boolean;
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
export type PercolationResultV2 = {
|
|
180
|
+
readonly schemaVersion: typeof PERCOLATION_RESULT_SCHEMA_VERSION;
|
|
181
|
+
readonly candidates: readonly PercolationCandidateV2[];
|
|
182
|
+
readonly truncated: boolean;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
export type PercolationResult = PercolationResultV2;
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* @deprecated Historical JSON envelope emitted by `wordcell percolate --json`,
|
|
189
|
+
* retained through 0.19.x and removable no earlier than 0.20.0.
|
|
190
|
+
*/
|
|
191
|
+
export type PercolationCliOutputV1 = {
|
|
192
|
+
readonly root: string;
|
|
193
|
+
readonly note: string | null;
|
|
194
|
+
readonly minSupport: number;
|
|
195
|
+
readonly candidates: readonly PercolationCandidateV1[];
|
|
196
|
+
readonly truncated: boolean;
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
export type PercolationCliOutputV2 = {
|
|
200
|
+
readonly root: string;
|
|
201
|
+
/** The caller's free-form lookup text, not a canonical note identity. */
|
|
202
|
+
readonly note: string | null;
|
|
203
|
+
readonly minSupport: number;
|
|
204
|
+
readonly limit: number;
|
|
205
|
+
readonly schemaVersion: typeof PERCOLATION_RESULT_SCHEMA_VERSION;
|
|
206
|
+
readonly candidates: readonly PercolationCandidateV2[];
|
|
207
|
+
readonly truncated: boolean;
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
type AuthoredRelationLike = {
|
|
211
|
+
readonly source: string;
|
|
212
|
+
readonly target: string;
|
|
213
|
+
readonly predicate: string;
|
|
214
|
+
readonly provenance: {
|
|
215
|
+
readonly kind: "frontmatter";
|
|
216
|
+
readonly source: string;
|
|
217
|
+
readonly line: number;
|
|
218
|
+
readonly authoredTarget: string;
|
|
219
|
+
};
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
type RelationIssueLike =
|
|
223
|
+
| {
|
|
224
|
+
readonly kind: "malformed";
|
|
225
|
+
readonly source: string;
|
|
226
|
+
readonly line: number;
|
|
227
|
+
readonly predicate?: string;
|
|
228
|
+
readonly target?: string;
|
|
229
|
+
readonly message: string;
|
|
230
|
+
}
|
|
231
|
+
| {
|
|
232
|
+
readonly kind: "broken";
|
|
233
|
+
readonly source: string;
|
|
234
|
+
readonly line: number;
|
|
235
|
+
readonly predicate: string;
|
|
236
|
+
readonly target: string;
|
|
237
|
+
}
|
|
238
|
+
| {
|
|
239
|
+
readonly kind: "ambiguous";
|
|
240
|
+
readonly source: string;
|
|
241
|
+
readonly line: number;
|
|
242
|
+
readonly predicate: string;
|
|
243
|
+
readonly target: string;
|
|
244
|
+
readonly candidates: readonly string[];
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
type AnalysisWithRelations = VaultAnalysis & {
|
|
248
|
+
readonly authoredRelations?: readonly AuthoredRelationLike[];
|
|
249
|
+
readonly relationIssues?: readonly RelationIssueLike[];
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
type SharedEvidence = SharedTagEvidence | SharedConceptEvidence;
|
|
253
|
+
type AnyPercolationCandidate = PercolationCandidateV1 | PercolationCandidateV2;
|
|
254
|
+
|
|
255
|
+
type SharedAccumulation = {
|
|
256
|
+
support: number;
|
|
257
|
+
evidenceCount: number;
|
|
258
|
+
readonly evidence: SharedEvidence[];
|
|
259
|
+
};
|
|
260
|
+
|
|
261
|
+
type MentionAccumulation = {
|
|
262
|
+
support: number;
|
|
263
|
+
readonly evidence: MentionEvidence[];
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
function compareText(left: string, right: string): number {
|
|
267
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function pairKey(left: string, right: string): string {
|
|
271
|
+
return compareText(left, right) <= 0
|
|
272
|
+
? `${left}\u0000${right}`
|
|
273
|
+
: `${right}\u0000${left}`;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function directedKey(source: string, target: string): string {
|
|
277
|
+
return `${source}\u0000${target}`;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function relationKey(
|
|
281
|
+
source: string,
|
|
282
|
+
predicate: string,
|
|
283
|
+
target: string,
|
|
284
|
+
): string {
|
|
285
|
+
return `${source}\u0000${predicate}\u0000${target}`;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function checkedLine(line: number, context: string): number {
|
|
289
|
+
if (!Number.isSafeInteger(line) || line < 1) {
|
|
290
|
+
throw new TypeError(`${context} has an invalid evidence line.`);
|
|
291
|
+
}
|
|
292
|
+
return line;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function checkedOptions(
|
|
296
|
+
options: PercolateOptions,
|
|
297
|
+
): { readonly limit: number; readonly minSupport: number } {
|
|
298
|
+
const limit = options.limit ?? DEFAULT_PERCOLATION_LIMIT;
|
|
299
|
+
if (
|
|
300
|
+
!Number.isSafeInteger(limit)
|
|
301
|
+
|| limit < 0
|
|
302
|
+
|| limit > MAX_PERCOLATION_LIMIT
|
|
303
|
+
) {
|
|
304
|
+
throw new RangeError(
|
|
305
|
+
`Percolation limit must be a safe integer from 0 to ${MAX_PERCOLATION_LIMIT}.`,
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
const minSupport = options.minSupport ?? DEFAULT_PERCOLATION_MIN_SUPPORT;
|
|
309
|
+
if (
|
|
310
|
+
!Number.isSafeInteger(minSupport)
|
|
311
|
+
|| minSupport < 1
|
|
312
|
+
|| minSupport > MAX_PERCOLATION_EVIDENCE
|
|
313
|
+
) {
|
|
314
|
+
throw new RangeError(
|
|
315
|
+
`Percolation minimum support must be a safe integer from 1 to ${MAX_PERCOLATION_EVIDENCE}.`,
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
return { limit, minSupport };
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function indexedContentNotes(
|
|
322
|
+
notes: readonly Note[],
|
|
323
|
+
analysis: VaultAnalysis,
|
|
324
|
+
): {
|
|
325
|
+
readonly notes: readonly Note[];
|
|
326
|
+
readonly byId: ReadonlyMap<string, Note>;
|
|
327
|
+
readonly byPath: ReadonlyMap<string, Note>;
|
|
328
|
+
} {
|
|
329
|
+
if (analysis.noteConnections.length > MAX_PERCOLATION_NOTES) {
|
|
330
|
+
throw new RangeError(
|
|
331
|
+
`Percolation exceeds the ${MAX_PERCOLATION_NOTES} note limit.`,
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
const allById = new Map<string, Note>();
|
|
335
|
+
for (const note of notes) {
|
|
336
|
+
if (allById.has(note.id)) {
|
|
337
|
+
throw new Error(`Duplicate note identity in percolation: ${note.id}.`);
|
|
338
|
+
}
|
|
339
|
+
allById.set(note.id, note);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const byId = new Map<string, Note>();
|
|
343
|
+
const byPath = new Map<string, Note>();
|
|
344
|
+
for (const connection of analysis.noteConnections) {
|
|
345
|
+
if (byId.has(connection.id)) {
|
|
346
|
+
throw new Error(`Duplicate analyzed note identity: ${connection.id}.`);
|
|
347
|
+
}
|
|
348
|
+
const note = allById.get(connection.id);
|
|
349
|
+
if (note === undefined) {
|
|
350
|
+
throw new Error(`Analysis references missing note identity: ${connection.id}.`);
|
|
351
|
+
}
|
|
352
|
+
if (byPath.has(note.path)) {
|
|
353
|
+
throw new Error(`Duplicate note path in percolation: ${note.path}.`);
|
|
354
|
+
}
|
|
355
|
+
byId.set(note.id, note);
|
|
356
|
+
byPath.set(note.path, note);
|
|
357
|
+
}
|
|
358
|
+
return {
|
|
359
|
+
notes: [...byId.values()].toSorted((left, right) =>
|
|
360
|
+
compareText(left.id, right.id)),
|
|
361
|
+
byId,
|
|
362
|
+
byPath,
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
function conceptKey(value: string): string {
|
|
367
|
+
return value
|
|
368
|
+
.normalize("NFKC")
|
|
369
|
+
.toLocaleLowerCase("en-US")
|
|
370
|
+
.replace(/^#+/u, "")
|
|
371
|
+
.replace(/[^\p{Letter}\p{Number}]+/gu, "-")
|
|
372
|
+
.replace(/^-+|-+$/gu, "");
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function isConcept(note: Note): boolean {
|
|
376
|
+
return note.properties.type
|
|
377
|
+
?.normalize("NFC")
|
|
378
|
+
.toLocaleLowerCase("en-US") === "concept";
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function conceptLabels(note: Note): readonly string[] {
|
|
382
|
+
return [
|
|
383
|
+
note.title,
|
|
384
|
+
...note.aliases,
|
|
385
|
+
posix.basename(note.id),
|
|
386
|
+
].map(conceptKey).filter((value) => value !== "");
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function naturalConceptId(tag: string): string {
|
|
390
|
+
const slug = conceptKey(tag);
|
|
391
|
+
const digest = createHash("sha256").update(tag.normalize("NFC")).digest("hex");
|
|
392
|
+
if (slug === "") return `notes/concept-${digest.slice(0, 16)}`;
|
|
393
|
+
if (slug.length <= 160) return `notes/${slug}`;
|
|
394
|
+
let prefix = "";
|
|
395
|
+
let count = 0;
|
|
396
|
+
for (const character of slug) {
|
|
397
|
+
if (count >= 144) break;
|
|
398
|
+
prefix += character;
|
|
399
|
+
count += 1;
|
|
400
|
+
}
|
|
401
|
+
return `notes/${prefix.replace(/-+$/u, "")}-${digest.slice(0, 12)}`;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function suggestedConceptId(
|
|
405
|
+
tag: string,
|
|
406
|
+
occupiedIds: ReadonlyMap<string, string>,
|
|
407
|
+
reservedIds: Set<string>,
|
|
408
|
+
nextSuffixByNaturalId: Map<string, number>,
|
|
409
|
+
): { readonly id: string; readonly collidesWith: string | null } {
|
|
410
|
+
const natural = naturalConceptId(tag);
|
|
411
|
+
const foldedNatural = natural.toLocaleLowerCase("en-US");
|
|
412
|
+
const collidesWith = occupiedIds.get(foldedNatural) ?? null;
|
|
413
|
+
if (collidesWith === null && !reservedIds.has(foldedNatural)) {
|
|
414
|
+
reservedIds.add(foldedNatural);
|
|
415
|
+
return { id: natural, collidesWith: null };
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
const suffixed = `${natural}-concept`;
|
|
419
|
+
const foldedSuffixed = suffixed.toLocaleLowerCase("en-US");
|
|
420
|
+
if (!occupiedIds.has(foldedSuffixed) && !reservedIds.has(foldedSuffixed)) {
|
|
421
|
+
reservedIds.add(foldedSuffixed);
|
|
422
|
+
nextSuffixByNaturalId.set(foldedNatural, 2);
|
|
423
|
+
return { id: suffixed, collidesWith };
|
|
424
|
+
}
|
|
425
|
+
const nextSuffix = nextSuffixByNaturalId.get(foldedNatural) ?? 2;
|
|
426
|
+
for (
|
|
427
|
+
let suffix = nextSuffix;
|
|
428
|
+
suffix <= MAX_PERCOLATION_EVIDENCE + 2;
|
|
429
|
+
suffix += 1
|
|
430
|
+
) {
|
|
431
|
+
const candidate = `${suffixed}-${suffix}`;
|
|
432
|
+
const foldedCandidate = candidate.toLocaleLowerCase("en-US");
|
|
433
|
+
if (!occupiedIds.has(foldedCandidate) && !reservedIds.has(foldedCandidate)) {
|
|
434
|
+
reservedIds.add(foldedCandidate);
|
|
435
|
+
nextSuffixByNaturalId.set(foldedNatural, suffix + 1);
|
|
436
|
+
return { id: candidate, collidesWith };
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
throw new RangeError("Percolation could not choose an unoccupied concept ID.");
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function resolvedNoteFilter(
|
|
443
|
+
notes: readonly Note[],
|
|
444
|
+
query: string | undefined,
|
|
445
|
+
): string | null {
|
|
446
|
+
if (query === undefined) return null;
|
|
447
|
+
const result = lookupNote(notes, query);
|
|
448
|
+
if (result.kind === "found") return result.note.id;
|
|
449
|
+
if (result.kind === "ambiguous") {
|
|
450
|
+
throw new Error(
|
|
451
|
+
`Percolation note is ambiguous: ${result.candidates.map((note) => note.id).join(", ")}.`,
|
|
452
|
+
);
|
|
453
|
+
}
|
|
454
|
+
throw new Error(`Percolation note does not exist: ${query}.`);
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function candidateInvolvesNote(
|
|
458
|
+
candidate: PercolationCandidate,
|
|
459
|
+
note: string | null,
|
|
460
|
+
): boolean {
|
|
461
|
+
if (note === null) return true;
|
|
462
|
+
if (candidate.kind === "missing-concept") {
|
|
463
|
+
return candidate.evidence.some((evidence) => evidence.note === note);
|
|
464
|
+
}
|
|
465
|
+
return candidate.source === note || candidate.target === note;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function compareSharedEvidence(
|
|
469
|
+
left: SharedEvidence,
|
|
470
|
+
right: SharedEvidence,
|
|
471
|
+
): number {
|
|
472
|
+
return compareText(left.kind, right.kind)
|
|
473
|
+
|| compareText(left.kind === "shared-tag" ? left.tag : left.concept,
|
|
474
|
+
right.kind === "shared-tag" ? right.tag : right.concept)
|
|
475
|
+
|| compareText(left.note, right.note);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function candidateIdentity(candidate: AnyPercolationCandidate): string {
|
|
479
|
+
switch (candidate.kind) {
|
|
480
|
+
case "missing-concept":
|
|
481
|
+
return candidate.tag;
|
|
482
|
+
case "missing-relation":
|
|
483
|
+
return `${candidate.source}\u0000${candidate.target}`;
|
|
484
|
+
case "unlinked-mention":
|
|
485
|
+
return `${candidate.source}\u0000${candidate.target}`;
|
|
486
|
+
case "relation-hygiene":
|
|
487
|
+
return [
|
|
488
|
+
candidate.problem,
|
|
489
|
+
candidate.source,
|
|
490
|
+
candidate.predicate ?? "",
|
|
491
|
+
candidate.target ?? "",
|
|
492
|
+
candidate.message,
|
|
493
|
+
String(candidate.evidence[0]?.line ?? 0),
|
|
494
|
+
].join("\u0000");
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
function historicalCandidateIdentity(candidate: PercolationCandidateV1): string {
|
|
499
|
+
switch (candidate.kind) {
|
|
500
|
+
case "missing-concept":
|
|
501
|
+
return candidate.tag;
|
|
502
|
+
case "missing-relation":
|
|
503
|
+
case "unlinked-mention":
|
|
504
|
+
return `${candidate.source}\u0000${candidate.target}`;
|
|
505
|
+
case "relation-hygiene":
|
|
506
|
+
return [
|
|
507
|
+
candidate.problem,
|
|
508
|
+
candidate.source,
|
|
509
|
+
candidate.predicate ?? "",
|
|
510
|
+
candidate.target ?? "",
|
|
511
|
+
].join("\u0000");
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
const candidateKindRank: Readonly<Record<AnyPercolationCandidate["kind"], number>> = {
|
|
516
|
+
"relation-hygiene": 0,
|
|
517
|
+
"unlinked-mention": 1,
|
|
518
|
+
"missing-relation": 2,
|
|
519
|
+
"missing-concept": 3,
|
|
520
|
+
};
|
|
521
|
+
|
|
522
|
+
function compareCandidates(
|
|
523
|
+
left: AnyPercolationCandidate,
|
|
524
|
+
right: AnyPercolationCandidate,
|
|
525
|
+
): number {
|
|
526
|
+
return right.support - left.support
|
|
527
|
+
|| candidateKindRank[left.kind] - candidateKindRank[right.kind]
|
|
528
|
+
|| compareText(candidateIdentity(left), candidateIdentity(right));
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
function compareHistoricalCandidates(
|
|
532
|
+
left: PercolationCandidateV1,
|
|
533
|
+
right: PercolationCandidateV1,
|
|
534
|
+
): number {
|
|
535
|
+
return right.support - left.support
|
|
536
|
+
|| candidateKindRank[left.kind] - candidateKindRank[right.kind]
|
|
537
|
+
|| compareText(
|
|
538
|
+
historicalCandidateIdentity(left),
|
|
539
|
+
historicalCandidateIdentity(right),
|
|
540
|
+
);
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function relationEvidence(relation: AuthoredRelationLike): RelationEvidence {
|
|
544
|
+
return {
|
|
545
|
+
kind: "relation",
|
|
546
|
+
source: relation.source,
|
|
547
|
+
target: relation.target,
|
|
548
|
+
predicate: relation.predicate,
|
|
549
|
+
line: checkedLine(
|
|
550
|
+
relation.provenance.line,
|
|
551
|
+
`Authored relation ${relation.source} -> ${relation.target}`,
|
|
552
|
+
),
|
|
553
|
+
authoredTarget: relation.provenance.authoredTarget,
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function issueEvidence(
|
|
558
|
+
issue: RelationIssueLike,
|
|
559
|
+
source: string,
|
|
560
|
+
): RelationIssueEvidence {
|
|
561
|
+
const line = checkedLine(issue.line, `Relation issue in ${issue.source}`);
|
|
562
|
+
if (issue.kind === "malformed") {
|
|
563
|
+
return {
|
|
564
|
+
kind: "relation-issue",
|
|
565
|
+
issue: issue.kind,
|
|
566
|
+
source,
|
|
567
|
+
line,
|
|
568
|
+
predicate: issue.predicate ?? null,
|
|
569
|
+
target: issue.target ?? null,
|
|
570
|
+
candidates: [],
|
|
571
|
+
candidatesTruncated: false,
|
|
572
|
+
message: issue.message,
|
|
573
|
+
};
|
|
574
|
+
}
|
|
575
|
+
if (issue.kind === "broken") {
|
|
576
|
+
return {
|
|
577
|
+
kind: "relation-issue",
|
|
578
|
+
issue: issue.kind,
|
|
579
|
+
source,
|
|
580
|
+
line,
|
|
581
|
+
predicate: issue.predicate,
|
|
582
|
+
target: issue.target,
|
|
583
|
+
candidates: [],
|
|
584
|
+
candidatesTruncated: false,
|
|
585
|
+
message: `Relationship target does not exist: ${issue.target}.`,
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
return {
|
|
589
|
+
kind: "relation-issue",
|
|
590
|
+
issue: issue.kind,
|
|
591
|
+
source,
|
|
592
|
+
line,
|
|
593
|
+
predicate: issue.predicate,
|
|
594
|
+
target: issue.target,
|
|
595
|
+
candidates: [...issue.candidates]
|
|
596
|
+
.toSorted(compareText)
|
|
597
|
+
.slice(0, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE),
|
|
598
|
+
candidatesTruncated:
|
|
599
|
+
issue.candidates.length > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
|
|
600
|
+
message: `Relationship target is ambiguous: ${issue.target}.`,
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Derive bounded, evidence-backed graph maintenance candidates without
|
|
606
|
+
* changing notes, analysis, or any Markdown source.
|
|
607
|
+
*/
|
|
608
|
+
export function percolateVault(
|
|
609
|
+
notes: readonly Note[],
|
|
610
|
+
analysis: VaultAnalysis,
|
|
611
|
+
options: PercolateOptions = {},
|
|
612
|
+
): PercolationResult {
|
|
613
|
+
const { limit, minSupport } = checkedOptions(options);
|
|
614
|
+
const indexed = indexedContentNotes(notes, analysis);
|
|
615
|
+
const noteFilter = resolvedNoteFilter(indexed.notes, options.note);
|
|
616
|
+
const relations =
|
|
617
|
+
(analysis as AnalysisWithRelations).authoredRelations ?? [];
|
|
618
|
+
const relationIssues =
|
|
619
|
+
(analysis as AnalysisWithRelations).relationIssues ?? [];
|
|
620
|
+
let evidenceObservations =
|
|
621
|
+
analysis.mentions.length + relations.length + relationIssues.length;
|
|
622
|
+
for (const issue of relationIssues) {
|
|
623
|
+
if (issue.kind !== "ambiguous") continue;
|
|
624
|
+
evidenceObservations += issue.candidates.length;
|
|
625
|
+
if (evidenceObservations > MAX_PERCOLATION_EVIDENCE) break;
|
|
626
|
+
}
|
|
627
|
+
if (evidenceObservations > MAX_PERCOLATION_EVIDENCE) {
|
|
628
|
+
throw new RangeError(
|
|
629
|
+
`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} evidence limit.`,
|
|
630
|
+
);
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
const conceptIds = new Set(
|
|
634
|
+
indexed.notes.filter(isConcept).map((note) => note.id),
|
|
635
|
+
);
|
|
636
|
+
const occupiedIds = new Map(indexed.notes.map((note) => [
|
|
637
|
+
note.id.toLocaleLowerCase("en-US"),
|
|
638
|
+
note.id,
|
|
639
|
+
]));
|
|
640
|
+
const reservedConceptIds = new Set<string>();
|
|
641
|
+
const nextConceptSuffixByNaturalId = new Map<string, number>();
|
|
642
|
+
const nonConceptNotes = indexed.notes.filter((note) => !conceptIds.has(note.id));
|
|
643
|
+
const conceptLabelKeys = new Set(
|
|
644
|
+
indexed.notes
|
|
645
|
+
.filter((note) => conceptIds.has(note.id))
|
|
646
|
+
.flatMap(conceptLabels),
|
|
647
|
+
);
|
|
648
|
+
|
|
649
|
+
const candidates: PercolationCandidate[] = [];
|
|
650
|
+
const notesByTag = new Map<string, Note[]>();
|
|
651
|
+
let tagEvidenceCount = 0;
|
|
652
|
+
for (const note of nonConceptNotes) {
|
|
653
|
+
for (const tag of new Set(note.tags)) {
|
|
654
|
+
tagEvidenceCount += 1;
|
|
655
|
+
if (tagEvidenceCount > MAX_PERCOLATION_EVIDENCE) {
|
|
656
|
+
throw new RangeError(
|
|
657
|
+
`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} tag evidence limit.`,
|
|
658
|
+
);
|
|
659
|
+
}
|
|
660
|
+
const matches = notesByTag.get(tag) ?? [];
|
|
661
|
+
matches.push(note);
|
|
662
|
+
notesByTag.set(tag, matches);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
for (const [tag, matchingNotes] of [...notesByTag].toSorted(([left], [right]) =>
|
|
667
|
+
compareText(left, right))) {
|
|
668
|
+
const sortedMatches = matchingNotes.toSorted((left, right) =>
|
|
669
|
+
(left.id === noteFilter ? -1 : 0)
|
|
670
|
+
- (right.id === noteFilter ? -1 : 0)
|
|
671
|
+
|| compareText(left.id, right.id));
|
|
672
|
+
const evidence = sortedMatches
|
|
673
|
+
.slice(0, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE)
|
|
674
|
+
.map((note): MissingConceptEvidence => ({
|
|
675
|
+
kind: "tag",
|
|
676
|
+
note: note.id,
|
|
677
|
+
path: note.path,
|
|
678
|
+
tag,
|
|
679
|
+
}));
|
|
680
|
+
if (
|
|
681
|
+
matchingNotes.length >= Math.max(2, minSupport)
|
|
682
|
+
&& !conceptLabelKeys.has(conceptKey(tag))
|
|
683
|
+
&& (noteFilter === null || matchingNotes.some((note) => note.id === noteFilter))
|
|
684
|
+
) {
|
|
685
|
+
const suggestion = suggestedConceptId(
|
|
686
|
+
tag,
|
|
687
|
+
occupiedIds,
|
|
688
|
+
reservedConceptIds,
|
|
689
|
+
nextConceptSuffixByNaturalId,
|
|
690
|
+
);
|
|
691
|
+
candidates.push({
|
|
692
|
+
kind: "missing-concept",
|
|
693
|
+
tag,
|
|
694
|
+
suggestedId: suggestion.id,
|
|
695
|
+
collidesWith: suggestion.collidesWith,
|
|
696
|
+
support: matchingNotes.length,
|
|
697
|
+
evidenceTruncated:
|
|
698
|
+
matchingNotes.length > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
|
|
699
|
+
evidence,
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
const explicitPairs = new Set<string>();
|
|
705
|
+
const noteConcepts = new Map<string, Set<string>>();
|
|
706
|
+
const addConceptConnection = (noteId: string, conceptId: string): void => {
|
|
707
|
+
if (conceptIds.has(noteId) || !conceptIds.has(conceptId)) return;
|
|
708
|
+
const concepts = noteConcepts.get(noteId) ?? new Set<string>();
|
|
709
|
+
concepts.add(conceptId);
|
|
710
|
+
noteConcepts.set(noteId, concepts);
|
|
711
|
+
};
|
|
712
|
+
for (const link of analysis.contextualLinks) {
|
|
713
|
+
const source = indexed.byPath.get(link.source);
|
|
714
|
+
const target = indexed.byPath.get(link.target);
|
|
715
|
+
if (source === undefined || target === undefined) {
|
|
716
|
+
throw new Error(
|
|
717
|
+
`Contextual link references an unknown percolation note: ${link.source} -> ${link.target}.`,
|
|
718
|
+
);
|
|
719
|
+
}
|
|
720
|
+
explicitPairs.add(pairKey(source.id, target.id));
|
|
721
|
+
addConceptConnection(source.id, target.id);
|
|
722
|
+
addConceptConnection(target.id, source.id);
|
|
723
|
+
}
|
|
724
|
+
for (const relation of relations) {
|
|
725
|
+
if (!indexed.byId.has(relation.source) || !indexed.byId.has(relation.target)) {
|
|
726
|
+
throw new Error(
|
|
727
|
+
`Authored relation references an unknown percolation note: ${relation.source} -> ${relation.target}.`,
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
explicitPairs.add(pairKey(relation.source, relation.target));
|
|
731
|
+
addConceptConnection(relation.source, relation.target);
|
|
732
|
+
addConceptConnection(relation.target, relation.source);
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
const pairEvidence = new Map<string, SharedAccumulation>();
|
|
736
|
+
let pairObservations = 0;
|
|
737
|
+
const addPairEvidence = (
|
|
738
|
+
left: Note,
|
|
739
|
+
right: Note,
|
|
740
|
+
evidence: readonly SharedEvidence[],
|
|
741
|
+
): void => {
|
|
742
|
+
pairObservations += 1;
|
|
743
|
+
if (pairObservations > MAX_PERCOLATION_PAIR_OBSERVATIONS) {
|
|
744
|
+
throw new RangeError(
|
|
745
|
+
`Percolation exceeds the ${MAX_PERCOLATION_PAIR_OBSERVATIONS} pair observation limit.`,
|
|
746
|
+
);
|
|
747
|
+
}
|
|
748
|
+
const key = pairKey(left.id, right.id);
|
|
749
|
+
if (explicitPairs.has(key)) return;
|
|
750
|
+
const accumulated = pairEvidence.get(key) ?? {
|
|
751
|
+
support: 0,
|
|
752
|
+
evidenceCount: 0,
|
|
753
|
+
evidence: [],
|
|
754
|
+
};
|
|
755
|
+
accumulated.support += 1;
|
|
756
|
+
accumulated.evidenceCount += evidence.length;
|
|
757
|
+
for (const item of evidence) {
|
|
758
|
+
if (
|
|
759
|
+
accumulated.evidence.length
|
|
760
|
+
< MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE
|
|
761
|
+
) {
|
|
762
|
+
accumulated.evidence.push(item);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
pairEvidence.set(key, accumulated);
|
|
766
|
+
};
|
|
767
|
+
|
|
768
|
+
const addPairsForGroup = (
|
|
769
|
+
matchingNotes: readonly Note[],
|
|
770
|
+
evidenceFor: (left: Note, right: Note) => readonly SharedEvidence[],
|
|
771
|
+
): void => {
|
|
772
|
+
const sortedNotes = matchingNotes.toSorted((left, right) =>
|
|
773
|
+
compareText(left.id, right.id));
|
|
774
|
+
if (noteFilter !== null) {
|
|
775
|
+
const scoped = sortedNotes.find((note) => note.id === noteFilter);
|
|
776
|
+
if (scoped === undefined) return;
|
|
777
|
+
for (const other of sortedNotes) {
|
|
778
|
+
if (other.id === scoped.id) continue;
|
|
779
|
+
const [left, right] = compareText(scoped.id, other.id) < 0
|
|
780
|
+
? [scoped, other]
|
|
781
|
+
: [other, scoped];
|
|
782
|
+
addPairEvidence(left, right, evidenceFor(left, right));
|
|
783
|
+
}
|
|
784
|
+
return;
|
|
785
|
+
}
|
|
786
|
+
for (let leftIndex = 0; leftIndex < sortedNotes.length; leftIndex += 1) {
|
|
787
|
+
for (
|
|
788
|
+
let rightIndex = leftIndex + 1;
|
|
789
|
+
rightIndex < sortedNotes.length;
|
|
790
|
+
rightIndex += 1
|
|
791
|
+
) {
|
|
792
|
+
const left = sortedNotes[leftIndex];
|
|
793
|
+
const right = sortedNotes[rightIndex];
|
|
794
|
+
if (left === undefined || right === undefined) continue;
|
|
795
|
+
addPairEvidence(left, right, evidenceFor(left, right));
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
};
|
|
799
|
+
|
|
800
|
+
for (const [tag, matchingNotes] of [...notesByTag].toSorted(
|
|
801
|
+
([left], [right]) => compareText(left, right),
|
|
802
|
+
)) {
|
|
803
|
+
addPairsForGroup(matchingNotes, (left, right) => [
|
|
804
|
+
{ kind: "shared-tag", note: left.id, path: left.path, tag },
|
|
805
|
+
{ kind: "shared-tag", note: right.id, path: right.path, tag },
|
|
806
|
+
]);
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
const notesByConcept = new Map<string, Note[]>();
|
|
810
|
+
for (const [noteId, connectedConcepts] of noteConcepts) {
|
|
811
|
+
const note = indexed.byId.get(noteId);
|
|
812
|
+
if (note === undefined) continue;
|
|
813
|
+
for (const concept of connectedConcepts) {
|
|
814
|
+
const matches = notesByConcept.get(concept) ?? [];
|
|
815
|
+
matches.push(note);
|
|
816
|
+
notesByConcept.set(concept, matches);
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
for (const [concept, matchingNotes] of [...notesByConcept].toSorted(
|
|
820
|
+
([left], [right]) => compareText(left, right),
|
|
821
|
+
)) {
|
|
822
|
+
const conceptNote = indexed.byId.get(concept);
|
|
823
|
+
if (conceptNote === undefined) continue;
|
|
824
|
+
addPairsForGroup(matchingNotes, (left, right) => [
|
|
825
|
+
{
|
|
826
|
+
kind: "shared-concept",
|
|
827
|
+
note: left.id,
|
|
828
|
+
path: left.path,
|
|
829
|
+
concept,
|
|
830
|
+
conceptPath: conceptNote.path,
|
|
831
|
+
},
|
|
832
|
+
{
|
|
833
|
+
kind: "shared-concept",
|
|
834
|
+
note: right.id,
|
|
835
|
+
path: right.path,
|
|
836
|
+
concept,
|
|
837
|
+
conceptPath: conceptNote.path,
|
|
838
|
+
},
|
|
839
|
+
]);
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
for (const [key, accumulated] of pairEvidence) {
|
|
843
|
+
const separator = key.indexOf("\u0000");
|
|
844
|
+
const source = key.slice(0, separator);
|
|
845
|
+
const target = key.slice(separator + 1);
|
|
846
|
+
const evidence = accumulated.evidence.toSorted(compareSharedEvidence);
|
|
847
|
+
if (accumulated.support < minSupport) continue;
|
|
848
|
+
candidates.push({
|
|
849
|
+
kind: "missing-relation",
|
|
850
|
+
source,
|
|
851
|
+
target,
|
|
852
|
+
predicate: { kind: "required" },
|
|
853
|
+
support: accumulated.support,
|
|
854
|
+
evidenceTruncated:
|
|
855
|
+
accumulated.evidenceCount > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
|
|
856
|
+
evidence,
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
const mentionEvidenceByPair = new Map<string, MentionAccumulation>();
|
|
861
|
+
for (const mention of analysis.mentions) {
|
|
862
|
+
const source = indexed.byPath.get(mention.source);
|
|
863
|
+
const target = indexed.byPath.get(mention.target);
|
|
864
|
+
if (source === undefined || target === undefined) {
|
|
865
|
+
throw new Error(
|
|
866
|
+
`Mention references an unknown percolation note: ${mention.source} -> ${mention.target}.`,
|
|
867
|
+
);
|
|
868
|
+
}
|
|
869
|
+
if (
|
|
870
|
+
noteFilter !== null
|
|
871
|
+
&& source.id !== noteFilter
|
|
872
|
+
&& target.id !== noteFilter
|
|
873
|
+
) continue;
|
|
874
|
+
if (explicitPairs.has(pairKey(source.id, target.id))) continue;
|
|
875
|
+
const key = directedKey(source.id, target.id);
|
|
876
|
+
const accumulated = mentionEvidenceByPair.get(key) ?? {
|
|
877
|
+
support: 0,
|
|
878
|
+
evidence: [],
|
|
879
|
+
};
|
|
880
|
+
accumulated.support += 1;
|
|
881
|
+
if (
|
|
882
|
+
accumulated.evidence.length
|
|
883
|
+
< MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE
|
|
884
|
+
) accumulated.evidence.push({
|
|
885
|
+
kind: "mention",
|
|
886
|
+
source: source.id,
|
|
887
|
+
target: target.id,
|
|
888
|
+
line: checkedLine(
|
|
889
|
+
mention.line,
|
|
890
|
+
`Mention ${mention.source} -> ${mention.target}`,
|
|
891
|
+
),
|
|
892
|
+
phrase: mention.phrase,
|
|
893
|
+
});
|
|
894
|
+
mentionEvidenceByPair.set(key, accumulated);
|
|
895
|
+
}
|
|
896
|
+
for (const [key, accumulated] of mentionEvidenceByPair) {
|
|
897
|
+
const separator = key.indexOf("\u0000");
|
|
898
|
+
const source = key.slice(0, separator);
|
|
899
|
+
const target = key.slice(separator + 1);
|
|
900
|
+
const sortedEvidence = accumulated.evidence.toSorted((left, right) =>
|
|
901
|
+
left.line - right.line || compareText(left.phrase, right.phrase));
|
|
902
|
+
candidates.push({
|
|
903
|
+
kind: "unlinked-mention",
|
|
904
|
+
source,
|
|
905
|
+
target,
|
|
906
|
+
support: accumulated.support,
|
|
907
|
+
evidenceTruncated:
|
|
908
|
+
accumulated.support > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
|
|
909
|
+
evidence: sortedEvidence,
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
const relationsByKey = new Map(
|
|
914
|
+
relations.map((relation) => [
|
|
915
|
+
relationKey(relation.source, relation.predicate, relation.target),
|
|
916
|
+
relation,
|
|
917
|
+
]),
|
|
918
|
+
);
|
|
919
|
+
for (const relation of relations) {
|
|
920
|
+
if (
|
|
921
|
+
noteFilter !== null
|
|
922
|
+
&& relation.source !== noteFilter
|
|
923
|
+
&& relation.target !== noteFilter
|
|
924
|
+
) continue;
|
|
925
|
+
if (relation.source === relation.target) {
|
|
926
|
+
candidates.push({
|
|
927
|
+
kind: "relation-hygiene",
|
|
928
|
+
problem: "self-relation",
|
|
929
|
+
source: relation.source,
|
|
930
|
+
target: relation.target,
|
|
931
|
+
predicate: relation.predicate,
|
|
932
|
+
message: "Review an authored relationship whose source and target are the same note.",
|
|
933
|
+
support: 1,
|
|
934
|
+
evidenceTruncated: false,
|
|
935
|
+
evidence: [relationEvidence(relation)],
|
|
936
|
+
});
|
|
937
|
+
continue;
|
|
938
|
+
}
|
|
939
|
+
if (compareText(relation.source, relation.target) >= 0) continue;
|
|
940
|
+
const reciprocal = relationsByKey.get(
|
|
941
|
+
relationKey(relation.target, relation.predicate, relation.source),
|
|
942
|
+
);
|
|
943
|
+
if (reciprocal === undefined) continue;
|
|
944
|
+
const evidence = [
|
|
945
|
+
relationEvidence(relation),
|
|
946
|
+
relationEvidence(reciprocal),
|
|
947
|
+
].toSorted((left, right) =>
|
|
948
|
+
compareText(left.source, right.source)
|
|
949
|
+
|| compareText(left.target, right.target));
|
|
950
|
+
candidates.push({
|
|
951
|
+
kind: "relation-hygiene",
|
|
952
|
+
problem: "reciprocal-relation",
|
|
953
|
+
source: relation.source,
|
|
954
|
+
target: relation.target,
|
|
955
|
+
predicate: relation.predicate,
|
|
956
|
+
message: "Review reciprocal assertions of the same directional predicate.",
|
|
957
|
+
support: evidence.length,
|
|
958
|
+
evidenceTruncated: false,
|
|
959
|
+
evidence,
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
for (const issue of relationIssues) {
|
|
964
|
+
const sourceNote = indexed.byPath.get(issue.source);
|
|
965
|
+
if (sourceNote === undefined) continue;
|
|
966
|
+
if (noteFilter !== null && sourceNote.id !== noteFilter) continue;
|
|
967
|
+
const evidence = issueEvidence(issue, sourceNote.id);
|
|
968
|
+
const problem: RelationHygieneProblem = issue.kind === "malformed"
|
|
969
|
+
? "malformed-relation"
|
|
970
|
+
: issue.kind === "broken"
|
|
971
|
+
? "broken-relation"
|
|
972
|
+
: "ambiguous-relation";
|
|
973
|
+
candidates.push({
|
|
974
|
+
kind: "relation-hygiene",
|
|
975
|
+
problem,
|
|
976
|
+
source: sourceNote.id,
|
|
977
|
+
target: evidence.target,
|
|
978
|
+
predicate: evidence.predicate,
|
|
979
|
+
message: evidence.message,
|
|
980
|
+
support: 1,
|
|
981
|
+
evidenceTruncated: evidence.candidatesTruncated,
|
|
982
|
+
evidence: [evidence],
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
if (candidates.length > MAX_PERCOLATION_EVIDENCE) {
|
|
987
|
+
throw new RangeError(
|
|
988
|
+
`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} candidate limit.`,
|
|
989
|
+
);
|
|
990
|
+
}
|
|
991
|
+
const uniqueCandidates = new Map<string, PercolationCandidate>();
|
|
992
|
+
for (const candidate of candidates) {
|
|
993
|
+
const identity = `${candidate.kind}\u0000${candidateIdentity(candidate)}`;
|
|
994
|
+
if (!uniqueCandidates.has(identity)) uniqueCandidates.set(identity, candidate);
|
|
995
|
+
}
|
|
996
|
+
const sorted = [...uniqueCandidates.values()]
|
|
997
|
+
.filter((candidate) => candidateInvolvesNote(candidate, noteFilter))
|
|
998
|
+
.toSorted(compareCandidates);
|
|
999
|
+
return parsePercolationResultV2({
|
|
1000
|
+
schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION,
|
|
1001
|
+
candidates: sorted.slice(0, limit),
|
|
1002
|
+
truncated: sorted.length > limit,
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
type ParseBudget = {
|
|
1007
|
+
nodes: number;
|
|
1008
|
+
utf8Bytes: number;
|
|
1009
|
+
};
|
|
1010
|
+
|
|
1011
|
+
type DataRecord = Readonly<Record<string, unknown>>;
|
|
1012
|
+
|
|
1013
|
+
function countParseNode(budget: ParseBudget, label: string): void {
|
|
1014
|
+
budget.nodes += 1;
|
|
1015
|
+
if (budget.nodes > MAX_PERCOLATION_RESULT_NODES) {
|
|
1016
|
+
throw new RangeError(
|
|
1017
|
+
`${label} exceeds the ${MAX_PERCOLATION_RESULT_NODES.toLocaleString("en-US")}-node percolation result limit.`,
|
|
1018
|
+
);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
function dataRecord(
|
|
1023
|
+
value: unknown,
|
|
1024
|
+
label: string,
|
|
1025
|
+
budget: ParseBudget,
|
|
1026
|
+
): DataRecord {
|
|
1027
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
1028
|
+
throw new TypeError(`${label} must be a plain data object.`);
|
|
1029
|
+
}
|
|
1030
|
+
const prototype: unknown = Object.getPrototypeOf(value);
|
|
1031
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
1032
|
+
throw new TypeError(`${label} must be a plain data object.`);
|
|
1033
|
+
}
|
|
1034
|
+
countParseNode(budget, label);
|
|
1035
|
+
const output = Object.create(null) as Record<string, unknown>;
|
|
1036
|
+
const descriptors = Object.getOwnPropertyDescriptors(value);
|
|
1037
|
+
for (const key of Reflect.ownKeys(descriptors)) {
|
|
1038
|
+
if (typeof key !== "string") {
|
|
1039
|
+
throw new TypeError(`${label} must not contain symbol fields.`);
|
|
1040
|
+
}
|
|
1041
|
+
const descriptor = descriptors[key];
|
|
1042
|
+
if (
|
|
1043
|
+
descriptor === undefined
|
|
1044
|
+
|| !("value" in descriptor)
|
|
1045
|
+
|| !descriptor.enumerable
|
|
1046
|
+
) {
|
|
1047
|
+
throw new TypeError(`${label}.${key} must be an enumerable data property.`);
|
|
1048
|
+
}
|
|
1049
|
+
Object.defineProperty(output, key, {
|
|
1050
|
+
configurable: false,
|
|
1051
|
+
enumerable: true,
|
|
1052
|
+
value: descriptor.value,
|
|
1053
|
+
writable: false,
|
|
1054
|
+
});
|
|
1055
|
+
}
|
|
1056
|
+
return Object.freeze(output);
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
function exactKeys(
|
|
1060
|
+
record: DataRecord,
|
|
1061
|
+
keys: readonly string[],
|
|
1062
|
+
label: string,
|
|
1063
|
+
): void {
|
|
1064
|
+
const actual = Reflect.ownKeys(record);
|
|
1065
|
+
const expected = new Set(keys);
|
|
1066
|
+
if (
|
|
1067
|
+
actual.length !== keys.length
|
|
1068
|
+
|| actual.some((key) => typeof key !== "string" || !expected.has(key))
|
|
1069
|
+
) {
|
|
1070
|
+
throw new TypeError(`${label} must contain exactly: ${keys.join(", ")}.`);
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
function dataArray(
|
|
1075
|
+
value: unknown,
|
|
1076
|
+
label: string,
|
|
1077
|
+
maximum: number,
|
|
1078
|
+
budget: ParseBudget,
|
|
1079
|
+
): readonly unknown[] {
|
|
1080
|
+
if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype) {
|
|
1081
|
+
throw new TypeError(`${label} must be an ordinary array.`);
|
|
1082
|
+
}
|
|
1083
|
+
if (value.length > maximum) {
|
|
1084
|
+
throw new RangeError(
|
|
1085
|
+
`${label} exceeds its ${maximum.toLocaleString("en-US")}-entry limit.`,
|
|
1086
|
+
);
|
|
1087
|
+
}
|
|
1088
|
+
countParseNode(budget, label);
|
|
1089
|
+
const descriptors = Object.getOwnPropertyDescriptors(value);
|
|
1090
|
+
for (const key of Reflect.ownKeys(descriptors)) {
|
|
1091
|
+
if (typeof key !== "string") {
|
|
1092
|
+
throw new TypeError(`${label} must not contain symbol fields.`);
|
|
1093
|
+
}
|
|
1094
|
+
if (key === "length") continue;
|
|
1095
|
+
const index = Number(key);
|
|
1096
|
+
if (
|
|
1097
|
+
!Number.isSafeInteger(index)
|
|
1098
|
+
|| index < 0
|
|
1099
|
+
|| index >= value.length
|
|
1100
|
+
|| String(index) !== key
|
|
1101
|
+
) {
|
|
1102
|
+
throw new TypeError(`${label} contains a non-index property.`);
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
const output: unknown[] = [];
|
|
1106
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
1107
|
+
const descriptor = descriptors[String(index)];
|
|
1108
|
+
if (
|
|
1109
|
+
descriptor === undefined
|
|
1110
|
+
|| !("value" in descriptor)
|
|
1111
|
+
|| !descriptor.enumerable
|
|
1112
|
+
) {
|
|
1113
|
+
throw new TypeError(`${label} must be a dense array of data properties.`);
|
|
1114
|
+
}
|
|
1115
|
+
output.push(descriptor.value);
|
|
1116
|
+
}
|
|
1117
|
+
return Object.freeze(output);
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
function hasUnpairedSurrogate(value: string): boolean {
|
|
1121
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
1122
|
+
const code = value.charCodeAt(index);
|
|
1123
|
+
if (code >= 0xd800 && code <= 0xdbff) {
|
|
1124
|
+
const next = value.charCodeAt(index + 1);
|
|
1125
|
+
if (next < 0xdc00 || next > 0xdfff) return true;
|
|
1126
|
+
index += 1;
|
|
1127
|
+
} else if (code >= 0xdc00 && code <= 0xdfff) {
|
|
1128
|
+
return true;
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
return false;
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
function parsedText(
|
|
1135
|
+
value: unknown,
|
|
1136
|
+
label: string,
|
|
1137
|
+
budget: ParseBudget,
|
|
1138
|
+
options: { readonly empty?: boolean } = {},
|
|
1139
|
+
): string {
|
|
1140
|
+
if (
|
|
1141
|
+
typeof value !== "string"
|
|
1142
|
+
|| (options.empty !== true && value === "")
|
|
1143
|
+
|| hasUnpairedSurrogate(value)
|
|
1144
|
+
) {
|
|
1145
|
+
throw new TypeError(`${label} must be a bounded Unicode string.`);
|
|
1146
|
+
}
|
|
1147
|
+
const bytes = new TextEncoder().encode(value).byteLength;
|
|
1148
|
+
if (bytes > MAX_PERCOLATION_TEXT_UTF8_BYTES) {
|
|
1149
|
+
throw new RangeError(
|
|
1150
|
+
`${label} exceeds its ${MAX_PERCOLATION_TEXT_UTF8_BYTES.toLocaleString("en-US")}-byte limit.`,
|
|
1151
|
+
);
|
|
1152
|
+
}
|
|
1153
|
+
budget.utf8Bytes += bytes;
|
|
1154
|
+
if (budget.utf8Bytes > MAX_PERCOLATION_RESULT_UTF8_BYTES) {
|
|
1155
|
+
throw new RangeError(
|
|
1156
|
+
`Percolation result text exceeds ${MAX_PERCOLATION_RESULT_UTF8_BYTES.toLocaleString("en-US")} UTF-8 bytes.`,
|
|
1157
|
+
);
|
|
1158
|
+
}
|
|
1159
|
+
return value;
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
function canonicalNote(
|
|
1163
|
+
value: unknown,
|
|
1164
|
+
label: string,
|
|
1165
|
+
budget: ParseBudget,
|
|
1166
|
+
): string {
|
|
1167
|
+
const parsed = parsedText(value, label, budget);
|
|
1168
|
+
if (!isCanonicalNoteId(parsed)) {
|
|
1169
|
+
throw new TypeError(`${label} must be a canonical note ID.`);
|
|
1170
|
+
}
|
|
1171
|
+
return parsed;
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
function canonicalMarkdownPath(
|
|
1175
|
+
value: unknown,
|
|
1176
|
+
label: string,
|
|
1177
|
+
budget: ParseBudget,
|
|
1178
|
+
): string {
|
|
1179
|
+
const parsed = parsedText(value, label, budget);
|
|
1180
|
+
if (
|
|
1181
|
+
!parsed.endsWith(".md")
|
|
1182
|
+
|| !isCanonicalNoteId(parsed.slice(0, -3))
|
|
1183
|
+
) {
|
|
1184
|
+
throw new TypeError(`${label} must be a canonical vault Markdown path.`);
|
|
1185
|
+
}
|
|
1186
|
+
return parsed;
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
function canonicalPredicate(
|
|
1190
|
+
value: unknown,
|
|
1191
|
+
label: string,
|
|
1192
|
+
budget: ParseBudget,
|
|
1193
|
+
): string {
|
|
1194
|
+
const parsed = parsedText(value, label, budget);
|
|
1195
|
+
if (!isCanonicalRelationPredicate(parsed)) {
|
|
1196
|
+
throw new TypeError(`${label} must be a canonical relation predicate.`);
|
|
1197
|
+
}
|
|
1198
|
+
return parsed;
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
function nullableText(
|
|
1202
|
+
value: unknown,
|
|
1203
|
+
label: string,
|
|
1204
|
+
budget: ParseBudget,
|
|
1205
|
+
): string | null {
|
|
1206
|
+
return value === null ? null : parsedText(value, label, budget, { empty: true });
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
function parsedBoolean(value: unknown, label: string): boolean {
|
|
1210
|
+
if (typeof value !== "boolean") throw new TypeError(`${label} must be a boolean.`);
|
|
1211
|
+
return value;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
function positiveSafeInteger(
|
|
1215
|
+
value: unknown,
|
|
1216
|
+
label: string,
|
|
1217
|
+
maximum = MAX_PERCOLATION_EVIDENCE,
|
|
1218
|
+
): number {
|
|
1219
|
+
if (
|
|
1220
|
+
typeof value !== "number"
|
|
1221
|
+
|| !Number.isSafeInteger(value)
|
|
1222
|
+
|| value < 1
|
|
1223
|
+
|| value > maximum
|
|
1224
|
+
) {
|
|
1225
|
+
throw new TypeError(`${label} must be a positive bounded safe integer.`);
|
|
1226
|
+
}
|
|
1227
|
+
return value;
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
function parsedMinSupport(value: unknown, label: string): number {
|
|
1231
|
+
if (
|
|
1232
|
+
typeof value !== "number"
|
|
1233
|
+
|| !Number.isSafeInteger(value)
|
|
1234
|
+
|| value < DEFAULT_PERCOLATION_MIN_SUPPORT
|
|
1235
|
+
|| value > MAX_PERCOLATION_LIMIT
|
|
1236
|
+
) {
|
|
1237
|
+
throw new TypeError(
|
|
1238
|
+
`${label} must be an integer from ${DEFAULT_PERCOLATION_MIN_SUPPORT} through ${MAX_PERCOLATION_LIMIT}.`,
|
|
1239
|
+
);
|
|
1240
|
+
}
|
|
1241
|
+
return value;
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
function predicateDisposition(
|
|
1245
|
+
value: unknown,
|
|
1246
|
+
label: string,
|
|
1247
|
+
budget: ParseBudget,
|
|
1248
|
+
): PredicateDisposition {
|
|
1249
|
+
const record = dataRecord(value, label, budget);
|
|
1250
|
+
const kind = parsedText(record.kind, `${label}.kind`, budget);
|
|
1251
|
+
if (kind === "required") {
|
|
1252
|
+
exactKeys(record, ["kind"], label);
|
|
1253
|
+
return Object.freeze({ kind: "required" });
|
|
1254
|
+
}
|
|
1255
|
+
throw new TypeError(`${label}.kind must be required.`);
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
function parsedMissingConceptEvidence(
|
|
1259
|
+
value: unknown,
|
|
1260
|
+
label: string,
|
|
1261
|
+
budget: ParseBudget,
|
|
1262
|
+
): MissingConceptEvidence {
|
|
1263
|
+
const record = dataRecord(value, label, budget);
|
|
1264
|
+
exactKeys(record, ["kind", "note", "path", "tag"], label);
|
|
1265
|
+
if (parsedText(record.kind, `${label}.kind`, budget) !== "tag") {
|
|
1266
|
+
throw new TypeError(`${label}.kind must be tag.`);
|
|
1267
|
+
}
|
|
1268
|
+
const note = canonicalNote(record.note, `${label}.note`, budget);
|
|
1269
|
+
const path = canonicalMarkdownPath(record.path, `${label}.path`, budget);
|
|
1270
|
+
if (path !== `${note}.md`) throw new TypeError(`${label}.path must identify its note.`);
|
|
1271
|
+
return Object.freeze({
|
|
1272
|
+
kind: "tag",
|
|
1273
|
+
note,
|
|
1274
|
+
path,
|
|
1275
|
+
tag: parsedText(record.tag, `${label}.tag`, budget),
|
|
1276
|
+
});
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
function parsedSharedEvidence(
|
|
1280
|
+
value: unknown,
|
|
1281
|
+
label: string,
|
|
1282
|
+
budget: ParseBudget,
|
|
1283
|
+
): SharedEvidence {
|
|
1284
|
+
const record = dataRecord(value, label, budget);
|
|
1285
|
+
const kind = parsedText(record.kind, `${label}.kind`, budget);
|
|
1286
|
+
if (kind === "shared-tag") {
|
|
1287
|
+
exactKeys(record, ["kind", "note", "path", "tag"], label);
|
|
1288
|
+
const note = canonicalNote(record.note, `${label}.note`, budget);
|
|
1289
|
+
const path = canonicalMarkdownPath(record.path, `${label}.path`, budget);
|
|
1290
|
+
if (path !== `${note}.md`) throw new TypeError(`${label}.path must identify its note.`);
|
|
1291
|
+
return Object.freeze({
|
|
1292
|
+
kind: "shared-tag",
|
|
1293
|
+
note,
|
|
1294
|
+
path,
|
|
1295
|
+
tag: parsedText(record.tag, `${label}.tag`, budget),
|
|
1296
|
+
});
|
|
1297
|
+
}
|
|
1298
|
+
if (kind === "shared-concept") {
|
|
1299
|
+
exactKeys(
|
|
1300
|
+
record,
|
|
1301
|
+
["kind", "note", "path", "concept", "conceptPath"],
|
|
1302
|
+
label,
|
|
1303
|
+
);
|
|
1304
|
+
const note = canonicalNote(record.note, `${label}.note`, budget);
|
|
1305
|
+
const path = canonicalMarkdownPath(record.path, `${label}.path`, budget);
|
|
1306
|
+
const concept = canonicalNote(record.concept, `${label}.concept`, budget);
|
|
1307
|
+
const conceptPath = canonicalMarkdownPath(
|
|
1308
|
+
record.conceptPath,
|
|
1309
|
+
`${label}.conceptPath`,
|
|
1310
|
+
budget,
|
|
1311
|
+
);
|
|
1312
|
+
if (path !== `${note}.md`) throw new TypeError(`${label}.path must identify its note.`);
|
|
1313
|
+
if (conceptPath !== `${concept}.md`) {
|
|
1314
|
+
throw new TypeError(`${label}.conceptPath must identify its concept.`);
|
|
1315
|
+
}
|
|
1316
|
+
return Object.freeze({
|
|
1317
|
+
kind: "shared-concept",
|
|
1318
|
+
note,
|
|
1319
|
+
path,
|
|
1320
|
+
concept,
|
|
1321
|
+
conceptPath,
|
|
1322
|
+
});
|
|
1323
|
+
}
|
|
1324
|
+
throw new TypeError(`${label}.kind must be shared-tag or shared-concept.`);
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
function parsedMentionEvidence(
|
|
1328
|
+
value: unknown,
|
|
1329
|
+
label: string,
|
|
1330
|
+
budget: ParseBudget,
|
|
1331
|
+
): MentionEvidence {
|
|
1332
|
+
const record = dataRecord(value, label, budget);
|
|
1333
|
+
exactKeys(record, ["kind", "source", "target", "line", "phrase"], label);
|
|
1334
|
+
if (parsedText(record.kind, `${label}.kind`, budget) !== "mention") {
|
|
1335
|
+
throw new TypeError(`${label}.kind must be mention.`);
|
|
1336
|
+
}
|
|
1337
|
+
return Object.freeze({
|
|
1338
|
+
kind: "mention",
|
|
1339
|
+
source: canonicalNote(record.source, `${label}.source`, budget),
|
|
1340
|
+
target: canonicalNote(record.target, `${label}.target`, budget),
|
|
1341
|
+
line: positiveSafeInteger(record.line, `${label}.line`),
|
|
1342
|
+
phrase: parsedText(record.phrase, `${label}.phrase`, budget),
|
|
1343
|
+
});
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
function parsedRelationEvidence(
|
|
1347
|
+
value: unknown,
|
|
1348
|
+
label: string,
|
|
1349
|
+
budget: ParseBudget,
|
|
1350
|
+
): RelationEvidence {
|
|
1351
|
+
const record = dataRecord(value, label, budget);
|
|
1352
|
+
exactKeys(
|
|
1353
|
+
record,
|
|
1354
|
+
["kind", "source", "target", "predicate", "line", "authoredTarget"],
|
|
1355
|
+
label,
|
|
1356
|
+
);
|
|
1357
|
+
if (parsedText(record.kind, `${label}.kind`, budget) !== "relation") {
|
|
1358
|
+
throw new TypeError(`${label}.kind must be relation.`);
|
|
1359
|
+
}
|
|
1360
|
+
return Object.freeze({
|
|
1361
|
+
kind: "relation",
|
|
1362
|
+
source: canonicalNote(record.source, `${label}.source`, budget),
|
|
1363
|
+
target: canonicalNote(record.target, `${label}.target`, budget),
|
|
1364
|
+
predicate: canonicalPredicate(record.predicate, `${label}.predicate`, budget),
|
|
1365
|
+
line: positiveSafeInteger(record.line, `${label}.line`),
|
|
1366
|
+
authoredTarget: parsedText(record.authoredTarget, `${label}.authoredTarget`, budget),
|
|
1367
|
+
});
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
function parsedRelationIssueEvidence(
|
|
1371
|
+
value: unknown,
|
|
1372
|
+
label: string,
|
|
1373
|
+
budget: ParseBudget,
|
|
1374
|
+
): RelationIssueEvidence {
|
|
1375
|
+
const record = dataRecord(value, label, budget);
|
|
1376
|
+
exactKeys(
|
|
1377
|
+
record,
|
|
1378
|
+
[
|
|
1379
|
+
"kind",
|
|
1380
|
+
"issue",
|
|
1381
|
+
"source",
|
|
1382
|
+
"line",
|
|
1383
|
+
"predicate",
|
|
1384
|
+
"target",
|
|
1385
|
+
"candidates",
|
|
1386
|
+
"candidatesTruncated",
|
|
1387
|
+
"message",
|
|
1388
|
+
],
|
|
1389
|
+
label,
|
|
1390
|
+
);
|
|
1391
|
+
if (parsedText(record.kind, `${label}.kind`, budget) !== "relation-issue") {
|
|
1392
|
+
throw new TypeError(`${label}.kind must be relation-issue.`);
|
|
1393
|
+
}
|
|
1394
|
+
if (
|
|
1395
|
+
record.issue !== "malformed"
|
|
1396
|
+
&& record.issue !== "broken"
|
|
1397
|
+
&& record.issue !== "ambiguous"
|
|
1398
|
+
) {
|
|
1399
|
+
throw new TypeError(`${label}.issue is unsupported.`);
|
|
1400
|
+
}
|
|
1401
|
+
const issue = parsedText(record.issue, `${label}.issue`, budget) as
|
|
1402
|
+
RelationIssueEvidence["issue"];
|
|
1403
|
+
const candidates = dataArray(
|
|
1404
|
+
record.candidates,
|
|
1405
|
+
`${label}.candidates`,
|
|
1406
|
+
MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
|
|
1407
|
+
budget,
|
|
1408
|
+
).map((candidate, index) =>
|
|
1409
|
+
canonicalNote(candidate, `${label}.candidates[${index}]`, budget));
|
|
1410
|
+
for (let index = 0; index < candidates.length; index += 1) {
|
|
1411
|
+
const previous = candidates[index - 1];
|
|
1412
|
+
const candidate = candidates[index];
|
|
1413
|
+
if (candidate === undefined) continue;
|
|
1414
|
+
if (previous !== undefined && compareText(previous, candidate) >= 0) {
|
|
1415
|
+
throw new TypeError(`${label}.candidates must be sorted and unique.`);
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
if (issue !== "ambiguous" && candidates.length !== 0) {
|
|
1419
|
+
throw new TypeError(`${label}.candidates are only valid for ambiguous issues.`);
|
|
1420
|
+
}
|
|
1421
|
+
if (issue === "ambiguous" && candidates.length < 2) {
|
|
1422
|
+
throw new TypeError(`${label}.candidates must identify at least two ambiguous notes.`);
|
|
1423
|
+
}
|
|
1424
|
+
const candidatesTruncated = parsedBoolean(
|
|
1425
|
+
record.candidatesTruncated,
|
|
1426
|
+
`${label}.candidatesTruncated`,
|
|
1427
|
+
);
|
|
1428
|
+
if (
|
|
1429
|
+
(issue !== "ambiguous" && candidatesTruncated)
|
|
1430
|
+
|| (candidatesTruncated
|
|
1431
|
+
&& candidates.length !== MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE)
|
|
1432
|
+
) {
|
|
1433
|
+
throw new TypeError(`${label}.candidatesTruncated is inconsistent.`);
|
|
1434
|
+
}
|
|
1435
|
+
const predicate = issue === "malformed"
|
|
1436
|
+
? nullableText(record.predicate, `${label}.predicate`, budget)
|
|
1437
|
+
: canonicalPredicate(record.predicate, `${label}.predicate`, budget);
|
|
1438
|
+
const target = issue === "malformed"
|
|
1439
|
+
? nullableText(record.target, `${label}.target`, budget)
|
|
1440
|
+
: canonicalNote(record.target, `${label}.target`, budget);
|
|
1441
|
+
return Object.freeze({
|
|
1442
|
+
kind: "relation-issue",
|
|
1443
|
+
issue,
|
|
1444
|
+
source: canonicalNote(record.source, `${label}.source`, budget),
|
|
1445
|
+
line: positiveSafeInteger(record.line, `${label}.line`),
|
|
1446
|
+
predicate,
|
|
1447
|
+
target,
|
|
1448
|
+
candidates: Object.freeze(candidates),
|
|
1449
|
+
candidatesTruncated,
|
|
1450
|
+
message: parsedText(record.message, `${label}.message`, budget),
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
function evidenceArray<T>(
|
|
1455
|
+
value: unknown,
|
|
1456
|
+
label: string,
|
|
1457
|
+
budget: ParseBudget,
|
|
1458
|
+
parse: (entry: unknown, label: string, budget: ParseBudget) => T,
|
|
1459
|
+
): readonly T[] {
|
|
1460
|
+
const input = dataArray(
|
|
1461
|
+
value,
|
|
1462
|
+
label,
|
|
1463
|
+
MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE,
|
|
1464
|
+
budget,
|
|
1465
|
+
);
|
|
1466
|
+
if (input.length === 0) throw new TypeError(`${label} must not be empty.`);
|
|
1467
|
+
const output = input.map((entry, index) =>
|
|
1468
|
+
parse(entry, `${label}[${index}]`, budget));
|
|
1469
|
+
const identities = new Set<string>();
|
|
1470
|
+
for (const entry of output) {
|
|
1471
|
+
const identity = JSON.stringify(entry);
|
|
1472
|
+
if (identities.has(identity)) throw new TypeError(`${label} must be unique.`);
|
|
1473
|
+
identities.add(identity);
|
|
1474
|
+
}
|
|
1475
|
+
return Object.freeze(output);
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
function parsedRelationProblem(
|
|
1479
|
+
value: unknown,
|
|
1480
|
+
label: string,
|
|
1481
|
+
budget: ParseBudget,
|
|
1482
|
+
): RelationHygieneProblem {
|
|
1483
|
+
const parsed = parsedText(value, label, budget);
|
|
1484
|
+
if (
|
|
1485
|
+
parsed !== "self-relation"
|
|
1486
|
+
&& parsed !== "reciprocal-relation"
|
|
1487
|
+
&& parsed !== "malformed-relation"
|
|
1488
|
+
&& parsed !== "broken-relation"
|
|
1489
|
+
&& parsed !== "ambiguous-relation"
|
|
1490
|
+
) throw new TypeError(`${label} is unsupported.`);
|
|
1491
|
+
return parsed;
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
function parsedCommonCandidate(
|
|
1495
|
+
record: DataRecord,
|
|
1496
|
+
label: string,
|
|
1497
|
+
): {
|
|
1498
|
+
readonly support: number;
|
|
1499
|
+
readonly evidenceTruncated: boolean;
|
|
1500
|
+
} {
|
|
1501
|
+
return {
|
|
1502
|
+
support: positiveSafeInteger(record.support, `${label}.support`),
|
|
1503
|
+
evidenceTruncated: parsedBoolean(
|
|
1504
|
+
record.evidenceTruncated,
|
|
1505
|
+
`${label}.evidenceTruncated`,
|
|
1506
|
+
),
|
|
1507
|
+
};
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
function parseCandidate(
|
|
1511
|
+
value: unknown,
|
|
1512
|
+
label: string,
|
|
1513
|
+
budget: ParseBudget,
|
|
1514
|
+
version: 1 | 2,
|
|
1515
|
+
): PercolationCandidateV1 | PercolationCandidateV2 {
|
|
1516
|
+
const record = dataRecord(value, label, budget);
|
|
1517
|
+
const kind = parsedText(record.kind, `${label}.kind`, budget);
|
|
1518
|
+
if (kind === "missing-concept") {
|
|
1519
|
+
exactKeys(
|
|
1520
|
+
record,
|
|
1521
|
+
[
|
|
1522
|
+
"kind",
|
|
1523
|
+
"tag",
|
|
1524
|
+
"suggestedId",
|
|
1525
|
+
"collidesWith",
|
|
1526
|
+
"support",
|
|
1527
|
+
"evidenceTruncated",
|
|
1528
|
+
"evidence",
|
|
1529
|
+
],
|
|
1530
|
+
label,
|
|
1531
|
+
);
|
|
1532
|
+
const common = parsedCommonCandidate(record, label);
|
|
1533
|
+
const tag = parsedText(record.tag, `${label}.tag`, budget);
|
|
1534
|
+
const evidence = evidenceArray(
|
|
1535
|
+
record.evidence,
|
|
1536
|
+
`${label}.evidence`,
|
|
1537
|
+
budget,
|
|
1538
|
+
parsedMissingConceptEvidence,
|
|
1539
|
+
);
|
|
1540
|
+
if (evidence.some((entry) => entry.tag !== tag)) {
|
|
1541
|
+
throw new TypeError(`${label}.evidence must support the candidate tag.`);
|
|
1542
|
+
}
|
|
1543
|
+
if (
|
|
1544
|
+
(!common.evidenceTruncated && common.support !== evidence.length)
|
|
1545
|
+
|| (common.evidenceTruncated && common.support <= evidence.length)
|
|
1546
|
+
) {
|
|
1547
|
+
throw new TypeError(`${label}.support does not match its bounded evidence.`);
|
|
1548
|
+
}
|
|
1549
|
+
return Object.freeze({
|
|
1550
|
+
kind: "missing-concept",
|
|
1551
|
+
tag,
|
|
1552
|
+
suggestedId: canonicalNote(record.suggestedId, `${label}.suggestedId`, budget),
|
|
1553
|
+
collidesWith: record.collidesWith === null
|
|
1554
|
+
? null
|
|
1555
|
+
: canonicalNote(record.collidesWith, `${label}.collidesWith`, budget),
|
|
1556
|
+
...common,
|
|
1557
|
+
evidence,
|
|
1558
|
+
});
|
|
1559
|
+
}
|
|
1560
|
+
if (kind === "missing-relation") {
|
|
1561
|
+
exactKeys(
|
|
1562
|
+
record,
|
|
1563
|
+
version === 1
|
|
1564
|
+
? [
|
|
1565
|
+
"kind",
|
|
1566
|
+
"source",
|
|
1567
|
+
"target",
|
|
1568
|
+
"suggestedPredicate",
|
|
1569
|
+
"support",
|
|
1570
|
+
"evidenceTruncated",
|
|
1571
|
+
"evidence",
|
|
1572
|
+
]
|
|
1573
|
+
: [
|
|
1574
|
+
"kind",
|
|
1575
|
+
"source",
|
|
1576
|
+
"target",
|
|
1577
|
+
"predicate",
|
|
1578
|
+
"support",
|
|
1579
|
+
"evidenceTruncated",
|
|
1580
|
+
"evidence",
|
|
1581
|
+
],
|
|
1582
|
+
label,
|
|
1583
|
+
);
|
|
1584
|
+
const source = canonicalNote(record.source, `${label}.source`, budget);
|
|
1585
|
+
const target = canonicalNote(record.target, `${label}.target`, budget);
|
|
1586
|
+
if (compareText(source, target) >= 0) {
|
|
1587
|
+
throw new TypeError(`${label} endpoints must be an ordered, distinct pair.`);
|
|
1588
|
+
}
|
|
1589
|
+
const common = parsedCommonCandidate(record, label);
|
|
1590
|
+
const evidence = evidenceArray(
|
|
1591
|
+
record.evidence,
|
|
1592
|
+
`${label}.evidence`,
|
|
1593
|
+
budget,
|
|
1594
|
+
parsedSharedEvidence,
|
|
1595
|
+
);
|
|
1596
|
+
if (evidence.some((entry) => entry.note !== source && entry.note !== target)) {
|
|
1597
|
+
throw new TypeError(`${label}.evidence must belong to one of the unordered endpoints.`);
|
|
1598
|
+
}
|
|
1599
|
+
const signalEndpoints = new Map<string, Set<string>>();
|
|
1600
|
+
for (const entry of evidence) {
|
|
1601
|
+
const signal = entry.kind === "shared-tag"
|
|
1602
|
+
? `tag\u0000${entry.tag}`
|
|
1603
|
+
: `concept\u0000${entry.concept}`;
|
|
1604
|
+
const endpoints = signalEndpoints.get(signal) ?? new Set<string>();
|
|
1605
|
+
endpoints.add(entry.note);
|
|
1606
|
+
signalEndpoints.set(signal, endpoints);
|
|
1607
|
+
}
|
|
1608
|
+
if ([...signalEndpoints.values()].some((endpoints) =>
|
|
1609
|
+
endpoints.size !== 2 || !endpoints.has(source) || !endpoints.has(target))) {
|
|
1610
|
+
throw new TypeError(`${label}.evidence must pair both unordered endpoints per signal.`);
|
|
1611
|
+
}
|
|
1612
|
+
if (
|
|
1613
|
+
(!common.evidenceTruncated && common.support !== signalEndpoints.size)
|
|
1614
|
+
|| (common.evidenceTruncated
|
|
1615
|
+
&& (
|
|
1616
|
+
evidence.length !== MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE
|
|
1617
|
+
|| common.support <= signalEndpoints.size
|
|
1618
|
+
))
|
|
1619
|
+
) {
|
|
1620
|
+
throw new TypeError(`${label}.support does not match its bounded shared signals.`);
|
|
1621
|
+
}
|
|
1622
|
+
if (version === 1) {
|
|
1623
|
+
if (
|
|
1624
|
+
parsedText(
|
|
1625
|
+
record.suggestedPredicate,
|
|
1626
|
+
`${label}.suggestedPredicate`,
|
|
1627
|
+
budget,
|
|
1628
|
+
) !== "related-to"
|
|
1629
|
+
) {
|
|
1630
|
+
throw new TypeError(`${label}.suggestedPredicate must be related-to.`);
|
|
1631
|
+
}
|
|
1632
|
+
return Object.freeze({
|
|
1633
|
+
kind: "missing-relation",
|
|
1634
|
+
source,
|
|
1635
|
+
target,
|
|
1636
|
+
suggestedPredicate: "related-to",
|
|
1637
|
+
...common,
|
|
1638
|
+
evidence,
|
|
1639
|
+
});
|
|
1640
|
+
}
|
|
1641
|
+
return Object.freeze({
|
|
1642
|
+
kind: "missing-relation",
|
|
1643
|
+
source,
|
|
1644
|
+
target,
|
|
1645
|
+
predicate: predicateDisposition(record.predicate, `${label}.predicate`, budget),
|
|
1646
|
+
...common,
|
|
1647
|
+
evidence,
|
|
1648
|
+
});
|
|
1649
|
+
}
|
|
1650
|
+
if (kind === "unlinked-mention") {
|
|
1651
|
+
exactKeys(
|
|
1652
|
+
record,
|
|
1653
|
+
["kind", "source", "target", "support", "evidenceTruncated", "evidence"],
|
|
1654
|
+
label,
|
|
1655
|
+
);
|
|
1656
|
+
const source = canonicalNote(record.source, `${label}.source`, budget);
|
|
1657
|
+
const target = canonicalNote(record.target, `${label}.target`, budget);
|
|
1658
|
+
const common = parsedCommonCandidate(record, label);
|
|
1659
|
+
const evidence = evidenceArray(
|
|
1660
|
+
record.evidence,
|
|
1661
|
+
`${label}.evidence`,
|
|
1662
|
+
budget,
|
|
1663
|
+
parsedMentionEvidence,
|
|
1664
|
+
);
|
|
1665
|
+
if (evidence.some((entry) => entry.source !== source || entry.target !== target)) {
|
|
1666
|
+
throw new TypeError(`${label}.evidence must identify the candidate endpoints.`);
|
|
1667
|
+
}
|
|
1668
|
+
if (
|
|
1669
|
+
(!common.evidenceTruncated && common.support !== evidence.length)
|
|
1670
|
+
|| (common.evidenceTruncated && common.support <= evidence.length)
|
|
1671
|
+
) {
|
|
1672
|
+
throw new TypeError(`${label}.support does not match its bounded evidence.`);
|
|
1673
|
+
}
|
|
1674
|
+
return Object.freeze({
|
|
1675
|
+
kind: "unlinked-mention",
|
|
1676
|
+
source,
|
|
1677
|
+
target,
|
|
1678
|
+
...common,
|
|
1679
|
+
evidence,
|
|
1680
|
+
});
|
|
1681
|
+
}
|
|
1682
|
+
if (kind === "relation-hygiene") {
|
|
1683
|
+
exactKeys(
|
|
1684
|
+
record,
|
|
1685
|
+
[
|
|
1686
|
+
"kind",
|
|
1687
|
+
"problem",
|
|
1688
|
+
"source",
|
|
1689
|
+
"target",
|
|
1690
|
+
"predicate",
|
|
1691
|
+
"message",
|
|
1692
|
+
"support",
|
|
1693
|
+
"evidenceTruncated",
|
|
1694
|
+
"evidence",
|
|
1695
|
+
],
|
|
1696
|
+
label,
|
|
1697
|
+
);
|
|
1698
|
+
const problem = parsedRelationProblem(record.problem, `${label}.problem`, budget);
|
|
1699
|
+
const source = canonicalNote(record.source, `${label}.source`, budget);
|
|
1700
|
+
const target = problem === "malformed-relation"
|
|
1701
|
+
? nullableText(record.target, `${label}.target`, budget)
|
|
1702
|
+
: record.target === null
|
|
1703
|
+
? null
|
|
1704
|
+
: canonicalNote(record.target, `${label}.target`, budget);
|
|
1705
|
+
const predicate = problem === "malformed-relation"
|
|
1706
|
+
? nullableText(record.predicate, `${label}.predicate`, budget)
|
|
1707
|
+
: record.predicate === null
|
|
1708
|
+
? null
|
|
1709
|
+
: canonicalPredicate(record.predicate, `${label}.predicate`, budget);
|
|
1710
|
+
const common = parsedCommonCandidate(record, label);
|
|
1711
|
+
const relationProblem = problem === "self-relation"
|
|
1712
|
+
|| problem === "reciprocal-relation";
|
|
1713
|
+
const evidence = relationProblem
|
|
1714
|
+
? evidenceArray(
|
|
1715
|
+
record.evidence,
|
|
1716
|
+
`${label}.evidence`,
|
|
1717
|
+
budget,
|
|
1718
|
+
parsedRelationEvidence,
|
|
1719
|
+
)
|
|
1720
|
+
: evidenceArray(
|
|
1721
|
+
record.evidence,
|
|
1722
|
+
`${label}.evidence`,
|
|
1723
|
+
budget,
|
|
1724
|
+
parsedRelationIssueEvidence,
|
|
1725
|
+
);
|
|
1726
|
+
if (common.support !== evidence.length) {
|
|
1727
|
+
throw new TypeError(`${label}.support must equal its hygiene evidence count.`);
|
|
1728
|
+
}
|
|
1729
|
+
if (relationProblem) {
|
|
1730
|
+
const relations = evidence as readonly RelationEvidence[];
|
|
1731
|
+
if (
|
|
1732
|
+
target === null
|
|
1733
|
+
|| predicate === null
|
|
1734
|
+
|| common.evidenceTruncated
|
|
1735
|
+
|| (problem === "self-relation" && target !== source)
|
|
1736
|
+
|| (problem === "reciprocal-relation"
|
|
1737
|
+
&& (compareText(source, target) >= 0 || relations.length !== 2))
|
|
1738
|
+
|| relations.some((entry) =>
|
|
1739
|
+
entry.predicate !== predicate
|
|
1740
|
+
|| (problem === "self-relation"
|
|
1741
|
+
? entry.source !== source || entry.target !== target
|
|
1742
|
+
: !(
|
|
1743
|
+
(entry.source === source && entry.target === target)
|
|
1744
|
+
|| (entry.source === target && entry.target === source)
|
|
1745
|
+
)))
|
|
1746
|
+
) throw new TypeError(`${label}.evidence must identify the hygiene relation.`);
|
|
1747
|
+
} else {
|
|
1748
|
+
const issues = evidence as readonly RelationIssueEvidence[];
|
|
1749
|
+
const expectedIssue = problem.slice(0, -"-relation".length);
|
|
1750
|
+
if (issues.some((entry) =>
|
|
1751
|
+
entry.source !== source
|
|
1752
|
+
|| entry.issue !== expectedIssue
|
|
1753
|
+
|| entry.predicate !== predicate
|
|
1754
|
+
|| entry.target !== target
|
|
1755
|
+
|| entry.message !== record.message)
|
|
1756
|
+
|| common.evidenceTruncated
|
|
1757
|
+
!== issues.some((entry) => entry.candidatesTruncated)) {
|
|
1758
|
+
throw new TypeError(`${label}.evidence must identify the hygiene issue.`);
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
return Object.freeze({
|
|
1762
|
+
kind: "relation-hygiene",
|
|
1763
|
+
problem,
|
|
1764
|
+
source,
|
|
1765
|
+
target,
|
|
1766
|
+
predicate,
|
|
1767
|
+
message: parsedText(record.message, `${label}.message`, budget),
|
|
1768
|
+
...common,
|
|
1769
|
+
evidence,
|
|
1770
|
+
});
|
|
1771
|
+
}
|
|
1772
|
+
throw new TypeError(`${label}.kind is unsupported.`);
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
function parsedCandidates<V extends 1 | 2>(
|
|
1776
|
+
value: unknown,
|
|
1777
|
+
label: string,
|
|
1778
|
+
budget: ParseBudget,
|
|
1779
|
+
version: V,
|
|
1780
|
+
): readonly (V extends 1 ? PercolationCandidateV1 : PercolationCandidateV2)[] {
|
|
1781
|
+
const input = dataArray(value, label, MAX_PERCOLATION_LIMIT, budget);
|
|
1782
|
+
const output = input.map((entry, index) =>
|
|
1783
|
+
parseCandidate(entry, `${label}[${index}]`, budget, version));
|
|
1784
|
+
if (version === 1) {
|
|
1785
|
+
const historical = output as readonly PercolationCandidateV1[];
|
|
1786
|
+
for (let index = 1; index < historical.length; index += 1) {
|
|
1787
|
+
const previous = historical[index - 1];
|
|
1788
|
+
const candidate = historical[index];
|
|
1789
|
+
if (
|
|
1790
|
+
previous !== undefined
|
|
1791
|
+
&& candidate !== undefined
|
|
1792
|
+
&& compareHistoricalCandidates(previous, candidate) > 0
|
|
1793
|
+
) {
|
|
1794
|
+
throw new TypeError(`${label} must use historical percolation ordering.`);
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
return Object.freeze([...historical]) as readonly (
|
|
1798
|
+
V extends 1 ? PercolationCandidateV1 : PercolationCandidateV2
|
|
1799
|
+
)[];
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
const identities = new Set<string>();
|
|
1803
|
+
const suggestedConceptIds = new Set<string>();
|
|
1804
|
+
for (let index = 0; index < output.length; index += 1) {
|
|
1805
|
+
const candidate = output[index];
|
|
1806
|
+
if (candidate === undefined) continue;
|
|
1807
|
+
const identity = `${candidate.kind}\u0000${candidateIdentity(candidate)}`;
|
|
1808
|
+
if (identities.has(identity)) throw new TypeError(`${label} must be unique.`);
|
|
1809
|
+
identities.add(identity);
|
|
1810
|
+
if (candidate.kind === "missing-concept") {
|
|
1811
|
+
if (suggestedConceptIds.has(candidate.suggestedId)) {
|
|
1812
|
+
throw new TypeError(`${label} must use unique suggested concept IDs.`);
|
|
1813
|
+
}
|
|
1814
|
+
suggestedConceptIds.add(candidate.suggestedId);
|
|
1815
|
+
}
|
|
1816
|
+
const previous = output[index - 1];
|
|
1817
|
+
if (previous !== undefined && compareCandidates(previous, candidate) > 0) {
|
|
1818
|
+
throw new TypeError(`${label} must use canonical percolation ordering.`);
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
return Object.freeze(output) as readonly (
|
|
1822
|
+
V extends 1 ? PercolationCandidateV1 : PercolationCandidateV2
|
|
1823
|
+
)[];
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
function parseResultFields<V extends 1 | 2>(
|
|
1827
|
+
record: DataRecord,
|
|
1828
|
+
label: string,
|
|
1829
|
+
budget: ParseBudget,
|
|
1830
|
+
version: V,
|
|
1831
|
+
): {
|
|
1832
|
+
readonly candidates: readonly (
|
|
1833
|
+
V extends 1 ? PercolationCandidateV1 : PercolationCandidateV2
|
|
1834
|
+
)[];
|
|
1835
|
+
readonly truncated: boolean;
|
|
1836
|
+
} {
|
|
1837
|
+
return {
|
|
1838
|
+
candidates: parsedCandidates(record.candidates, `${label}.candidates`, budget, version),
|
|
1839
|
+
truncated: parsedBoolean(record.truncated, `${label}.truncated`),
|
|
1840
|
+
};
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
/**
|
|
1844
|
+
* Parse only the exact historical unversioned core result shape. This parser
|
|
1845
|
+
* deliberately preserves `related-to`; it never guesses a V2 disposition.
|
|
1846
|
+
* @deprecated Retained through 0.19.x; removable no earlier than 0.20.0.
|
|
1847
|
+
*/
|
|
1848
|
+
export function parsePercolationResultV1(value: unknown): PercolationResultV1 {
|
|
1849
|
+
const budget: ParseBudget = { nodes: 0, utf8Bytes: 0 };
|
|
1850
|
+
const record = dataRecord(value, "percolation result v1", budget);
|
|
1851
|
+
exactKeys(record, ["candidates", "truncated"], "percolation result v1");
|
|
1852
|
+
const fields = parseResultFields(record, "percolation result v1", budget, 1);
|
|
1853
|
+
return Object.freeze({
|
|
1854
|
+
candidates: fields.candidates,
|
|
1855
|
+
truncated: fields.truncated,
|
|
1856
|
+
});
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
/** Parse only the exact V2 core result shape; CLI envelopes are rejected. */
|
|
1860
|
+
export function parsePercolationResultV2(value: unknown): PercolationResultV2 {
|
|
1861
|
+
const budget: ParseBudget = { nodes: 0, utf8Bytes: 0 };
|
|
1862
|
+
const record = dataRecord(value, "percolation result v2", budget);
|
|
1863
|
+
exactKeys(
|
|
1864
|
+
record,
|
|
1865
|
+
["schemaVersion", "candidates", "truncated"],
|
|
1866
|
+
"percolation result v2",
|
|
1867
|
+
);
|
|
1868
|
+
if (record.schemaVersion !== PERCOLATION_RESULT_SCHEMA_VERSION) {
|
|
1869
|
+
throw new TypeError("percolation result v2.schemaVersion must be 2.");
|
|
1870
|
+
}
|
|
1871
|
+
const fields = parseResultFields(record, "percolation result v2", budget, 2);
|
|
1872
|
+
return Object.freeze({
|
|
1873
|
+
schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION,
|
|
1874
|
+
candidates: fields.candidates,
|
|
1875
|
+
truncated: fields.truncated,
|
|
1876
|
+
});
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
export const parsePercolationResult = parsePercolationResultV2;
|
|
1880
|
+
|
|
1881
|
+
/**
|
|
1882
|
+
* Parse only the exact historical V1 CLI envelope without upgrading it.
|
|
1883
|
+
* @deprecated Retained through 0.19.x; removable no earlier than 0.20.0.
|
|
1884
|
+
*/
|
|
1885
|
+
export function parsePercolationCliOutputV1(value: unknown): PercolationCliOutputV1 {
|
|
1886
|
+
const budget: ParseBudget = { nodes: 0, utf8Bytes: 0 };
|
|
1887
|
+
const label = "percolation CLI output v1";
|
|
1888
|
+
const record = dataRecord(value, label, budget);
|
|
1889
|
+
exactKeys(
|
|
1890
|
+
record,
|
|
1891
|
+
["root", "note", "minSupport", "candidates", "truncated"],
|
|
1892
|
+
label,
|
|
1893
|
+
);
|
|
1894
|
+
const fields = parseResultFields(record, label, budget, 1);
|
|
1895
|
+
return Object.freeze({
|
|
1896
|
+
root: parsedText(record.root, `${label}.root`, budget),
|
|
1897
|
+
note: nullableText(record.note, `${label}.note`, budget),
|
|
1898
|
+
minSupport: parsedMinSupport(record.minSupport, `${label}.minSupport`),
|
|
1899
|
+
candidates: fields.candidates,
|
|
1900
|
+
truncated: fields.truncated,
|
|
1901
|
+
});
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
/** Parse only the exact current V2 CLI envelope. */
|
|
1905
|
+
export function parsePercolationCliOutputV2(value: unknown): PercolationCliOutputV2 {
|
|
1906
|
+
const budget: ParseBudget = { nodes: 0, utf8Bytes: 0 };
|
|
1907
|
+
const label = "percolation CLI output v2";
|
|
1908
|
+
const record = dataRecord(value, label, budget);
|
|
1909
|
+
exactKeys(
|
|
1910
|
+
record,
|
|
1911
|
+
[
|
|
1912
|
+
"root",
|
|
1913
|
+
"note",
|
|
1914
|
+
"minSupport",
|
|
1915
|
+
"limit",
|
|
1916
|
+
"schemaVersion",
|
|
1917
|
+
"candidates",
|
|
1918
|
+
"truncated",
|
|
1919
|
+
],
|
|
1920
|
+
label,
|
|
1921
|
+
);
|
|
1922
|
+
if (record.schemaVersion !== PERCOLATION_RESULT_SCHEMA_VERSION) {
|
|
1923
|
+
throw new TypeError(`${label}.schemaVersion must be 2.`);
|
|
1924
|
+
}
|
|
1925
|
+
const fields = parseResultFields(record, label, budget, 2);
|
|
1926
|
+
const limit = positiveSafeInteger(
|
|
1927
|
+
record.limit,
|
|
1928
|
+
`${label}.limit`,
|
|
1929
|
+
MAX_PERCOLATION_LIMIT,
|
|
1930
|
+
);
|
|
1931
|
+
if (
|
|
1932
|
+
fields.candidates.length > limit
|
|
1933
|
+
|| (fields.truncated && fields.candidates.length !== limit)
|
|
1934
|
+
) {
|
|
1935
|
+
throw new TypeError(`${label}.limit is inconsistent with its candidates.`);
|
|
1936
|
+
}
|
|
1937
|
+
return Object.freeze({
|
|
1938
|
+
root: parsedText(record.root, `${label}.root`, budget),
|
|
1939
|
+
note: nullableText(record.note, `${label}.note`, budget),
|
|
1940
|
+
minSupport: parsedMinSupport(record.minSupport, `${label}.minSupport`),
|
|
1941
|
+
limit,
|
|
1942
|
+
schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION,
|
|
1943
|
+
candidates: fields.candidates,
|
|
1944
|
+
truncated: fields.truncated,
|
|
1945
|
+
});
|
|
1946
|
+
}
|
|
1947
|
+
|
|
1948
|
+
export const parsePercolationCliOutput = parsePercolationCliOutputV2;
|