@h1v35/hivex 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -126
- package/docs/CONTEXT.md +20 -32
- package/docs/README.md +4 -8
- package/docs/adr/0003-independent-bun-installation.md +5 -19
- package/docs/adr/0010-practical-knowledge-assistance.md +28 -81
- package/docs/adr/0011-shared-knowledge-and-selective-history.md +27 -0
- package/docs/engineering.md +46 -142
- package/package.json +31 -10
- package/skills/hivex/SKILL.md +30 -72
- package/skills/hivex/references/markdown.md +16 -31
- package/src/cli/diagnostic.ts +21 -11
- package/src/cli.ts +50 -32
- package/src/documents.ts +522 -315
- package/src/errors.ts +8 -6
- package/src/implementation.ts +185 -87
- package/src/ingestion-units.ts +107 -64
- package/src/knowledge-maintenance.ts +35 -22
- package/src/knowledge-model.ts +386 -267
- package/src/knowledge-serialization.ts +239 -0
- package/src/knowledge-snapshot.ts +163 -0
- package/src/knowledge-store.ts +645 -442
- package/src/knowledge.ts +1073 -696
- package/src/markdown.ts +107 -45
- package/src/model/connection.ts +134 -76
- package/src/model/failure.ts +46 -23
- package/src/model/invoke.ts +346 -166
- package/src/model/profile.ts +201 -103
- package/src/model/rpc-error.ts +21 -0
- package/src/model/server.ts +151 -82
- package/src/model/thread.ts +24 -14
- package/src/model/transcript.ts +87 -46
- package/src/ordering.ts +9 -0
- package/src/retrieval/lexical.ts +64 -41
- package/src/review.ts +83 -55
- package/src/runtime.d.ts +4 -0
- package/src/snapshot-command.ts +105 -0
package/src/knowledge-model.ts
CHANGED
|
@@ -1,50 +1,85 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
+
import { withLiveProvenance } from './knowledge-serialization.ts';
|
|
3
4
|
import { rawMarkdownLines, sourceRange } from './markdown.ts';
|
|
4
5
|
import type { Document, Project } from './documents.ts';
|
|
5
6
|
|
|
6
7
|
export const digest = (value: string | Uint8Array) =>
|
|
7
8
|
createHash('sha256').update(value).digest('hex');
|
|
9
|
+
// JSON property lists preserve identities created before the lint migration.
|
|
10
|
+
const decisionIdentityFields = [
|
|
11
|
+
'version',
|
|
12
|
+
'id',
|
|
13
|
+
'document',
|
|
14
|
+
'text',
|
|
15
|
+
'kind',
|
|
16
|
+
'status',
|
|
17
|
+
'conditions',
|
|
18
|
+
'exceptions',
|
|
19
|
+
'reason',
|
|
20
|
+
'lineStart',
|
|
21
|
+
'lineEnd',
|
|
22
|
+
];
|
|
23
|
+
const relationshipIdentityFields = [
|
|
24
|
+
'id',
|
|
25
|
+
'from',
|
|
26
|
+
'to',
|
|
27
|
+
'type',
|
|
28
|
+
'reason',
|
|
29
|
+
'evidence',
|
|
30
|
+
'document',
|
|
31
|
+
'lineStart',
|
|
32
|
+
'lineEnd',
|
|
33
|
+
'version',
|
|
34
|
+
];
|
|
8
35
|
const explanation = z.string().min(1).max(2048);
|
|
9
36
|
export const citationSchema = z.object({
|
|
10
37
|
document: z.string().min(1),
|
|
11
|
-
lineStart: z.number().int().positive(),
|
|
12
38
|
lineEnd: z.number().int().positive(),
|
|
39
|
+
lineStart: z.number().int().positive(),
|
|
13
40
|
});
|
|
14
|
-
export
|
|
15
|
-
|
|
41
|
+
export interface SuppliedDocument {
|
|
42
|
+
id: string;
|
|
43
|
+
lines: (string | number)[][];
|
|
44
|
+
}
|
|
45
|
+
export const suppliedCitation = function suppliedCitation(
|
|
16
46
|
entry: z.infer<typeof citationSchema>,
|
|
17
|
-
documents: SuppliedDocument[]
|
|
47
|
+
documents: SuppliedDocument[]
|
|
18
48
|
) {
|
|
19
|
-
if (entry.lineEnd < entry.lineStart)
|
|
49
|
+
if (entry.lineEnd < entry.lineStart) {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
20
52
|
const lines = new Set(
|
|
21
53
|
documents
|
|
22
54
|
.filter((document) => document.id === entry.document)
|
|
23
|
-
.flatMap((document) => document.lines.map(([number]) => Number(number)))
|
|
55
|
+
.flatMap((document) => document.lines.map(([number]) => Number(number)))
|
|
24
56
|
);
|
|
25
|
-
for (let line = entry.lineStart; line <= entry.lineEnd; line += 1)
|
|
26
|
-
if (!lines.has(line))
|
|
57
|
+
for (let line = entry.lineStart; line <= entry.lineEnd; line += 1) {
|
|
58
|
+
if (!lines.has(line)) {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
27
62
|
return true;
|
|
28
|
-
}
|
|
63
|
+
};
|
|
29
64
|
export const decisionSchema = z.object({
|
|
30
|
-
id: z.string().min(1),
|
|
31
|
-
document: z.string().min(1),
|
|
32
|
-
text: explanation,
|
|
33
|
-
kind: z.enum(['decision', 'constraint', 'definition', 'lesson']),
|
|
34
|
-
status: z.enum(['current', 'proposed', 'historical', 'uncertain']),
|
|
35
65
|
conditions: z.array(explanation).max(16),
|
|
66
|
+
document: z.string().min(1),
|
|
36
67
|
exceptions: z.array(explanation).max(16),
|
|
37
|
-
|
|
38
|
-
|
|
68
|
+
id: z.string().min(1),
|
|
69
|
+
kind: z.enum(['decision', 'constraint', 'definition', 'lesson']),
|
|
39
70
|
lineEnd: z.number().int().positive(),
|
|
71
|
+
lineStart: z.number().int().positive(),
|
|
72
|
+
reason: explanation,
|
|
73
|
+
status: z.enum(['current', 'proposed', 'historical', 'uncertain']),
|
|
74
|
+
text: explanation,
|
|
40
75
|
});
|
|
41
76
|
export const relationshipSchema = z.object({
|
|
42
|
-
|
|
77
|
+
evidence: z.array(citationSchema).min(1).max(8),
|
|
43
78
|
from: z.string().min(1),
|
|
79
|
+
id: z.string().min(1),
|
|
80
|
+
reason: explanation,
|
|
44
81
|
to: z.string().min(1),
|
|
45
82
|
type: z.enum(['requires', 'exception-to', 'supersedes', 'supports', 'contradicts']),
|
|
46
|
-
reason: explanation,
|
|
47
|
-
evidence: z.array(citationSchema).min(1).max(8),
|
|
48
83
|
});
|
|
49
84
|
export const extractionSchema = z.object({
|
|
50
85
|
decisions: z.array(decisionSchema).max(64),
|
|
@@ -52,10 +87,15 @@ export const extractionSchema = z.object({
|
|
|
52
87
|
uncertainties: z.array(explanation).max(32),
|
|
53
88
|
});
|
|
54
89
|
export const checkSchema = z.object({
|
|
55
|
-
findings: z.array(z.object({ target: z.string().min(1)
|
|
90
|
+
findings: z.array(z.object({ reason: explanation, target: z.string().min(1) })).max(64),
|
|
56
91
|
});
|
|
57
92
|
const quality = z.enum(['unchecked', 'checked', 'uncertain']);
|
|
58
|
-
const provenance = {
|
|
93
|
+
const provenance = {
|
|
94
|
+
batch: z.string(),
|
|
95
|
+
localId: z.string(),
|
|
96
|
+
quality,
|
|
97
|
+
version: z.string(),
|
|
98
|
+
};
|
|
59
99
|
const warningScopeSchema = citationSchema.extend({ version: z.string() });
|
|
60
100
|
export type WarningScope = z.infer<typeof warningScopeSchema>;
|
|
61
101
|
const warningSchema = z.union([
|
|
@@ -63,66 +103,77 @@ const warningSchema = z.union([
|
|
|
63
103
|
z.object({ message: z.string(), scope: z.array(warningScopeSchema) }),
|
|
64
104
|
]);
|
|
65
105
|
|
|
66
|
-
|
|
106
|
+
const fullWarningScope = function fullWarningScope(document: Document) {
|
|
107
|
+
return {
|
|
108
|
+
document: document.id,
|
|
109
|
+
lineEnd: rawMarkdownLines(document.text).length,
|
|
110
|
+
lineStart: 1,
|
|
111
|
+
};
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
export const warningScope = function warningScope(
|
|
67
115
|
documents: Document[],
|
|
68
|
-
ranges?: z.infer<typeof citationSchema>[]
|
|
116
|
+
ranges?: z.infer<typeof citationSchema>[]
|
|
69
117
|
): WarningScope[] {
|
|
70
|
-
return (
|
|
71
|
-
ranges ??
|
|
72
|
-
documents.map((document) => ({
|
|
73
|
-
document: document.id,
|
|
74
|
-
lineStart: 1,
|
|
75
|
-
lineEnd: rawMarkdownLines(document.text).length,
|
|
76
|
-
}))
|
|
77
|
-
).flatMap((range) => {
|
|
118
|
+
return (ranges ?? documents.map(fullWarningScope)).flatMap((range) => {
|
|
78
119
|
const source = documents.find((document) => document.id === range.document);
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
120
|
+
if (source === undefined) {
|
|
121
|
+
return [];
|
|
122
|
+
}
|
|
123
|
+
return [
|
|
124
|
+
{
|
|
125
|
+
document: range.document,
|
|
126
|
+
lineEnd: range.lineEnd,
|
|
127
|
+
lineStart: range.lineStart,
|
|
128
|
+
version: source.hash,
|
|
129
|
+
},
|
|
130
|
+
];
|
|
89
131
|
});
|
|
90
|
-
}
|
|
132
|
+
};
|
|
91
133
|
|
|
92
134
|
export const graphSchema = z.object({
|
|
93
|
-
version: z.literal(1),
|
|
94
|
-
lastExtraction: z.string().optional(),
|
|
95
|
-
documents: z.record(z.string(), z.string()),
|
|
96
|
-
units: z
|
|
97
|
-
.record(
|
|
98
|
-
z.string(),
|
|
99
|
-
z.object({ document: z.string(), version: z.string(), workKey: z.string().optional() }),
|
|
100
|
-
)
|
|
101
|
-
.default({}),
|
|
102
135
|
decisions: z.array(decisionSchema.extend(provenance)),
|
|
136
|
+
documents: z.record(z.string(), z.string()),
|
|
137
|
+
lastExtraction: z.string().optional(),
|
|
103
138
|
relationships: z.array(
|
|
104
139
|
relationshipSchema.extend({
|
|
105
140
|
batch: z.string(),
|
|
141
|
+
evidence: z.array(citationSchema.extend({ version: z.string().optional() })),
|
|
106
142
|
localId: z.string(),
|
|
107
143
|
quality,
|
|
108
|
-
|
|
109
|
-
}),
|
|
144
|
+
})
|
|
110
145
|
),
|
|
146
|
+
units: z
|
|
147
|
+
.record(
|
|
148
|
+
z.string(),
|
|
149
|
+
z.object({
|
|
150
|
+
document: z.string(),
|
|
151
|
+
version: z.string(),
|
|
152
|
+
workKey: z.string().optional(),
|
|
153
|
+
})
|
|
154
|
+
)
|
|
155
|
+
.default({}),
|
|
156
|
+
version: z.literal(1),
|
|
111
157
|
warnings: z.array(warningSchema),
|
|
112
158
|
});
|
|
113
159
|
export type Graph = z.infer<typeof graphSchema>;
|
|
114
160
|
export type Extraction = z.infer<typeof extractionSchema>;
|
|
115
161
|
export type KnowledgeCheck = z.infer<typeof checkSchema>;
|
|
116
|
-
export const emptyGraph = (): Graph
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
162
|
+
export const emptyGraph = function emptyGraph(): Graph {
|
|
163
|
+
return {
|
|
164
|
+
decisions: [],
|
|
165
|
+
documents: {},
|
|
166
|
+
relationships: [],
|
|
167
|
+
units: {},
|
|
168
|
+
version: 1,
|
|
169
|
+
warnings: [],
|
|
170
|
+
};
|
|
171
|
+
};
|
|
124
172
|
|
|
125
|
-
export
|
|
173
|
+
export const validCitation = function validCitation(
|
|
174
|
+
entry: z.infer<typeof citationSchema>,
|
|
175
|
+
documents: Document[]
|
|
176
|
+
) {
|
|
126
177
|
const document = documents.find((item) => item.id === entry.document);
|
|
127
178
|
return (
|
|
128
179
|
document !== undefined &&
|
|
@@ -130,213 +181,296 @@ export function validCitation(entry: z.infer<typeof citationSchema>, documents:
|
|
|
130
181
|
entry.lineEnd <= rawMarkdownLines(document.text).length &&
|
|
131
182
|
sourceRange(document.text, entry.lineStart, entry.lineEnd).trim().length > 0
|
|
132
183
|
);
|
|
133
|
-
}
|
|
184
|
+
};
|
|
134
185
|
|
|
135
|
-
export
|
|
186
|
+
export const sourceEvidence = function sourceEvidence(
|
|
187
|
+
entry: z.infer<typeof citationSchema>,
|
|
188
|
+
project: Project
|
|
189
|
+
) {
|
|
136
190
|
const document = project.documents.find((item) => item.id === entry.document);
|
|
137
|
-
if (
|
|
191
|
+
if (document === undefined || !validCitation(entry, project.documents)) {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
138
194
|
return {
|
|
139
195
|
document: entry.document,
|
|
140
|
-
|
|
196
|
+
historical: document.historical,
|
|
141
197
|
lineEnd: entry.lineEnd,
|
|
142
|
-
|
|
198
|
+
lineStart: entry.lineStart,
|
|
143
199
|
text: sourceRange(document.text, entry.lineStart, entry.lineEnd),
|
|
200
|
+
version: document.hash,
|
|
144
201
|
};
|
|
145
|
-
}
|
|
202
|
+
};
|
|
146
203
|
|
|
147
|
-
function inRanges(
|
|
204
|
+
const inRanges = function inRanges(
|
|
148
205
|
entry: z.infer<typeof citationSchema>,
|
|
149
|
-
ranges?: z.infer<typeof citationSchema>[]
|
|
206
|
+
ranges?: z.infer<typeof citationSchema>[]
|
|
150
207
|
) {
|
|
151
|
-
if (
|
|
208
|
+
if (ranges === undefined) {
|
|
209
|
+
return true;
|
|
210
|
+
}
|
|
152
211
|
for (let line = entry.lineStart; line <= entry.lineEnd; line += 1) {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
)
|
|
212
|
+
const isCovered = ranges.some((range) => {
|
|
213
|
+
if (range.document !== entry.document) {
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
return range.lineStart <= line && range.lineEnd >= line;
|
|
217
|
+
});
|
|
218
|
+
if (!isCovered) {
|
|
159
219
|
return false;
|
|
220
|
+
}
|
|
160
221
|
}
|
|
161
222
|
return true;
|
|
162
|
-
}
|
|
223
|
+
};
|
|
163
224
|
|
|
164
|
-
|
|
165
|
-
graph: Graph;
|
|
166
|
-
extraction: Extraction;
|
|
167
|
-
documents: Document[];
|
|
225
|
+
interface ExtractionOptions {
|
|
168
226
|
batch: string;
|
|
169
227
|
contextDocuments?: Document[];
|
|
228
|
+
contextRanges?: z.infer<typeof citationSchema>[];
|
|
229
|
+
documents: Document[];
|
|
170
230
|
existingIds?: string[];
|
|
231
|
+
extraction: Extraction;
|
|
232
|
+
graph: Graph;
|
|
171
233
|
targetRanges?: z.infer<typeof citationSchema>[];
|
|
172
|
-
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const retainedWarnings = function retainedWarnings(graph: Graph, scope: WarningScope[]) {
|
|
237
|
+
return graph.warnings.filter((warning) => {
|
|
238
|
+
if (typeof warning === 'string') {
|
|
239
|
+
return true;
|
|
240
|
+
}
|
|
241
|
+
return warning.scope.every((old) => {
|
|
242
|
+
const hasOverlap = scope.some((current) => {
|
|
243
|
+
if (current.document !== old.document) {
|
|
244
|
+
return false;
|
|
245
|
+
}
|
|
246
|
+
return (
|
|
247
|
+
current.version !== old.version ||
|
|
248
|
+
(current.lineStart <= old.lineEnd && current.lineEnd >= old.lineStart)
|
|
249
|
+
);
|
|
250
|
+
});
|
|
251
|
+
return !hasOverlap;
|
|
252
|
+
});
|
|
253
|
+
});
|
|
173
254
|
};
|
|
174
255
|
|
|
175
|
-
function
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
256
|
+
const extractedRelationships = function extractedRelationships(input: {
|
|
257
|
+
options: ExtractionOptions;
|
|
258
|
+
decisions: Graph['decisions'];
|
|
259
|
+
ids: Map<string, string>;
|
|
260
|
+
warnings: string[];
|
|
261
|
+
}) {
|
|
262
|
+
const { options, decisions, ids, warnings } = input;
|
|
263
|
+
const { graph, extraction, documents, batch } = options;
|
|
264
|
+
const available = new Set(decisions.map((entry) => entry.id));
|
|
265
|
+
const relationships = graph.relationships.filter((entry) => {
|
|
266
|
+
const hasChangedEvidence = entry.evidence.some((citation) => {
|
|
267
|
+
const isTargetRange =
|
|
268
|
+
options.targetRanges?.some((range) => {
|
|
269
|
+
if (range.document !== citation.document) {
|
|
270
|
+
return false;
|
|
271
|
+
}
|
|
272
|
+
return range.lineStart <= citation.lineEnd && range.lineEnd >= citation.lineStart;
|
|
273
|
+
}) === true;
|
|
274
|
+
const hasChangedDocument = (options.contextDocuments ?? documents).some((document) => {
|
|
275
|
+
if (document.id !== citation.document) {
|
|
276
|
+
return false;
|
|
277
|
+
}
|
|
278
|
+
return document.hash !== citation.version;
|
|
279
|
+
});
|
|
280
|
+
return isTargetRange || hasChangedDocument;
|
|
281
|
+
});
|
|
282
|
+
return available.has(entry.from) && available.has(entry.to) && !hasChangedEvidence;
|
|
283
|
+
});
|
|
284
|
+
const seen = new Set<string>();
|
|
285
|
+
for (const entry of extraction.relationships) {
|
|
286
|
+
const from = ids.get(entry.from);
|
|
287
|
+
const to = ids.get(entry.to);
|
|
288
|
+
const hasInvalidEvidence = entry.evidence.some((item) => {
|
|
289
|
+
if (!validCitation(item, options.contextDocuments ?? documents)) {
|
|
290
|
+
return true;
|
|
291
|
+
}
|
|
292
|
+
return !inRanges(item, options.contextRanges);
|
|
293
|
+
});
|
|
294
|
+
const hasMissingEndpoint = from === undefined || to === undefined;
|
|
295
|
+
const isDuplicate = seen.has(entry.id);
|
|
296
|
+
if (hasMissingEndpoint || isDuplicate || hasInvalidEvidence) {
|
|
297
|
+
warnings.push(
|
|
298
|
+
`Relationship ${entry.id} has an unknown endpoint, duplicate ID or invalid reference.`
|
|
299
|
+
);
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
seen.add(entry.id);
|
|
303
|
+
const evidence = entry.evidence.map((citation) => {
|
|
304
|
+
const source = (options.contextDocuments ?? documents).find(
|
|
305
|
+
(document) => document.id === citation.document
|
|
306
|
+
);
|
|
307
|
+
return { ...citation, version: source?.hash };
|
|
308
|
+
});
|
|
309
|
+
const id = digest(JSON.stringify({ ...entry, evidence, from, to }, relationshipIdentityFields));
|
|
310
|
+
const previous = relationships.findIndex((relationship) => relationship.id === id);
|
|
311
|
+
if (previous !== -1) {
|
|
312
|
+
relationships.splice(previous, 1);
|
|
313
|
+
}
|
|
314
|
+
relationships.push(
|
|
315
|
+
withLiveProvenance<Graph['relationships'][number]>(
|
|
316
|
+
{
|
|
317
|
+
...entry,
|
|
318
|
+
batch,
|
|
319
|
+
evidence,
|
|
320
|
+
from,
|
|
321
|
+
id,
|
|
322
|
+
localId: entry.id,
|
|
323
|
+
quality: 'unchecked',
|
|
324
|
+
to,
|
|
325
|
+
},
|
|
326
|
+
'relationship'
|
|
327
|
+
)
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
return relationships;
|
|
331
|
+
};
|
|
189
332
|
|
|
190
|
-
export function applyExtraction(options: ExtractionOptions) {
|
|
333
|
+
export const applyExtraction = function applyExtraction(options: ExtractionOptions) {
|
|
191
334
|
const { graph, extraction, documents, batch } = options;
|
|
192
335
|
const decisions = graph.decisions.filter((entry) => {
|
|
193
336
|
const source = documents.find((document) => document.id === entry.document);
|
|
337
|
+
const isTargetRange =
|
|
338
|
+
options.targetRanges?.some((range) => {
|
|
339
|
+
if (range.document !== entry.document) {
|
|
340
|
+
return false;
|
|
341
|
+
}
|
|
342
|
+
return range.lineStart <= entry.lineEnd && range.lineEnd >= entry.lineStart;
|
|
343
|
+
}) === true;
|
|
194
344
|
return (
|
|
195
|
-
|
|
196
|
-
(source.hash === entry.version &&
|
|
197
|
-
validCitation(entry, [source]) &&
|
|
198
|
-
!options.targetRanges?.some(
|
|
199
|
-
(range) =>
|
|
200
|
-
range.document === entry.document &&
|
|
201
|
-
range.lineStart <= entry.lineEnd &&
|
|
202
|
-
range.lineEnd >= entry.lineStart,
|
|
203
|
-
))
|
|
345
|
+
source === undefined ||
|
|
346
|
+
(source.hash === entry.version && validCitation(entry, [source]) && !isTargetRange)
|
|
204
347
|
);
|
|
205
348
|
});
|
|
206
349
|
const ids = new Map(
|
|
207
350
|
decisions
|
|
208
|
-
.filter((entry) => options.existingIds?.includes(entry.id))
|
|
209
|
-
.map((entry) => [entry.id, entry.id])
|
|
351
|
+
.filter((entry) => options.existingIds?.includes(entry.id) === true)
|
|
352
|
+
.map((entry) => [entry.id, entry.id])
|
|
210
353
|
);
|
|
211
354
|
const warnings = [...extraction.uncertainties];
|
|
212
355
|
for (const entry of extraction.decisions) {
|
|
213
356
|
const source = documents.find((document) => document.id === entry.document);
|
|
214
|
-
if (
|
|
357
|
+
if (source === undefined || ids.has(entry.id)) {
|
|
215
358
|
warnings.push(`Decision ${entry.id} has an unknown, duplicate or invalid source reference.`);
|
|
216
359
|
continue;
|
|
217
360
|
}
|
|
218
|
-
const
|
|
219
|
-
if (!
|
|
361
|
+
const isLocated = validCitation(entry, documents) && inRanges(entry, options.targetRanges);
|
|
362
|
+
if (!isLocated) {
|
|
220
363
|
warnings.push(
|
|
221
|
-
`Decision ${entry.id} has an unverified line range; its document remains available
|
|
364
|
+
`Decision ${entry.id} has an unverified line range; its document remains available.`
|
|
222
365
|
);
|
|
223
|
-
|
|
366
|
+
}
|
|
367
|
+
const id = digest(JSON.stringify({ version: source.hash, ...entry }, decisionIdentityFields));
|
|
224
368
|
ids.set(entry.id, id);
|
|
225
369
|
const previous = decisions.findIndex((decision) => decision.id === id);
|
|
226
|
-
if (previous
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
370
|
+
if (previous !== -1) {
|
|
371
|
+
decisions.splice(previous, 1);
|
|
372
|
+
}
|
|
373
|
+
decisions.push(
|
|
374
|
+
withLiveProvenance<Graph['decisions'][number]>(
|
|
375
|
+
{
|
|
376
|
+
...entry,
|
|
377
|
+
batch,
|
|
378
|
+
id,
|
|
379
|
+
localId: entry.id,
|
|
380
|
+
quality: isLocated ? 'unchecked' : 'uncertain',
|
|
381
|
+
version: source.hash,
|
|
382
|
+
},
|
|
383
|
+
'decision'
|
|
384
|
+
)
|
|
385
|
+
);
|
|
235
386
|
}
|
|
236
|
-
const relationships = extractedRelationships({
|
|
387
|
+
const relationships = extractedRelationships({
|
|
388
|
+
decisions,
|
|
389
|
+
ids,
|
|
390
|
+
options,
|
|
391
|
+
warnings,
|
|
392
|
+
});
|
|
237
393
|
return {
|
|
238
|
-
|
|
239
|
-
lastExtraction: batch,
|
|
394
|
+
decisions,
|
|
240
395
|
documents: Object.fromEntries(
|
|
241
|
-
Object.entries(graph.documents).filter(
|
|
242
|
-
(
|
|
243
|
-
|
|
244
|
-
|
|
396
|
+
Object.entries(graph.documents).filter(([id, version]) => {
|
|
397
|
+
const hasChanged = documents.some((document) => {
|
|
398
|
+
if (document.id !== id) {
|
|
399
|
+
return false;
|
|
400
|
+
}
|
|
401
|
+
return document.hash !== version;
|
|
402
|
+
});
|
|
403
|
+
return !hasChanged;
|
|
404
|
+
})
|
|
245
405
|
),
|
|
406
|
+
lastExtraction: batch,
|
|
407
|
+
relationships,
|
|
246
408
|
units: Object.fromEntries(
|
|
247
|
-
Object.entries(graph.units).filter(
|
|
248
|
-
(
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
409
|
+
Object.entries(graph.units).filter(([, unit]) => {
|
|
410
|
+
const hasChanged = documents.some((document) => {
|
|
411
|
+
if (document.id !== unit.document) {
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
return document.hash !== unit.version;
|
|
415
|
+
});
|
|
416
|
+
return !hasChanged;
|
|
417
|
+
})
|
|
253
418
|
),
|
|
254
|
-
|
|
255
|
-
relationships,
|
|
419
|
+
version: 1 as const,
|
|
256
420
|
warnings: [
|
|
257
421
|
...retainedWarnings(graph, warningScope(documents, options.targetRanges)),
|
|
258
|
-
...warnings.map((message) =>
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
})
|
|
422
|
+
...warnings.map((message) => {
|
|
423
|
+
const scope = warningScope(documents, options.targetRanges);
|
|
424
|
+
return { message, scope };
|
|
425
|
+
}),
|
|
262
426
|
],
|
|
263
427
|
};
|
|
264
|
-
}
|
|
428
|
+
};
|
|
265
429
|
|
|
266
|
-
function
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
const available = new Set(decisions.map((entry) => entry.id));
|
|
275
|
-
const relationships = graph.relationships.filter(
|
|
276
|
-
(entry) =>
|
|
277
|
-
available.has(entry.from) &&
|
|
278
|
-
available.has(entry.to) &&
|
|
279
|
-
!entry.evidence.some(
|
|
280
|
-
(citation) =>
|
|
281
|
-
options.targetRanges?.some(
|
|
282
|
-
(range) =>
|
|
283
|
-
range.document === citation.document &&
|
|
284
|
-
range.lineStart <= citation.lineEnd &&
|
|
285
|
-
range.lineEnd >= citation.lineStart,
|
|
286
|
-
) ||
|
|
287
|
-
(options.contextDocuments ?? documents).some(
|
|
288
|
-
(document) => document.id === citation.document && document.hash !== citation.version,
|
|
289
|
-
),
|
|
290
|
-
),
|
|
291
|
-
);
|
|
292
|
-
const seen = new Set<string>();
|
|
293
|
-
for (const entry of extraction.relationships) {
|
|
294
|
-
const from = ids.get(entry.from);
|
|
295
|
-
const to = ids.get(entry.to);
|
|
296
|
-
if (
|
|
297
|
-
!from ||
|
|
298
|
-
!to ||
|
|
299
|
-
seen.has(entry.id) ||
|
|
300
|
-
entry.evidence.some(
|
|
301
|
-
(item) =>
|
|
302
|
-
!validCitation(item, options.contextDocuments ?? documents) ||
|
|
303
|
-
!inRanges(item, options.contextRanges),
|
|
304
|
-
)
|
|
305
|
-
) {
|
|
306
|
-
warnings.push(
|
|
307
|
-
`Relationship ${entry.id} has an unknown endpoint, duplicate ID or invalid reference.`,
|
|
308
|
-
);
|
|
309
|
-
continue;
|
|
430
|
+
const findingScope = function findingScope(
|
|
431
|
+
graph: Graph,
|
|
432
|
+
target: string,
|
|
433
|
+
{ batch, fallback }: { batch: string; fallback: WarningScope[] }
|
|
434
|
+
): WarningScope[] {
|
|
435
|
+
const decisions = graph.decisions.filter((entry) => {
|
|
436
|
+
if (entry.id === target) {
|
|
437
|
+
return true;
|
|
310
438
|
}
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
from,
|
|
326
|
-
to,
|
|
327
|
-
localId: entry.id,
|
|
328
|
-
batch,
|
|
329
|
-
quality: 'unchecked',
|
|
439
|
+
return entry.batch === batch && (entry.localId === target || entry.document === target);
|
|
440
|
+
});
|
|
441
|
+
const relationships = graph.relationships.filter((entry) => {
|
|
442
|
+
if (entry.id === target) {
|
|
443
|
+
return true;
|
|
444
|
+
}
|
|
445
|
+
return entry.batch === batch && entry.localId === target;
|
|
446
|
+
});
|
|
447
|
+
const evidenceScopeFor = function evidenceScopeFor(entry: Graph['relationships'][number]) {
|
|
448
|
+
return entry.evidence.flatMap((citation) => {
|
|
449
|
+
if (citation.version === undefined) {
|
|
450
|
+
return [];
|
|
451
|
+
}
|
|
452
|
+
return [{ ...citation, version: citation.version }];
|
|
330
453
|
});
|
|
454
|
+
};
|
|
455
|
+
const relationshipScope = relationships.flatMap(evidenceScopeFor);
|
|
456
|
+
const scope = [
|
|
457
|
+
...decisions.map((entry) => {
|
|
458
|
+
const { document, lineEnd, lineStart, version } = entry;
|
|
459
|
+
return { document, lineEnd, lineStart, version };
|
|
460
|
+
}),
|
|
461
|
+
...relationshipScope,
|
|
462
|
+
];
|
|
463
|
+
if (scope.length > 0) {
|
|
464
|
+
return scope;
|
|
331
465
|
}
|
|
332
|
-
|
|
333
|
-
|
|
466
|
+
const documentScope = fallback.filter((entry) => entry.document === target);
|
|
467
|
+
return documentScope.length > 0 ? documentScope : fallback;
|
|
468
|
+
};
|
|
334
469
|
|
|
335
|
-
export function applyCheck(
|
|
470
|
+
export const applyCheck = function applyCheck(
|
|
336
471
|
graph: Graph,
|
|
337
472
|
check: KnowledgeCheck,
|
|
338
|
-
batch: string
|
|
339
|
-
scope: WarningScope[] = [],
|
|
473
|
+
{ batch, scope = [] }: { batch: string; scope?: WarningScope[] }
|
|
340
474
|
): Graph {
|
|
341
475
|
const targets = new Set(check.findings.map((finding) => finding.target));
|
|
342
476
|
const known = new Set([
|
|
@@ -344,32 +478,46 @@ export function applyCheck(
|
|
|
344
478
|
...scope.map((entry) => entry.document),
|
|
345
479
|
...graph.decisions.map((entry) => entry.id),
|
|
346
480
|
...graph.relationships.map((entry) => entry.id),
|
|
347
|
-
...graph.decisions
|
|
348
|
-
|
|
349
|
-
|
|
481
|
+
...graph.decisions.flatMap((entry) => {
|
|
482
|
+
if (entry.batch !== batch) {
|
|
483
|
+
return [];
|
|
484
|
+
}
|
|
485
|
+
return [entry.localId, entry.document];
|
|
486
|
+
}),
|
|
350
487
|
...graph.relationships.filter((entry) => entry.batch === batch).map((entry) => entry.localId),
|
|
351
488
|
]);
|
|
352
|
-
const
|
|
489
|
+
const isUncertainBatch =
|
|
490
|
+
targets.has('batch') || [...targets].some((target) => !known.has(target));
|
|
353
491
|
const decisions = graph.decisions.map((entry) => {
|
|
354
|
-
if (entry.batch !== batch && !targets.has(entry.id))
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
492
|
+
if (entry.batch !== batch && !targets.has(entry.id)) {
|
|
493
|
+
return entry;
|
|
494
|
+
}
|
|
495
|
+
const isTargeted =
|
|
496
|
+
targets.has(entry.id) || targets.has(entry.localId) || targets.has(entry.document);
|
|
497
|
+
const isUncertain = isTargeted || entry.quality === 'uncertain' || isUncertainBatch;
|
|
498
|
+
return {
|
|
499
|
+
...entry,
|
|
500
|
+
quality: quality.parse(isUncertain ? 'uncertain' : 'checked'),
|
|
501
|
+
};
|
|
362
502
|
});
|
|
363
503
|
const relationships = graph.relationships.map((entry) => {
|
|
364
|
-
if (entry.batch !== batch && !targets.has(entry.id))
|
|
365
|
-
|
|
504
|
+
if (entry.batch !== batch && !targets.has(entry.id)) {
|
|
505
|
+
return entry;
|
|
506
|
+
}
|
|
507
|
+
const isUncertain =
|
|
366
508
|
targets.has(entry.id) ||
|
|
367
|
-
|
|
509
|
+
isUncertainBatch ||
|
|
368
510
|
targets.has(entry.localId) ||
|
|
369
|
-
decisions.some(
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
511
|
+
decisions.some((node) => {
|
|
512
|
+
if (node.id !== entry.from && node.id !== entry.to) {
|
|
513
|
+
return false;
|
|
514
|
+
}
|
|
515
|
+
return node.quality === 'uncertain';
|
|
516
|
+
});
|
|
517
|
+
return {
|
|
518
|
+
...entry,
|
|
519
|
+
quality: quality.parse(isUncertain ? 'uncertain' : 'checked'),
|
|
520
|
+
};
|
|
373
521
|
});
|
|
374
522
|
return {
|
|
375
523
|
...graph,
|
|
@@ -377,42 +525,13 @@ export function applyCheck(
|
|
|
377
525
|
relationships,
|
|
378
526
|
warnings: [
|
|
379
527
|
...graph.warnings,
|
|
380
|
-
...check.findings.map((finding) =>
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
528
|
+
...check.findings.map((finding) => {
|
|
529
|
+
const findingWarningScope = findingScope(graph, finding.target, {
|
|
530
|
+
batch,
|
|
531
|
+
fallback: scope,
|
|
532
|
+
});
|
|
533
|
+
return { message: finding.reason, scope: findingWarningScope };
|
|
534
|
+
}),
|
|
384
535
|
],
|
|
385
536
|
};
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
function findingScope(
|
|
389
|
-
graph: Graph,
|
|
390
|
-
target: string,
|
|
391
|
-
batch: string,
|
|
392
|
-
fallback: WarningScope[],
|
|
393
|
-
): WarningScope[] {
|
|
394
|
-
const decisions = graph.decisions.filter(
|
|
395
|
-
(entry) =>
|
|
396
|
-
entry.id === target ||
|
|
397
|
-
(entry.batch === batch && (entry.localId === target || entry.document === target)),
|
|
398
|
-
);
|
|
399
|
-
const relationships = graph.relationships.filter(
|
|
400
|
-
(entry) => entry.id === target || (entry.batch === batch && entry.localId === target),
|
|
401
|
-
);
|
|
402
|
-
const scope = [
|
|
403
|
-
...decisions.map(({ document, version, lineStart, lineEnd }) => ({
|
|
404
|
-
document,
|
|
405
|
-
version,
|
|
406
|
-
lineStart,
|
|
407
|
-
lineEnd,
|
|
408
|
-
})),
|
|
409
|
-
...relationships.flatMap((entry) =>
|
|
410
|
-
entry.evidence.flatMap((citation) =>
|
|
411
|
-
citation.version ? [{ ...citation, version: citation.version }] : [],
|
|
412
|
-
),
|
|
413
|
-
),
|
|
414
|
-
];
|
|
415
|
-
const documentScope = fallback.filter((entry) => entry.document === target);
|
|
416
|
-
if (scope.length) return scope;
|
|
417
|
-
return documentScope.length ? documentScope : fallback;
|
|
418
|
-
}
|
|
537
|
+
};
|