@h1v35/hivex 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -163
- package/docs/CONTEXT.md +20 -36
- package/docs/README.md +6 -12
- package/docs/adr/0003-independent-bun-installation.md +5 -19
- package/docs/adr/0010-practical-knowledge-assistance.md +28 -81
- package/docs/adr/0011-shared-knowledge-and-selective-history.md +16 -43
- package/docs/guidelines/engineering.md +74 -0
- package/docs/procedures/self-hosted-runner.md +7 -0
- package/package.json +32 -11
- package/skills/hivex/SKILL.md +28 -92
- package/skills/hivex/references/markdown.md +12 -42
- package/src/cli/diagnostic.ts +21 -11
- package/src/cli.ts +46 -36
- package/src/documents.ts +502 -320
- package/src/errors.ts +8 -6
- package/src/implementation.ts +185 -87
- package/src/ingestion-units.ts +107 -64
- package/src/knowledge-maintenance.ts +35 -22
- package/src/knowledge-model.ts +386 -268
- package/src/knowledge-serialization.ts +239 -0
- package/src/knowledge-snapshot.ts +100 -77
- package/src/knowledge-store.ts +634 -453
- package/src/knowledge.ts +1001 -758
- package/src/markdown.ts +107 -45
- package/src/model/connection.ts +134 -76
- package/src/model/failure.ts +46 -23
- package/src/model/invoke.ts +346 -166
- package/src/model/profile.ts +201 -103
- package/src/model/rpc-error.ts +21 -0
- package/src/model/server.ts +151 -82
- package/src/model/thread.ts +24 -14
- package/src/model/transcript.ts +87 -46
- package/src/ordering.ts +9 -0
- package/src/retrieval/lexical.ts +64 -41
- package/src/review.ts +83 -55
- package/src/runtime.d.ts +4 -0
- package/src/snapshot-command.ts +82 -43
- package/src/source-relocation.ts +222 -0
- package/docs/engineering.md +0 -174
package/src/knowledge-model.ts
CHANGED
|
@@ -1,50 +1,85 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
+
import { withLiveProvenance } from './knowledge-serialization.ts';
|
|
3
4
|
import { rawMarkdownLines, sourceRange } from './markdown.ts';
|
|
4
5
|
import type { Document, Project } from './documents.ts';
|
|
5
6
|
|
|
6
7
|
export const digest = (value: string | Uint8Array) =>
|
|
7
8
|
createHash('sha256').update(value).digest('hex');
|
|
9
|
+
// JSON property lists preserve identities created before the lint migration.
|
|
10
|
+
const decisionIdentityFields = [
|
|
11
|
+
'version',
|
|
12
|
+
'id',
|
|
13
|
+
'document',
|
|
14
|
+
'text',
|
|
15
|
+
'kind',
|
|
16
|
+
'status',
|
|
17
|
+
'conditions',
|
|
18
|
+
'exceptions',
|
|
19
|
+
'reason',
|
|
20
|
+
'lineStart',
|
|
21
|
+
'lineEnd',
|
|
22
|
+
];
|
|
23
|
+
const relationshipIdentityFields = [
|
|
24
|
+
'id',
|
|
25
|
+
'from',
|
|
26
|
+
'to',
|
|
27
|
+
'type',
|
|
28
|
+
'reason',
|
|
29
|
+
'evidence',
|
|
30
|
+
'document',
|
|
31
|
+
'lineStart',
|
|
32
|
+
'lineEnd',
|
|
33
|
+
'version',
|
|
34
|
+
];
|
|
8
35
|
const explanation = z.string().min(1).max(2048);
|
|
9
36
|
export const citationSchema = z.object({
|
|
10
37
|
document: z.string().min(1),
|
|
11
|
-
lineStart: z.number().int().positive(),
|
|
12
38
|
lineEnd: z.number().int().positive(),
|
|
39
|
+
lineStart: z.number().int().positive(),
|
|
13
40
|
});
|
|
14
|
-
export
|
|
15
|
-
|
|
41
|
+
export interface SuppliedDocument {
|
|
42
|
+
id: string;
|
|
43
|
+
lines: (string | number)[][];
|
|
44
|
+
}
|
|
45
|
+
export const suppliedCitation = function suppliedCitation(
|
|
16
46
|
entry: z.infer<typeof citationSchema>,
|
|
17
|
-
documents: SuppliedDocument[]
|
|
47
|
+
documents: SuppliedDocument[]
|
|
18
48
|
) {
|
|
19
|
-
if (entry.lineEnd < entry.lineStart)
|
|
49
|
+
if (entry.lineEnd < entry.lineStart) {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
20
52
|
const lines = new Set(
|
|
21
53
|
documents
|
|
22
54
|
.filter((document) => document.id === entry.document)
|
|
23
|
-
.flatMap((document) => document.lines.map(([number]) => Number(number)))
|
|
55
|
+
.flatMap((document) => document.lines.map(([number]) => Number(number)))
|
|
24
56
|
);
|
|
25
|
-
for (let line = entry.lineStart; line <= entry.lineEnd; line += 1)
|
|
26
|
-
if (!lines.has(line))
|
|
57
|
+
for (let line = entry.lineStart; line <= entry.lineEnd; line += 1) {
|
|
58
|
+
if (!lines.has(line)) {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
27
62
|
return true;
|
|
28
|
-
}
|
|
63
|
+
};
|
|
29
64
|
export const decisionSchema = z.object({
|
|
30
|
-
id: z.string().min(1),
|
|
31
|
-
document: z.string().min(1),
|
|
32
|
-
text: explanation,
|
|
33
|
-
kind: z.enum(['decision', 'constraint', 'definition', 'lesson']),
|
|
34
|
-
status: z.enum(['current', 'proposed', 'historical', 'uncertain']),
|
|
35
65
|
conditions: z.array(explanation).max(16),
|
|
66
|
+
document: z.string().min(1),
|
|
36
67
|
exceptions: z.array(explanation).max(16),
|
|
37
|
-
|
|
38
|
-
|
|
68
|
+
id: z.string().min(1),
|
|
69
|
+
kind: z.enum(['decision', 'constraint', 'definition', 'lesson']),
|
|
39
70
|
lineEnd: z.number().int().positive(),
|
|
71
|
+
lineStart: z.number().int().positive(),
|
|
72
|
+
reason: explanation,
|
|
73
|
+
status: z.enum(['current', 'proposed', 'historical', 'uncertain']),
|
|
74
|
+
text: explanation,
|
|
40
75
|
});
|
|
41
76
|
export const relationshipSchema = z.object({
|
|
42
|
-
|
|
77
|
+
evidence: z.array(citationSchema).min(1).max(8),
|
|
43
78
|
from: z.string().min(1),
|
|
79
|
+
id: z.string().min(1),
|
|
80
|
+
reason: explanation,
|
|
44
81
|
to: z.string().min(1),
|
|
45
82
|
type: z.enum(['requires', 'exception-to', 'supersedes', 'supports', 'contradicts']),
|
|
46
|
-
reason: explanation,
|
|
47
|
-
evidence: z.array(citationSchema).min(1).max(8),
|
|
48
83
|
});
|
|
49
84
|
export const extractionSchema = z.object({
|
|
50
85
|
decisions: z.array(decisionSchema).max(64),
|
|
@@ -52,10 +87,15 @@ export const extractionSchema = z.object({
|
|
|
52
87
|
uncertainties: z.array(explanation).max(32),
|
|
53
88
|
});
|
|
54
89
|
export const checkSchema = z.object({
|
|
55
|
-
findings: z.array(z.object({ target: z.string().min(1)
|
|
90
|
+
findings: z.array(z.object({ reason: explanation, target: z.string().min(1) })).max(64),
|
|
56
91
|
});
|
|
57
92
|
const quality = z.enum(['unchecked', 'checked', 'uncertain']);
|
|
58
|
-
const provenance = {
|
|
93
|
+
const provenance = {
|
|
94
|
+
batch: z.string(),
|
|
95
|
+
localId: z.string(),
|
|
96
|
+
quality,
|
|
97
|
+
version: z.string(),
|
|
98
|
+
};
|
|
59
99
|
const warningScopeSchema = citationSchema.extend({ version: z.string() });
|
|
60
100
|
export type WarningScope = z.infer<typeof warningScopeSchema>;
|
|
61
101
|
const warningSchema = z.union([
|
|
@@ -63,66 +103,77 @@ const warningSchema = z.union([
|
|
|
63
103
|
z.object({ message: z.string(), scope: z.array(warningScopeSchema) }),
|
|
64
104
|
]);
|
|
65
105
|
|
|
66
|
-
|
|
106
|
+
const fullWarningScope = function fullWarningScope(document: Document) {
|
|
107
|
+
return {
|
|
108
|
+
document: document.id,
|
|
109
|
+
lineEnd: rawMarkdownLines(document.text).length,
|
|
110
|
+
lineStart: 1,
|
|
111
|
+
};
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
export const warningScope = function warningScope(
|
|
67
115
|
documents: Document[],
|
|
68
|
-
ranges?: z.infer<typeof citationSchema>[]
|
|
116
|
+
ranges?: z.infer<typeof citationSchema>[]
|
|
69
117
|
): WarningScope[] {
|
|
70
|
-
return (
|
|
71
|
-
ranges ??
|
|
72
|
-
documents.map((document) => ({
|
|
73
|
-
document: document.id,
|
|
74
|
-
lineStart: 1,
|
|
75
|
-
lineEnd: rawMarkdownLines(document.text).length,
|
|
76
|
-
}))
|
|
77
|
-
).flatMap((range) => {
|
|
118
|
+
return (ranges ?? documents.map(fullWarningScope)).flatMap((range) => {
|
|
78
119
|
const source = documents.find((document) => document.id === range.document);
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
120
|
+
if (source === undefined) {
|
|
121
|
+
return [];
|
|
122
|
+
}
|
|
123
|
+
return [
|
|
124
|
+
{
|
|
125
|
+
document: range.document,
|
|
126
|
+
lineEnd: range.lineEnd,
|
|
127
|
+
lineStart: range.lineStart,
|
|
128
|
+
version: source.hash,
|
|
129
|
+
},
|
|
130
|
+
];
|
|
89
131
|
});
|
|
90
|
-
}
|
|
132
|
+
};
|
|
91
133
|
|
|
92
134
|
export const graphSchema = z.object({
|
|
93
|
-
version: z.literal(1),
|
|
94
|
-
lastExtraction: z.string().optional(),
|
|
95
|
-
documents: z.record(z.string(), z.string()),
|
|
96
|
-
units: z
|
|
97
|
-
.record(
|
|
98
|
-
z.string(),
|
|
99
|
-
z.object({ document: z.string(), version: z.string(), workKey: z.string().optional() }),
|
|
100
|
-
)
|
|
101
|
-
.default({}),
|
|
102
135
|
decisions: z.array(decisionSchema.extend(provenance)),
|
|
136
|
+
documents: z.record(z.string(), z.string()),
|
|
137
|
+
lastExtraction: z.string().optional(),
|
|
103
138
|
relationships: z.array(
|
|
104
139
|
relationshipSchema.extend({
|
|
105
140
|
batch: z.string(),
|
|
141
|
+
evidence: z.array(citationSchema.extend({ version: z.string().optional() })),
|
|
106
142
|
localId: z.string(),
|
|
107
143
|
quality,
|
|
108
|
-
|
|
109
|
-
}),
|
|
144
|
+
})
|
|
110
145
|
),
|
|
146
|
+
units: z
|
|
147
|
+
.record(
|
|
148
|
+
z.string(),
|
|
149
|
+
z.object({
|
|
150
|
+
document: z.string(),
|
|
151
|
+
version: z.string(),
|
|
152
|
+
workKey: z.string().optional(),
|
|
153
|
+
})
|
|
154
|
+
)
|
|
155
|
+
.default({}),
|
|
156
|
+
version: z.literal(1),
|
|
111
157
|
warnings: z.array(warningSchema),
|
|
112
158
|
});
|
|
113
159
|
export type Graph = z.infer<typeof graphSchema>;
|
|
114
160
|
export type Extraction = z.infer<typeof extractionSchema>;
|
|
115
161
|
export type KnowledgeCheck = z.infer<typeof checkSchema>;
|
|
116
|
-
export const emptyGraph = (): Graph
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
162
|
+
export const emptyGraph = function emptyGraph(): Graph {
|
|
163
|
+
return {
|
|
164
|
+
decisions: [],
|
|
165
|
+
documents: {},
|
|
166
|
+
relationships: [],
|
|
167
|
+
units: {},
|
|
168
|
+
version: 1,
|
|
169
|
+
warnings: [],
|
|
170
|
+
};
|
|
171
|
+
};
|
|
124
172
|
|
|
125
|
-
export
|
|
173
|
+
export const validCitation = function validCitation(
|
|
174
|
+
entry: z.infer<typeof citationSchema>,
|
|
175
|
+
documents: Document[]
|
|
176
|
+
) {
|
|
126
177
|
const document = documents.find((item) => item.id === entry.document);
|
|
127
178
|
return (
|
|
128
179
|
document !== undefined &&
|
|
@@ -130,214 +181,296 @@ export function validCitation(entry: z.infer<typeof citationSchema>, documents:
|
|
|
130
181
|
entry.lineEnd <= rawMarkdownLines(document.text).length &&
|
|
131
182
|
sourceRange(document.text, entry.lineStart, entry.lineEnd).trim().length > 0
|
|
132
183
|
);
|
|
133
|
-
}
|
|
184
|
+
};
|
|
134
185
|
|
|
135
|
-
export
|
|
186
|
+
export const sourceEvidence = function sourceEvidence(
|
|
187
|
+
entry: z.infer<typeof citationSchema>,
|
|
188
|
+
project: Project
|
|
189
|
+
) {
|
|
136
190
|
const document = project.documents.find((item) => item.id === entry.document);
|
|
137
|
-
if (
|
|
191
|
+
if (document === undefined || !validCitation(entry, project.documents)) {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
138
194
|
return {
|
|
139
195
|
document: entry.document,
|
|
140
|
-
lineStart: entry.lineStart,
|
|
141
|
-
lineEnd: entry.lineEnd,
|
|
142
|
-
version: document.hash,
|
|
143
196
|
historical: document.historical,
|
|
197
|
+
lineEnd: entry.lineEnd,
|
|
198
|
+
lineStart: entry.lineStart,
|
|
144
199
|
text: sourceRange(document.text, entry.lineStart, entry.lineEnd),
|
|
200
|
+
version: document.hash,
|
|
145
201
|
};
|
|
146
|
-
}
|
|
202
|
+
};
|
|
147
203
|
|
|
148
|
-
function inRanges(
|
|
204
|
+
const inRanges = function inRanges(
|
|
149
205
|
entry: z.infer<typeof citationSchema>,
|
|
150
|
-
ranges?: z.infer<typeof citationSchema>[]
|
|
206
|
+
ranges?: z.infer<typeof citationSchema>[]
|
|
151
207
|
) {
|
|
152
|
-
if (
|
|
208
|
+
if (ranges === undefined) {
|
|
209
|
+
return true;
|
|
210
|
+
}
|
|
153
211
|
for (let line = entry.lineStart; line <= entry.lineEnd; line += 1) {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
)
|
|
212
|
+
const isCovered = ranges.some((range) => {
|
|
213
|
+
if (range.document !== entry.document) {
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
return range.lineStart <= line && range.lineEnd >= line;
|
|
217
|
+
});
|
|
218
|
+
if (!isCovered) {
|
|
160
219
|
return false;
|
|
220
|
+
}
|
|
161
221
|
}
|
|
162
222
|
return true;
|
|
163
|
-
}
|
|
223
|
+
};
|
|
164
224
|
|
|
165
|
-
|
|
166
|
-
graph: Graph;
|
|
167
|
-
extraction: Extraction;
|
|
168
|
-
documents: Document[];
|
|
225
|
+
interface ExtractionOptions {
|
|
169
226
|
batch: string;
|
|
170
227
|
contextDocuments?: Document[];
|
|
228
|
+
contextRanges?: z.infer<typeof citationSchema>[];
|
|
229
|
+
documents: Document[];
|
|
171
230
|
existingIds?: string[];
|
|
231
|
+
extraction: Extraction;
|
|
232
|
+
graph: Graph;
|
|
172
233
|
targetRanges?: z.infer<typeof citationSchema>[];
|
|
173
|
-
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const retainedWarnings = function retainedWarnings(graph: Graph, scope: WarningScope[]) {
|
|
237
|
+
return graph.warnings.filter((warning) => {
|
|
238
|
+
if (typeof warning === 'string') {
|
|
239
|
+
return true;
|
|
240
|
+
}
|
|
241
|
+
return warning.scope.every((old) => {
|
|
242
|
+
const hasOverlap = scope.some((current) => {
|
|
243
|
+
if (current.document !== old.document) {
|
|
244
|
+
return false;
|
|
245
|
+
}
|
|
246
|
+
return (
|
|
247
|
+
current.version !== old.version ||
|
|
248
|
+
(current.lineStart <= old.lineEnd && current.lineEnd >= old.lineStart)
|
|
249
|
+
);
|
|
250
|
+
});
|
|
251
|
+
return !hasOverlap;
|
|
252
|
+
});
|
|
253
|
+
});
|
|
174
254
|
};
|
|
175
255
|
|
|
176
|
-
function
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
256
|
+
const extractedRelationships = function extractedRelationships(input: {
|
|
257
|
+
options: ExtractionOptions;
|
|
258
|
+
decisions: Graph['decisions'];
|
|
259
|
+
ids: Map<string, string>;
|
|
260
|
+
warnings: string[];
|
|
261
|
+
}) {
|
|
262
|
+
const { options, decisions, ids, warnings } = input;
|
|
263
|
+
const { graph, extraction, documents, batch } = options;
|
|
264
|
+
const available = new Set(decisions.map((entry) => entry.id));
|
|
265
|
+
const relationships = graph.relationships.filter((entry) => {
|
|
266
|
+
const hasChangedEvidence = entry.evidence.some((citation) => {
|
|
267
|
+
const isTargetRange =
|
|
268
|
+
options.targetRanges?.some((range) => {
|
|
269
|
+
if (range.document !== citation.document) {
|
|
270
|
+
return false;
|
|
271
|
+
}
|
|
272
|
+
return range.lineStart <= citation.lineEnd && range.lineEnd >= citation.lineStart;
|
|
273
|
+
}) === true;
|
|
274
|
+
const hasChangedDocument = (options.contextDocuments ?? documents).some((document) => {
|
|
275
|
+
if (document.id !== citation.document) {
|
|
276
|
+
return false;
|
|
277
|
+
}
|
|
278
|
+
return document.hash !== citation.version;
|
|
279
|
+
});
|
|
280
|
+
return isTargetRange || hasChangedDocument;
|
|
281
|
+
});
|
|
282
|
+
return available.has(entry.from) && available.has(entry.to) && !hasChangedEvidence;
|
|
283
|
+
});
|
|
284
|
+
const seen = new Set<string>();
|
|
285
|
+
for (const entry of extraction.relationships) {
|
|
286
|
+
const from = ids.get(entry.from);
|
|
287
|
+
const to = ids.get(entry.to);
|
|
288
|
+
const hasInvalidEvidence = entry.evidence.some((item) => {
|
|
289
|
+
if (!validCitation(item, options.contextDocuments ?? documents)) {
|
|
290
|
+
return true;
|
|
291
|
+
}
|
|
292
|
+
return !inRanges(item, options.contextRanges);
|
|
293
|
+
});
|
|
294
|
+
const hasMissingEndpoint = from === undefined || to === undefined;
|
|
295
|
+
const isDuplicate = seen.has(entry.id);
|
|
296
|
+
if (hasMissingEndpoint || isDuplicate || hasInvalidEvidence) {
|
|
297
|
+
warnings.push(
|
|
298
|
+
`Relationship ${entry.id} has an unknown endpoint, duplicate ID or invalid reference.`
|
|
299
|
+
);
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
seen.add(entry.id);
|
|
303
|
+
const evidence = entry.evidence.map((citation) => {
|
|
304
|
+
const source = (options.contextDocuments ?? documents).find(
|
|
305
|
+
(document) => document.id === citation.document
|
|
306
|
+
);
|
|
307
|
+
return { ...citation, version: source?.hash };
|
|
308
|
+
});
|
|
309
|
+
const id = digest(JSON.stringify({ ...entry, evidence, from, to }, relationshipIdentityFields));
|
|
310
|
+
const previous = relationships.findIndex((relationship) => relationship.id === id);
|
|
311
|
+
if (previous !== -1) {
|
|
312
|
+
relationships.splice(previous, 1);
|
|
313
|
+
}
|
|
314
|
+
relationships.push(
|
|
315
|
+
withLiveProvenance<Graph['relationships'][number]>(
|
|
316
|
+
{
|
|
317
|
+
...entry,
|
|
318
|
+
batch,
|
|
319
|
+
evidence,
|
|
320
|
+
from,
|
|
321
|
+
id,
|
|
322
|
+
localId: entry.id,
|
|
323
|
+
quality: 'unchecked',
|
|
324
|
+
to,
|
|
325
|
+
},
|
|
326
|
+
'relationship'
|
|
327
|
+
)
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
return relationships;
|
|
331
|
+
};
|
|
190
332
|
|
|
191
|
-
export function applyExtraction(options: ExtractionOptions) {
|
|
333
|
+
export const applyExtraction = function applyExtraction(options: ExtractionOptions) {
|
|
192
334
|
const { graph, extraction, documents, batch } = options;
|
|
193
335
|
const decisions = graph.decisions.filter((entry) => {
|
|
194
336
|
const source = documents.find((document) => document.id === entry.document);
|
|
337
|
+
const isTargetRange =
|
|
338
|
+
options.targetRanges?.some((range) => {
|
|
339
|
+
if (range.document !== entry.document) {
|
|
340
|
+
return false;
|
|
341
|
+
}
|
|
342
|
+
return range.lineStart <= entry.lineEnd && range.lineEnd >= entry.lineStart;
|
|
343
|
+
}) === true;
|
|
195
344
|
return (
|
|
196
|
-
|
|
197
|
-
(source.hash === entry.version &&
|
|
198
|
-
validCitation(entry, [source]) &&
|
|
199
|
-
!options.targetRanges?.some(
|
|
200
|
-
(range) =>
|
|
201
|
-
range.document === entry.document &&
|
|
202
|
-
range.lineStart <= entry.lineEnd &&
|
|
203
|
-
range.lineEnd >= entry.lineStart,
|
|
204
|
-
))
|
|
345
|
+
source === undefined ||
|
|
346
|
+
(source.hash === entry.version && validCitation(entry, [source]) && !isTargetRange)
|
|
205
347
|
);
|
|
206
348
|
});
|
|
207
349
|
const ids = new Map(
|
|
208
350
|
decisions
|
|
209
|
-
.filter((entry) => options.existingIds?.includes(entry.id))
|
|
210
|
-
.map((entry) => [entry.id, entry.id])
|
|
351
|
+
.filter((entry) => options.existingIds?.includes(entry.id) === true)
|
|
352
|
+
.map((entry) => [entry.id, entry.id])
|
|
211
353
|
);
|
|
212
354
|
const warnings = [...extraction.uncertainties];
|
|
213
355
|
for (const entry of extraction.decisions) {
|
|
214
356
|
const source = documents.find((document) => document.id === entry.document);
|
|
215
|
-
if (
|
|
357
|
+
if (source === undefined || ids.has(entry.id)) {
|
|
216
358
|
warnings.push(`Decision ${entry.id} has an unknown, duplicate or invalid source reference.`);
|
|
217
359
|
continue;
|
|
218
360
|
}
|
|
219
|
-
const
|
|
220
|
-
if (!
|
|
361
|
+
const isLocated = validCitation(entry, documents) && inRanges(entry, options.targetRanges);
|
|
362
|
+
if (!isLocated) {
|
|
221
363
|
warnings.push(
|
|
222
|
-
`Decision ${entry.id} has an unverified line range; its document remains available
|
|
364
|
+
`Decision ${entry.id} has an unverified line range; its document remains available.`
|
|
223
365
|
);
|
|
224
|
-
|
|
366
|
+
}
|
|
367
|
+
const id = digest(JSON.stringify({ version: source.hash, ...entry }, decisionIdentityFields));
|
|
225
368
|
ids.set(entry.id, id);
|
|
226
369
|
const previous = decisions.findIndex((decision) => decision.id === id);
|
|
227
|
-
if (previous
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
370
|
+
if (previous !== -1) {
|
|
371
|
+
decisions.splice(previous, 1);
|
|
372
|
+
}
|
|
373
|
+
decisions.push(
|
|
374
|
+
withLiveProvenance<Graph['decisions'][number]>(
|
|
375
|
+
{
|
|
376
|
+
...entry,
|
|
377
|
+
batch,
|
|
378
|
+
id,
|
|
379
|
+
localId: entry.id,
|
|
380
|
+
quality: isLocated ? 'unchecked' : 'uncertain',
|
|
381
|
+
version: source.hash,
|
|
382
|
+
},
|
|
383
|
+
'decision'
|
|
384
|
+
)
|
|
385
|
+
);
|
|
236
386
|
}
|
|
237
|
-
const relationships = extractedRelationships({
|
|
387
|
+
const relationships = extractedRelationships({
|
|
388
|
+
decisions,
|
|
389
|
+
ids,
|
|
390
|
+
options,
|
|
391
|
+
warnings,
|
|
392
|
+
});
|
|
238
393
|
return {
|
|
239
|
-
|
|
240
|
-
lastExtraction: batch,
|
|
394
|
+
decisions,
|
|
241
395
|
documents: Object.fromEntries(
|
|
242
|
-
Object.entries(graph.documents).filter(
|
|
243
|
-
(
|
|
244
|
-
|
|
245
|
-
|
|
396
|
+
Object.entries(graph.documents).filter(([id, version]) => {
|
|
397
|
+
const hasChanged = documents.some((document) => {
|
|
398
|
+
if (document.id !== id) {
|
|
399
|
+
return false;
|
|
400
|
+
}
|
|
401
|
+
return document.hash !== version;
|
|
402
|
+
});
|
|
403
|
+
return !hasChanged;
|
|
404
|
+
})
|
|
246
405
|
),
|
|
406
|
+
lastExtraction: batch,
|
|
407
|
+
relationships,
|
|
247
408
|
units: Object.fromEntries(
|
|
248
|
-
Object.entries(graph.units).filter(
|
|
249
|
-
(
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
409
|
+
Object.entries(graph.units).filter(([, unit]) => {
|
|
410
|
+
const hasChanged = documents.some((document) => {
|
|
411
|
+
if (document.id !== unit.document) {
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
return document.hash !== unit.version;
|
|
415
|
+
});
|
|
416
|
+
return !hasChanged;
|
|
417
|
+
})
|
|
254
418
|
),
|
|
255
|
-
|
|
256
|
-
relationships,
|
|
419
|
+
version: 1 as const,
|
|
257
420
|
warnings: [
|
|
258
421
|
...retainedWarnings(graph, warningScope(documents, options.targetRanges)),
|
|
259
|
-
...warnings.map((message) =>
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
})
|
|
422
|
+
...warnings.map((message) => {
|
|
423
|
+
const scope = warningScope(documents, options.targetRanges);
|
|
424
|
+
return { message, scope };
|
|
425
|
+
}),
|
|
263
426
|
],
|
|
264
427
|
};
|
|
265
|
-
}
|
|
428
|
+
};
|
|
266
429
|
|
|
267
|
-
function
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
const available = new Set(decisions.map((entry) => entry.id));
|
|
276
|
-
const relationships = graph.relationships.filter(
|
|
277
|
-
(entry) =>
|
|
278
|
-
available.has(entry.from) &&
|
|
279
|
-
available.has(entry.to) &&
|
|
280
|
-
!entry.evidence.some(
|
|
281
|
-
(citation) =>
|
|
282
|
-
options.targetRanges?.some(
|
|
283
|
-
(range) =>
|
|
284
|
-
range.document === citation.document &&
|
|
285
|
-
range.lineStart <= citation.lineEnd &&
|
|
286
|
-
range.lineEnd >= citation.lineStart,
|
|
287
|
-
) ||
|
|
288
|
-
(options.contextDocuments ?? documents).some(
|
|
289
|
-
(document) => document.id === citation.document && document.hash !== citation.version,
|
|
290
|
-
),
|
|
291
|
-
),
|
|
292
|
-
);
|
|
293
|
-
const seen = new Set<string>();
|
|
294
|
-
for (const entry of extraction.relationships) {
|
|
295
|
-
const from = ids.get(entry.from);
|
|
296
|
-
const to = ids.get(entry.to);
|
|
297
|
-
if (
|
|
298
|
-
!from ||
|
|
299
|
-
!to ||
|
|
300
|
-
seen.has(entry.id) ||
|
|
301
|
-
entry.evidence.some(
|
|
302
|
-
(item) =>
|
|
303
|
-
!validCitation(item, options.contextDocuments ?? documents) ||
|
|
304
|
-
!inRanges(item, options.contextRanges),
|
|
305
|
-
)
|
|
306
|
-
) {
|
|
307
|
-
warnings.push(
|
|
308
|
-
`Relationship ${entry.id} has an unknown endpoint, duplicate ID or invalid reference.`,
|
|
309
|
-
);
|
|
310
|
-
continue;
|
|
430
|
+
const findingScope = function findingScope(
|
|
431
|
+
graph: Graph,
|
|
432
|
+
target: string,
|
|
433
|
+
{ batch, fallback }: { batch: string; fallback: WarningScope[] }
|
|
434
|
+
): WarningScope[] {
|
|
435
|
+
const decisions = graph.decisions.filter((entry) => {
|
|
436
|
+
if (entry.id === target) {
|
|
437
|
+
return true;
|
|
311
438
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
from,
|
|
327
|
-
to,
|
|
328
|
-
localId: entry.id,
|
|
329
|
-
batch,
|
|
330
|
-
quality: 'unchecked',
|
|
439
|
+
return entry.batch === batch && (entry.localId === target || entry.document === target);
|
|
440
|
+
});
|
|
441
|
+
const relationships = graph.relationships.filter((entry) => {
|
|
442
|
+
if (entry.id === target) {
|
|
443
|
+
return true;
|
|
444
|
+
}
|
|
445
|
+
return entry.batch === batch && entry.localId === target;
|
|
446
|
+
});
|
|
447
|
+
const evidenceScopeFor = function evidenceScopeFor(entry: Graph['relationships'][number]) {
|
|
448
|
+
return entry.evidence.flatMap((citation) => {
|
|
449
|
+
if (citation.version === undefined) {
|
|
450
|
+
return [];
|
|
451
|
+
}
|
|
452
|
+
return [{ ...citation, version: citation.version }];
|
|
331
453
|
});
|
|
454
|
+
};
|
|
455
|
+
const relationshipScope = relationships.flatMap(evidenceScopeFor);
|
|
456
|
+
const scope = [
|
|
457
|
+
...decisions.map((entry) => {
|
|
458
|
+
const { document, lineEnd, lineStart, version } = entry;
|
|
459
|
+
return { document, lineEnd, lineStart, version };
|
|
460
|
+
}),
|
|
461
|
+
...relationshipScope,
|
|
462
|
+
];
|
|
463
|
+
if (scope.length > 0) {
|
|
464
|
+
return scope;
|
|
332
465
|
}
|
|
333
|
-
|
|
334
|
-
|
|
466
|
+
const documentScope = fallback.filter((entry) => entry.document === target);
|
|
467
|
+
return documentScope.length > 0 ? documentScope : fallback;
|
|
468
|
+
};
|
|
335
469
|
|
|
336
|
-
export function applyCheck(
|
|
470
|
+
export const applyCheck = function applyCheck(
|
|
337
471
|
graph: Graph,
|
|
338
472
|
check: KnowledgeCheck,
|
|
339
|
-
batch: string
|
|
340
|
-
scope: WarningScope[] = [],
|
|
473
|
+
{ batch, scope = [] }: { batch: string; scope?: WarningScope[] }
|
|
341
474
|
): Graph {
|
|
342
475
|
const targets = new Set(check.findings.map((finding) => finding.target));
|
|
343
476
|
const known = new Set([
|
|
@@ -345,32 +478,46 @@ export function applyCheck(
|
|
|
345
478
|
...scope.map((entry) => entry.document),
|
|
346
479
|
...graph.decisions.map((entry) => entry.id),
|
|
347
480
|
...graph.relationships.map((entry) => entry.id),
|
|
348
|
-
...graph.decisions
|
|
349
|
-
|
|
350
|
-
|
|
481
|
+
...graph.decisions.flatMap((entry) => {
|
|
482
|
+
if (entry.batch !== batch) {
|
|
483
|
+
return [];
|
|
484
|
+
}
|
|
485
|
+
return [entry.localId, entry.document];
|
|
486
|
+
}),
|
|
351
487
|
...graph.relationships.filter((entry) => entry.batch === batch).map((entry) => entry.localId),
|
|
352
488
|
]);
|
|
353
|
-
const
|
|
489
|
+
const isUncertainBatch =
|
|
490
|
+
targets.has('batch') || [...targets].some((target) => !known.has(target));
|
|
354
491
|
const decisions = graph.decisions.map((entry) => {
|
|
355
|
-
if (entry.batch !== batch && !targets.has(entry.id))
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
492
|
+
if (entry.batch !== batch && !targets.has(entry.id)) {
|
|
493
|
+
return entry;
|
|
494
|
+
}
|
|
495
|
+
const isTargeted =
|
|
496
|
+
targets.has(entry.id) || targets.has(entry.localId) || targets.has(entry.document);
|
|
497
|
+
const isUncertain = isTargeted || entry.quality === 'uncertain' || isUncertainBatch;
|
|
498
|
+
return {
|
|
499
|
+
...entry,
|
|
500
|
+
quality: quality.parse(isUncertain ? 'uncertain' : 'checked'),
|
|
501
|
+
};
|
|
363
502
|
});
|
|
364
503
|
const relationships = graph.relationships.map((entry) => {
|
|
365
|
-
if (entry.batch !== batch && !targets.has(entry.id))
|
|
366
|
-
|
|
504
|
+
if (entry.batch !== batch && !targets.has(entry.id)) {
|
|
505
|
+
return entry;
|
|
506
|
+
}
|
|
507
|
+
const isUncertain =
|
|
367
508
|
targets.has(entry.id) ||
|
|
368
|
-
|
|
509
|
+
isUncertainBatch ||
|
|
369
510
|
targets.has(entry.localId) ||
|
|
370
|
-
decisions.some(
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
511
|
+
decisions.some((node) => {
|
|
512
|
+
if (node.id !== entry.from && node.id !== entry.to) {
|
|
513
|
+
return false;
|
|
514
|
+
}
|
|
515
|
+
return node.quality === 'uncertain';
|
|
516
|
+
});
|
|
517
|
+
return {
|
|
518
|
+
...entry,
|
|
519
|
+
quality: quality.parse(isUncertain ? 'uncertain' : 'checked'),
|
|
520
|
+
};
|
|
374
521
|
});
|
|
375
522
|
return {
|
|
376
523
|
...graph,
|
|
@@ -378,42 +525,13 @@ export function applyCheck(
|
|
|
378
525
|
relationships,
|
|
379
526
|
warnings: [
|
|
380
527
|
...graph.warnings,
|
|
381
|
-
...check.findings.map((finding) =>
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
528
|
+
...check.findings.map((finding) => {
|
|
529
|
+
const findingWarningScope = findingScope(graph, finding.target, {
|
|
530
|
+
batch,
|
|
531
|
+
fallback: scope,
|
|
532
|
+
});
|
|
533
|
+
return { message: finding.reason, scope: findingWarningScope };
|
|
534
|
+
}),
|
|
385
535
|
],
|
|
386
536
|
};
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
function findingScope(
|
|
390
|
-
graph: Graph,
|
|
391
|
-
target: string,
|
|
392
|
-
batch: string,
|
|
393
|
-
fallback: WarningScope[],
|
|
394
|
-
): WarningScope[] {
|
|
395
|
-
const decisions = graph.decisions.filter(
|
|
396
|
-
(entry) =>
|
|
397
|
-
entry.id === target ||
|
|
398
|
-
(entry.batch === batch && (entry.localId === target || entry.document === target)),
|
|
399
|
-
);
|
|
400
|
-
const relationships = graph.relationships.filter(
|
|
401
|
-
(entry) => entry.id === target || (entry.batch === batch && entry.localId === target),
|
|
402
|
-
);
|
|
403
|
-
const scope = [
|
|
404
|
-
...decisions.map(({ document, version, lineStart, lineEnd }) => ({
|
|
405
|
-
document,
|
|
406
|
-
version,
|
|
407
|
-
lineStart,
|
|
408
|
-
lineEnd,
|
|
409
|
-
})),
|
|
410
|
-
...relationships.flatMap((entry) =>
|
|
411
|
-
entry.evidence.flatMap((citation) =>
|
|
412
|
-
citation.version ? [{ ...citation, version: citation.version }] : [],
|
|
413
|
-
),
|
|
414
|
-
),
|
|
415
|
-
];
|
|
416
|
-
const documentScope = fallback.filter((entry) => entry.document === target);
|
|
417
|
-
if (scope.length) return scope;
|
|
418
|
-
return documentScope.length ? documentScope : fallback;
|
|
419
|
-
}
|
|
537
|
+
};
|