@oortmem/core 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.d.ts +37 -0
- package/dist/config.d.ts +19 -0
- package/dist/config.js +48 -2
- package/dist/display.d.ts +9 -0
- package/dist/display.js +10 -0
- package/dist/export-format.d.ts +4 -1
- package/dist/export-format.js +7 -2
- package/dist/fusion.d.ts +2 -0
- package/dist/fusion.js +3 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +3 -1
- package/dist/judgment.d.ts +526 -0
- package/dist/judgment.js +1357 -0
- package/dist/model.d.ts +14 -1
- package/dist/observe.js +3 -1
- package/dist/recall.d.ts +2 -0
- package/dist/recall.js +2 -0
- package/dist/store-conformance.js +29 -0
- package/package.json +1 -1
package/dist/judgment.js
ADDED
|
@@ -0,0 +1,1357 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { addressed } from './address.js';
|
|
3
|
+
import { INVISIBLE_SOURCE } from './display.js';
|
|
4
|
+
import { homeAddress } from './grant.js';
|
|
5
|
+
import { isCurrent } from './model.js';
|
|
6
|
+
import { informativeTokens } from './relevance.js';
|
|
7
|
+
import { scrub } from './scrub.js';
|
|
8
|
+
/**
|
|
9
|
+
* Judgment memory (CON-71): an expert's approve or reject decisions, with their reasons in their
|
|
10
|
+
* own words, kept so an agent can apply them to a new item, say why, and say when it is not sure.
|
|
11
|
+
*
|
|
12
|
+
* A judgment is a `judgment` memory whose `judgment` field holds the decision. It is written only
|
|
13
|
+
* through the judgment functions here (never inferred from a conversation), its reason is never
|
|
14
|
+
* paraphrased (the write path skips judgments), and it is pinned so it never loses its embedding
|
|
15
|
+
* to cold.
|
|
16
|
+
*
|
|
17
|
+
* Who recorded it matters. A judgment recorded by a person (the CLI, a bulk import) counts when
|
|
18
|
+
* judging. One recorded by an agent over MCP (`via: 'mcp'`) is agent-recorded and unconfirmed: it
|
|
19
|
+
* never corrects a person's judgment, and judging leaves it out until a person confirms it
|
|
20
|
+
* (`confirmJudgment`, CLI only) or the caller asks for agent-recorded judgments explicitly. One
|
|
21
|
+
* imported from a pack names an expert only as the pack states it, so judging leaves it out unless
|
|
22
|
+
* asked.
|
|
23
|
+
*/
|
|
24
|
+
/** The verdict set a judgment uses when none is given. */
|
|
25
|
+
export const DEFAULT_VERDICTS = Object.freeze(['approved', 'rejected']);
|
|
26
|
+
export const JUDGMENT_LIMITS = Object.freeze({
|
|
27
|
+
summary: 500,
|
|
28
|
+
ref: 2000,
|
|
29
|
+
reason: 2000,
|
|
30
|
+
details: 4000,
|
|
31
|
+
criteria: 12,
|
|
32
|
+
criterion: 64,
|
|
33
|
+
verdicts: 12,
|
|
34
|
+
by: 128,
|
|
35
|
+
/** Rows one bulk import or pack may hold; a larger one is refused before anything is written. */
|
|
36
|
+
importRows: 5000,
|
|
37
|
+
});
|
|
38
|
+
/** A judgment input that cannot be recorded; the message names the field, never its value. */
|
|
39
|
+
export class JudgmentInputError extends Error {
|
|
40
|
+
name = 'JudgmentInputError';
|
|
41
|
+
}
|
|
42
|
+
// oxlint-disable-next-line no-control-regex -- rejecting control characters is the point
|
|
43
|
+
const CONTROL = /[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/;
|
|
44
|
+
/**
|
|
45
|
+
* Bidi controls (Trojan Source) and zero-width or other invisible format characters: never in
|
|
46
|
+
* anything a judgment holds. The same set the CLI's `shown()` escapes (core's display.ts).
|
|
47
|
+
*/
|
|
48
|
+
const INVISIBLE = new RegExp(INVISIBLE_SOURCE);
|
|
49
|
+
const LABEL = /^[a-z0-9][a-z0-9_-]{0,31}$/;
|
|
50
|
+
const CRITERION = /^[a-z0-9][a-z0-9_-]{0,63}$/;
|
|
51
|
+
const DOMAIN = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
52
|
+
function text(field, v, max, multiline = false) {
|
|
53
|
+
if (typeof v !== 'string')
|
|
54
|
+
throw new JudgmentInputError(`${field} must be text`);
|
|
55
|
+
const t = v.trim();
|
|
56
|
+
if (t === '')
|
|
57
|
+
throw new JudgmentInputError(`${field} is empty`);
|
|
58
|
+
if (t.length > max)
|
|
59
|
+
throw new JudgmentInputError(`${field} is longer than ${max} characters`);
|
|
60
|
+
if (CONTROL.test(t) || (!multiline && /[\r\n]/.test(t))) {
|
|
61
|
+
throw new JudgmentInputError(`${field} holds control characters`);
|
|
62
|
+
}
|
|
63
|
+
// The raw value: trim() would quietly drop a leading or trailing U+FEFF.
|
|
64
|
+
if (INVISIBLE.test(v)) {
|
|
65
|
+
throw new JudgmentInputError(`${field} holds bidi or invisible format characters`);
|
|
66
|
+
}
|
|
67
|
+
return t;
|
|
68
|
+
}
|
|
69
|
+
/** A verdict or criterion label: lowercase, spaces as `-`. */
|
|
70
|
+
export function judgmentLabel(field, v, re = LABEL) {
|
|
71
|
+
if (typeof v !== 'string')
|
|
72
|
+
throw new JudgmentInputError(`${field} must be text`);
|
|
73
|
+
const t = v.trim().toLowerCase().replace(/\s+/g, '-');
|
|
74
|
+
if (!re.test(t) || INVISIBLE.test(v)) {
|
|
75
|
+
throw new JudgmentInputError(`${field} must be a short label: lowercase letters, digits, "-" or "_"`);
|
|
76
|
+
}
|
|
77
|
+
return t;
|
|
78
|
+
}
|
|
79
|
+
/** A list of labels, its type and length checked before anything is mapped. */
|
|
80
|
+
function labels(field, v, max) {
|
|
81
|
+
if (v === undefined || v === null)
|
|
82
|
+
return undefined;
|
|
83
|
+
if (!Array.isArray(v))
|
|
84
|
+
throw new JudgmentInputError(`${field} must be a list`);
|
|
85
|
+
if (v.length > max)
|
|
86
|
+
throw new JudgmentInputError(`${field} has more than ${max} entries`);
|
|
87
|
+
if (!v.every((x) => typeof x === 'string')) {
|
|
88
|
+
throw new JudgmentInputError(`${field} must be a list of text labels`);
|
|
89
|
+
}
|
|
90
|
+
return v;
|
|
91
|
+
}
|
|
92
|
+
/** Checks and normalises `input` (labels lowercased, text trimmed). Throws `JudgmentInputError`. */
|
|
93
|
+
export function normaliseJudgment(input, now) {
|
|
94
|
+
if (typeof input !== 'object' || input === null) {
|
|
95
|
+
throw new JudgmentInputError('a judgment must be an object');
|
|
96
|
+
}
|
|
97
|
+
const rawVerdicts = labels('verdicts', input.verdicts, JUDGMENT_LIMITS.verdicts);
|
|
98
|
+
const verdicts = [
|
|
99
|
+
...new Set((rawVerdicts ?? DEFAULT_VERDICTS).map((v) => judgmentLabel('verdicts', v))),
|
|
100
|
+
];
|
|
101
|
+
if (verdicts.length < 2)
|
|
102
|
+
throw new JudgmentInputError('verdicts needs at least two labels');
|
|
103
|
+
const verdict = judgmentLabel('verdict', input.verdict);
|
|
104
|
+
if (!verdicts.includes(verdict)) {
|
|
105
|
+
throw new JudgmentInputError(`verdict must be one of: ${verdicts.join(', ')}`);
|
|
106
|
+
}
|
|
107
|
+
const criteria = [
|
|
108
|
+
...new Set((labels('criteria', input.criteria, JUDGMENT_LIMITS.criteria) ?? [])
|
|
109
|
+
.map((c) => c.trim())
|
|
110
|
+
.filter((c) => c !== '')
|
|
111
|
+
.map((c) => judgmentLabel('criteria', c, CRITERION))),
|
|
112
|
+
];
|
|
113
|
+
let decidedAt = now.toISOString();
|
|
114
|
+
if (input.decidedAt !== undefined) {
|
|
115
|
+
if (typeof input.decidedAt !== 'string') {
|
|
116
|
+
throw new JudgmentInputError('decidedAt must be an ISO date or time');
|
|
117
|
+
}
|
|
118
|
+
const t = Date.parse(input.decidedAt);
|
|
119
|
+
if (Number.isNaN(t))
|
|
120
|
+
throw new JudgmentInputError('decidedAt is not an ISO date or time');
|
|
121
|
+
decidedAt = new Date(t).toISOString();
|
|
122
|
+
}
|
|
123
|
+
if (input.ref !== undefined && typeof input.ref !== 'string') {
|
|
124
|
+
throw new JudgmentInputError('ref must be text');
|
|
125
|
+
}
|
|
126
|
+
const ref = input.ref === undefined || input.ref.trim() === '' ? undefined : input.ref;
|
|
127
|
+
return {
|
|
128
|
+
subject: {
|
|
129
|
+
summary: text('subject', input.subject, JUDGMENT_LIMITS.summary),
|
|
130
|
+
...(ref !== undefined && { ref: text('ref', ref, JUDGMENT_LIMITS.ref) }),
|
|
131
|
+
},
|
|
132
|
+
verdict,
|
|
133
|
+
verdicts,
|
|
134
|
+
reason: text('reason', input.reason, JUDGMENT_LIMITS.reason, true),
|
|
135
|
+
criteria,
|
|
136
|
+
domain: judgmentLabel('domain', input.domain, DOMAIN),
|
|
137
|
+
by: text('by', input.by, JUDGMENT_LIMITS.by),
|
|
138
|
+
decidedAt,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* An expert id as agent-facing text may show it: letters, digits and `. _ @ + -` kept, anything
|
|
143
|
+
* else `_`, at most 64 characters. Ids are checked on the way in; this is the second line.
|
|
144
|
+
*/
|
|
145
|
+
export function safeExpertId(by) {
|
|
146
|
+
return by.replace(/[^A-Za-z0-9._@+-]/g, '_').slice(0, 64);
|
|
147
|
+
}
|
|
148
|
+
/** The memory text recall shows for a judgment: who decided what, and why, in their words. */
|
|
149
|
+
export function judgmentText(j) {
|
|
150
|
+
const criteria = j.criteria.length === 0 ? '' : ` Criteria: ${j.criteria.join(', ')}.`;
|
|
151
|
+
const who = j.via === 'mcp' && j.confirmed === undefined ? ' (agent-recorded, unconfirmed)' : '';
|
|
152
|
+
return `Judgment by ${j.by}${who} (${j.domain}): ${j.verdict} "${j.subject.summary}". Reason: ${j.reason}${criteria}`;
|
|
153
|
+
}
|
|
154
|
+
/** True for a `judgment` memory that still counts: current, and not withdrawn. */
|
|
155
|
+
export function isActiveJudgment(m, now) {
|
|
156
|
+
return (m.type === 'judgment' && m.judgment !== undefined && !m.judgment.withdrawn && isCurrent(m, now));
|
|
157
|
+
}
|
|
158
|
+
/** Recorded by an agent over MCP and not confirmed by a person. */
|
|
159
|
+
export function isUnconfirmedAgentJudgment(j) {
|
|
160
|
+
return j.via === 'mcp' && j.confirmed === undefined;
|
|
161
|
+
}
|
|
162
|
+
/** Case and whitespace folded, for matching a decision whatever its spacing or capitals. */
|
|
163
|
+
const fold = (s) => s.normalize('NFKC').toLowerCase().replace(/\s+/g, ' ').trim();
|
|
164
|
+
/**
|
|
165
|
+
* The key a withdrawn, corrected or purged decision leaves behind: expert, domain, folded subject,
|
|
166
|
+
* folded reason and verdict, hashed. It leaves out `decidedAt` (a pack could shift it) and holds no
|
|
167
|
+
* text (a purged judgment keeps it).
|
|
168
|
+
*/
|
|
169
|
+
export function tombstoneKey(j) {
|
|
170
|
+
return createHash('sha256')
|
|
171
|
+
.update(JSON.stringify([j.by, j.domain, fold(j.subject.summary), fold(j.reason), j.verdict]))
|
|
172
|
+
.digest('hex');
|
|
173
|
+
}
|
|
174
|
+
/** The exact decision, for spotting the same one again whatever its origin. */
|
|
175
|
+
function fingerprint(j) {
|
|
176
|
+
return JSON.stringify([
|
|
177
|
+
j.by,
|
|
178
|
+
j.domain,
|
|
179
|
+
j.subject.summary,
|
|
180
|
+
j.subject.ref ?? null,
|
|
181
|
+
j.verdict,
|
|
182
|
+
j.reason,
|
|
183
|
+
j.decidedAt,
|
|
184
|
+
]);
|
|
185
|
+
}
|
|
186
|
+
function redactJudgment(j) {
|
|
187
|
+
const kinds = new Set();
|
|
188
|
+
const clean = (s) => {
|
|
189
|
+
const r = scrub(s);
|
|
190
|
+
for (const x of r.redactions)
|
|
191
|
+
kinds.add(x.kind);
|
|
192
|
+
return r.text;
|
|
193
|
+
};
|
|
194
|
+
return {
|
|
195
|
+
judgment: {
|
|
196
|
+
...j,
|
|
197
|
+
subject: {
|
|
198
|
+
summary: clean(j.subject.summary),
|
|
199
|
+
...(j.subject.ref !== undefined && { ref: clean(j.subject.ref) }),
|
|
200
|
+
},
|
|
201
|
+
reason: clean(j.reason),
|
|
202
|
+
by: clean(j.by),
|
|
203
|
+
},
|
|
204
|
+
redacted: [...kinds],
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Who owns a judgment: a person (recorded by a person, or an agent's record a person confirmed) or
|
|
209
|
+
* an agent (recorded over MCP, unconfirmed). A write only ever corrects a judgment of its own kind,
|
|
210
|
+
* so an agent can never retract what a person recorded or confirmed.
|
|
211
|
+
*/
|
|
212
|
+
const recordedBy = (j) => isUnconfirmedAgentJudgment(j) ? 'agent' : 'person';
|
|
213
|
+
const itemKey = (j) => j.subject.ref === undefined
|
|
214
|
+
? undefined
|
|
215
|
+
: JSON.stringify([recordedBy(j), j.subject.ref, j.domain, j.by]);
|
|
216
|
+
async function indexJudgments(store, grant, now) {
|
|
217
|
+
const index = {
|
|
218
|
+
items: new Map(),
|
|
219
|
+
fingerprints: new Set(),
|
|
220
|
+
tombstones: new Set(),
|
|
221
|
+
imported: new Set(),
|
|
222
|
+
};
|
|
223
|
+
const all = await store.list(grant, { validOnly: false, embeddings: false });
|
|
224
|
+
for (const m of all) {
|
|
225
|
+
if (m.type !== 'judgment' || m.judgment === undefined)
|
|
226
|
+
continue;
|
|
227
|
+
const j = m.judgment;
|
|
228
|
+
// An agent's unconfirmed record is no evidence the decision is already here.
|
|
229
|
+
if (!isUnconfirmedAgentJudgment(j))
|
|
230
|
+
index.fingerprints.add(fingerprint(j));
|
|
231
|
+
if (j.pack !== undefined)
|
|
232
|
+
index.imported.add(`${j.pack.id}:${j.pack.originalId}`);
|
|
233
|
+
// Only a person's decision leaves a tombstone that blocks imports: an agent withdrawing or
|
|
234
|
+
// correcting its own unconfirmed record must not keep a genuine expert decision out.
|
|
235
|
+
if ((j.withdrawn !== undefined || j.purged !== undefined || m.endReason === 'corrected') &&
|
|
236
|
+
!isUnconfirmedAgentJudgment(j)) {
|
|
237
|
+
index.tombstones.add(j.tombstone ?? tombstoneKey(j));
|
|
238
|
+
}
|
|
239
|
+
const key = itemKey(j);
|
|
240
|
+
if (key !== undefined && j.pack === undefined && isActiveJudgment(m, now)) {
|
|
241
|
+
index.items.set(key, m);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return index;
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Plans one judgment against `index` (and updates the index, so later rows of the same import see
|
|
248
|
+
* it). `tombstones`: skip a decision withdrawn, corrected or purged here (imports and packs).
|
|
249
|
+
*/
|
|
250
|
+
function plan(input, ctx) {
|
|
251
|
+
const { judgment: base, redacted } = redactJudgment(normaliseJudgment(input, ctx.now));
|
|
252
|
+
const { index } = ctx;
|
|
253
|
+
if (ctx.importedKey !== undefined && index.imported.has(ctx.importedKey)) {
|
|
254
|
+
return { kind: 'duplicate' };
|
|
255
|
+
}
|
|
256
|
+
if (ctx.tombstones && index.tombstones.has(tombstoneKey(base)))
|
|
257
|
+
return { kind: 'tombstoned' };
|
|
258
|
+
if (ctx.via === 'pack' && index.fingerprints.has(fingerprint(base)))
|
|
259
|
+
return { kind: 'duplicate' };
|
|
260
|
+
const j = {
|
|
261
|
+
...base,
|
|
262
|
+
via: ctx.via,
|
|
263
|
+
...(ctx.pack !== undefined && { pack: ctx.pack }),
|
|
264
|
+
};
|
|
265
|
+
const key = itemKey(j);
|
|
266
|
+
// A pack never corrects anything: it only adds.
|
|
267
|
+
const found = ctx.corrects ?? (ctx.via === 'pack' || key === undefined ? undefined : index.items.get(key));
|
|
268
|
+
// An agent never retracts a person-owned judgment (recorded or confirmed by a person): it is
|
|
269
|
+
// recorded alongside instead. The item key already separates them; this is the second line.
|
|
270
|
+
const prior = ctx.via === 'mcp' && found !== undefined && recordedBy(found.judgment) === 'person'
|
|
271
|
+
? undefined
|
|
272
|
+
: found;
|
|
273
|
+
if (prior !== undefined && ctx.corrects === undefined && sameDecision(prior.judgment, j)) {
|
|
274
|
+
return { kind: 'unchanged', memory: prior };
|
|
275
|
+
}
|
|
276
|
+
const memory = judgmentMemory(j, ctx.deps.scope, ctx.deps.grant, ctx.now, ctx.deps.harness ?? ctx.via);
|
|
277
|
+
if (prior !== undefined) {
|
|
278
|
+
memory.supersedes = prior.id;
|
|
279
|
+
if (recordedBy(prior.judgment) === 'person') {
|
|
280
|
+
index.tombstones.add(prior.judgment.tombstone ?? tombstoneKey(prior.judgment));
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
if (key !== undefined && ctx.via !== 'pack')
|
|
284
|
+
index.items.set(key, memory);
|
|
285
|
+
if (ctx.via !== 'mcp')
|
|
286
|
+
index.fingerprints.add(fingerprint(base));
|
|
287
|
+
if (ctx.importedKey !== undefined)
|
|
288
|
+
index.imported.add(ctx.importedKey);
|
|
289
|
+
return { kind: 'write', planned: { memory, ...(prior !== undefined && { prior }), redacted } };
|
|
290
|
+
}
|
|
291
|
+
/** How many texts one embedding call carries during an import. */
|
|
292
|
+
const EMBED_BATCH = 128;
|
|
293
|
+
/**
|
|
294
|
+
* Embeds the planned judgments in batches, then writes them in order: a correction retracts its
|
|
295
|
+
* prior (kept, linked, its validity closed, its tombstone set) in the same atomic write. Returns
|
|
296
|
+
* how many could not be embedded (they are kept, lexical-only).
|
|
297
|
+
*/
|
|
298
|
+
async function commit(planned, deps, now) {
|
|
299
|
+
let embedFailed = 0;
|
|
300
|
+
if (deps.embed !== undefined) {
|
|
301
|
+
for (let i = 0; i < planned.length; i += EMBED_BATCH) {
|
|
302
|
+
const chunk = planned.slice(i, i + EMBED_BATCH);
|
|
303
|
+
try {
|
|
304
|
+
const vs = await deps.embed.embed(chunk.map((p) => embedText(p.memory.judgment)), 'document');
|
|
305
|
+
chunk.forEach((p, k) => {
|
|
306
|
+
const v = vs[k];
|
|
307
|
+
if (v !== undefined)
|
|
308
|
+
p.memory.embedding = v;
|
|
309
|
+
else
|
|
310
|
+
embedFailed++;
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
embedFailed += chunk.length;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
const iso = now.toISOString();
|
|
319
|
+
for (const p of planned) {
|
|
320
|
+
if (p.prior === undefined) {
|
|
321
|
+
await deps.store.upsert(deps.grant, p.memory);
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
// The full record (a listing leaves out the embedding), so the retracted one keeps it.
|
|
325
|
+
const old = (await deps.store.get(deps.grant, p.prior.id)) ?? p.prior;
|
|
326
|
+
const oldJ = old.judgment ?? p.prior.judgment;
|
|
327
|
+
await deps.store.upsertMany(deps.grant, [
|
|
328
|
+
{
|
|
329
|
+
...old,
|
|
330
|
+
judgment: { ...oldJ, tombstone: oldJ.tombstone ?? tombstoneKey(oldJ) },
|
|
331
|
+
expiredAt: iso,
|
|
332
|
+
endReason: 'corrected',
|
|
333
|
+
validTo: closeAt(old, now),
|
|
334
|
+
supersededBy: p.memory.id,
|
|
335
|
+
updatedAt: iso,
|
|
336
|
+
},
|
|
337
|
+
p.memory,
|
|
338
|
+
]);
|
|
339
|
+
}
|
|
340
|
+
return embedFailed;
|
|
341
|
+
}
|
|
342
|
+
/** When a judgment's validity ends on withdrawal or correction: now, never before it began. */
|
|
343
|
+
function closeAt(m, now) {
|
|
344
|
+
if (m.validTo !== null && Date.parse(m.validTo) <= now.getTime())
|
|
345
|
+
return m.validTo;
|
|
346
|
+
return new Date(Math.max(now.getTime(), Date.parse(m.validFrom))).toISOString();
|
|
347
|
+
}
|
|
348
|
+
/**
|
|
349
|
+
* Records one judgment. When the same expert already judged the same item (same `ref` and domain)
|
|
350
|
+
* and it was recorded the same way (by a person, or by an agent over MCP), the new judgment
|
|
351
|
+
* corrects the old one (kept, retracted, linked both ways), unless it says the same thing. An
|
|
352
|
+
* agent-recorded judgment never corrects a person's: it is recorded alongside. Throws
|
|
353
|
+
* `JudgmentInputError` for bad input.
|
|
354
|
+
*/
|
|
355
|
+
export async function recordJudgment(input, deps) {
|
|
356
|
+
const now = deps.now ?? new Date();
|
|
357
|
+
const corrects = deps.corrects !== undefined && deps.corrects.judgment !== undefined
|
|
358
|
+
? deps.corrects
|
|
359
|
+
: undefined;
|
|
360
|
+
if (corrects !== undefined && deps.via === 'mcp' && recordedBy(corrects.judgment) === 'person') {
|
|
361
|
+
throw new JudgmentInputError('an agent-recorded judgment cannot correct one a person recorded or confirmed');
|
|
362
|
+
}
|
|
363
|
+
const hasRef = typeof input?.ref === 'string' && input.ref.trim() !== '';
|
|
364
|
+
const index = hasRef && corrects === undefined
|
|
365
|
+
? await indexJudgments(deps.store, deps.grant, now)
|
|
366
|
+
: {
|
|
367
|
+
items: new Map(),
|
|
368
|
+
fingerprints: new Set(),
|
|
369
|
+
tombstones: new Set(),
|
|
370
|
+
imported: new Set(),
|
|
371
|
+
};
|
|
372
|
+
const r = plan(input, {
|
|
373
|
+
deps,
|
|
374
|
+
via: deps.via,
|
|
375
|
+
...(deps.pack !== undefined && { pack: deps.pack }),
|
|
376
|
+
...(corrects !== undefined && { corrects }),
|
|
377
|
+
index,
|
|
378
|
+
now,
|
|
379
|
+
tombstones: false,
|
|
380
|
+
});
|
|
381
|
+
if (r.kind === 'unchanged') {
|
|
382
|
+
return {
|
|
383
|
+
outcome: 'unchanged',
|
|
384
|
+
memory: r.memory,
|
|
385
|
+
redacted: redactJudgment(normaliseJudgment(input, now)).redacted,
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
if (r.kind !== 'write')
|
|
389
|
+
throw new Error('unreachable: a single record is never skipped');
|
|
390
|
+
const failed = await commit([r.planned], deps, now);
|
|
391
|
+
const { memory, prior, redacted } = r.planned;
|
|
392
|
+
return {
|
|
393
|
+
outcome: prior === undefined ? 'added' : 'corrected',
|
|
394
|
+
memory,
|
|
395
|
+
...(prior !== undefined && { supersedes: prior.id }),
|
|
396
|
+
redacted,
|
|
397
|
+
...(failed > 0 && { embedFailed: true }),
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Records many judgments (a bulk import of past decisions), reading the store once and embedding
|
|
402
|
+
* in batches. Every row is checked before anything is written, and more than
|
|
403
|
+
* `JUDGMENT_LIMITS.importRows` is refused. A row matching a decision withdrawn, corrected or purged
|
|
404
|
+
* here is skipped and counted (`tombstoned`).
|
|
405
|
+
*/
|
|
406
|
+
export async function importJudgments(rows, deps) {
|
|
407
|
+
if (rows.length > JUDGMENT_LIMITS.importRows) {
|
|
408
|
+
throw new JudgmentInputError(`an import holds at most ${JUDGMENT_LIMITS.importRows} rows; this one has ${rows.length}`);
|
|
409
|
+
}
|
|
410
|
+
const now = deps.now ?? new Date();
|
|
411
|
+
for (const [i, row] of rows.entries()) {
|
|
412
|
+
try {
|
|
413
|
+
normaliseJudgment(row, now);
|
|
414
|
+
}
|
|
415
|
+
catch (e) {
|
|
416
|
+
throw new JudgmentInputError(`row ${i + 1}: ${e.message}`);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
const index = await indexJudgments(deps.store, deps.grant, now);
|
|
420
|
+
const out = {
|
|
421
|
+
added: 0,
|
|
422
|
+
corrected: 0,
|
|
423
|
+
unchanged: 0,
|
|
424
|
+
tombstoned: 0,
|
|
425
|
+
redacted: 0,
|
|
426
|
+
embedFailed: 0,
|
|
427
|
+
};
|
|
428
|
+
const planned = [];
|
|
429
|
+
for (const row of rows) {
|
|
430
|
+
const r = plan(row, { deps, via: deps.via ?? 'import', index, now, tombstones: true });
|
|
431
|
+
if (r.kind === 'unchanged' || r.kind === 'duplicate')
|
|
432
|
+
out.unchanged++;
|
|
433
|
+
else if (r.kind === 'tombstoned')
|
|
434
|
+
out.tombstoned++;
|
|
435
|
+
else {
|
|
436
|
+
planned.push(r.planned);
|
|
437
|
+
if (r.planned.prior === undefined)
|
|
438
|
+
out.added++;
|
|
439
|
+
else
|
|
440
|
+
out.corrected++;
|
|
441
|
+
if (r.planned.redacted.length > 0)
|
|
442
|
+
out.redacted++;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
out.embedFailed = await commit(planned, deps, now);
|
|
446
|
+
return out;
|
|
447
|
+
}
|
|
448
|
+
/** The text a judgment is embedded and matched by: what was judged and why. */
|
|
449
|
+
function embedText(j) {
|
|
450
|
+
return `${j.subject.summary}\n${j.reason}\n${j.criteria.join(' ')} ${j.domain}`;
|
|
451
|
+
}
|
|
452
|
+
function sameDecision(a, b) {
|
|
453
|
+
return (a.verdict === b.verdict &&
|
|
454
|
+
a.reason === b.reason &&
|
|
455
|
+
a.subject.summary === b.subject.summary &&
|
|
456
|
+
a.criteria.join(',') === b.criteria.join(','));
|
|
457
|
+
}
|
|
458
|
+
function judgmentMemory(j, scope, grant, now, harness) {
|
|
459
|
+
const iso = now.toISOString();
|
|
460
|
+
// Judgments belong to the expert, not to a repository: unbound to any project.
|
|
461
|
+
const { projectId: _p, ...unbound } = scope;
|
|
462
|
+
const at = grant.home.level === 'user'
|
|
463
|
+
? { scope: unbound }
|
|
464
|
+
: addressed(homeAddress(grant), { agentId: scope.agentId });
|
|
465
|
+
return {
|
|
466
|
+
id: randomUUID(),
|
|
467
|
+
...at,
|
|
468
|
+
type: 'judgment',
|
|
469
|
+
text: judgmentText(j),
|
|
470
|
+
stub: `${j.verdict}: ${j.subject.summary.slice(0, 80)}`,
|
|
471
|
+
entities: [...new Set([j.domain, ...j.criteria])],
|
|
472
|
+
salience: 0.8,
|
|
473
|
+
// Pinned: never sent to cold, so judging keeps its embedding.
|
|
474
|
+
pinned: true,
|
|
475
|
+
tier: 'hot',
|
|
476
|
+
accessLog: { count: 0, recent: [], compactedSum: 0, compactedAt: null },
|
|
477
|
+
validFrom: j.decidedAt,
|
|
478
|
+
validTo: null,
|
|
479
|
+
expiredAt: null,
|
|
480
|
+
endReason: null,
|
|
481
|
+
lastConfirmedAt: null,
|
|
482
|
+
supersedes: null,
|
|
483
|
+
supersededBy: null,
|
|
484
|
+
source: { harness, decisions: [] },
|
|
485
|
+
judgment: j,
|
|
486
|
+
createdAt: iso,
|
|
487
|
+
updatedAt: iso,
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* Withdraws a judgment: it is marked withdrawn (by which operator, when), retracted and its
|
|
492
|
+
* validity closed, so judging, recall (every time mode), export and packs stop using it at once,
|
|
493
|
+
* and its tombstone keeps imports from bringing it back. The record stays for
|
|
494
|
+
* `oort judge list --all`; `purgeJudgment` removes its text. Returns false when it was already
|
|
495
|
+
* withdrawn or is not a judgment.
|
|
496
|
+
*/
|
|
497
|
+
export async function withdrawJudgment(store, grant, m, by, now = new Date()) {
|
|
498
|
+
if (m.type !== 'judgment' || m.judgment === undefined || m.judgment.withdrawn !== undefined) {
|
|
499
|
+
return false;
|
|
500
|
+
}
|
|
501
|
+
const iso = now.toISOString();
|
|
502
|
+
const full = (await store.get(grant, m.id)) ?? m;
|
|
503
|
+
const j = full.judgment ?? m.judgment;
|
|
504
|
+
await store.upsert(grant, {
|
|
505
|
+
...full,
|
|
506
|
+
judgment: {
|
|
507
|
+
...j,
|
|
508
|
+
withdrawn: { at: iso, by },
|
|
509
|
+
tombstone: j.tombstone ?? tombstoneKey(j),
|
|
510
|
+
},
|
|
511
|
+
expiredAt: full.expiredAt ?? iso,
|
|
512
|
+
endReason: full.endReason ?? 'expired',
|
|
513
|
+
validTo: closeAt(full, now),
|
|
514
|
+
updatedAt: iso,
|
|
515
|
+
});
|
|
516
|
+
return true;
|
|
517
|
+
}
|
|
518
|
+
/**
|
|
519
|
+
* Confirms an agent-recorded (`mcp`) judgment: a person (`by`, the operator) vouches that the
|
|
520
|
+
* expert really decided it, and judging uses it from then on. Only for a current, unconfirmed
|
|
521
|
+
* agent-recorded judgment; returns false otherwise. Never offered over MCP.
|
|
522
|
+
*/
|
|
523
|
+
export async function confirmJudgment(store, grant, m, by, now = new Date()) {
|
|
524
|
+
if (!isActiveJudgment(m, now) || !isUnconfirmedAgentJudgment(m.judgment))
|
|
525
|
+
return false;
|
|
526
|
+
const iso = now.toISOString();
|
|
527
|
+
const full = (await store.get(grant, m.id)) ?? m;
|
|
528
|
+
const j = { ...(full.judgment ?? m.judgment), confirmed: { at: iso, by } };
|
|
529
|
+
await store.upsert(grant, { ...full, judgment: j, text: judgmentText(j), updatedAt: iso });
|
|
530
|
+
return true;
|
|
531
|
+
}
|
|
532
|
+
/** The text a purged judgment keeps. */
|
|
533
|
+
const PURGED = '[purged]';
|
|
534
|
+
/**
|
|
535
|
+
* Purges a judgment's text: withdraws it if it is not already, keeps its tombstone, and blanks
|
|
536
|
+
* the subject, ref, reason, criteria, memory text and embedding. What is left is an audit stub:
|
|
537
|
+
* the expert, domain, verdict, times, who withdrew and purged it, and the tombstone. Returns false
|
|
538
|
+
* when it was already purged or is not a judgment.
|
|
539
|
+
*/
|
|
540
|
+
export async function purgeJudgment(store, grant, m, by, now = new Date()) {
|
|
541
|
+
if (m.type !== 'judgment' || m.judgment === undefined || m.judgment.purged !== undefined) {
|
|
542
|
+
return false;
|
|
543
|
+
}
|
|
544
|
+
const iso = now.toISOString();
|
|
545
|
+
const full = (await store.get(grant, m.id)) ?? m;
|
|
546
|
+
const j = full.judgment ?? m.judgment;
|
|
547
|
+
const { embedding: _e, ...rest } = full;
|
|
548
|
+
const stub = {
|
|
549
|
+
...j,
|
|
550
|
+
subject: { summary: PURGED },
|
|
551
|
+
reason: PURGED,
|
|
552
|
+
criteria: [],
|
|
553
|
+
withdrawn: j.withdrawn ?? { at: iso, by },
|
|
554
|
+
purged: { at: iso, by },
|
|
555
|
+
tombstone: j.tombstone ?? tombstoneKey(j),
|
|
556
|
+
};
|
|
557
|
+
await store.upsert(grant, {
|
|
558
|
+
...rest,
|
|
559
|
+
judgment: stub,
|
|
560
|
+
text: `Purged judgment by ${j.by} (${j.domain}).`,
|
|
561
|
+
stub: PURGED,
|
|
562
|
+
entities: [j.domain],
|
|
563
|
+
expiredAt: full.expiredAt ?? iso,
|
|
564
|
+
endReason: full.endReason ?? 'expired',
|
|
565
|
+
validTo: closeAt(full, now),
|
|
566
|
+
updatedAt: iso,
|
|
567
|
+
});
|
|
568
|
+
return true;
|
|
569
|
+
}
|
|
570
|
+
export const JUDGE_DEFAULTS = Object.freeze({
|
|
571
|
+
k: 5,
|
|
572
|
+
threshold: 0.7,
|
|
573
|
+
minSimilarityDense: 0.45,
|
|
574
|
+
minSimilarityLexical: 0.25,
|
|
575
|
+
minAgreement: 0.6,
|
|
576
|
+
/** A suggestion needs at least this many near precedents agreeing with it. */
|
|
577
|
+
minAgreeing: 2,
|
|
578
|
+
});
|
|
579
|
+
function cosine(a, b) {
|
|
580
|
+
let dot = 0;
|
|
581
|
+
let na = 0;
|
|
582
|
+
let nb = 0;
|
|
583
|
+
for (let i = 0; i < a.length; i++) {
|
|
584
|
+
const x = a[i] ?? 0;
|
|
585
|
+
const y = b[i] ?? 0;
|
|
586
|
+
dot += x * y;
|
|
587
|
+
na += x * x;
|
|
588
|
+
nb += y * y;
|
|
589
|
+
}
|
|
590
|
+
return na === 0 || nb === 0 ? 0 : dot / Math.sqrt(na * nb);
|
|
591
|
+
}
|
|
592
|
+
/** Overlap of informative words, as the cosine of the two word sets (0 to 1). */
|
|
593
|
+
function lexicalSimilarity(q, d) {
|
|
594
|
+
if (q.size === 0 || d.size === 0)
|
|
595
|
+
return 0;
|
|
596
|
+
let shared = 0;
|
|
597
|
+
for (const t of q)
|
|
598
|
+
if (d.has(t))
|
|
599
|
+
shared++;
|
|
600
|
+
return shared / Math.sqrt(q.size * d.size);
|
|
601
|
+
}
|
|
602
|
+
/**
|
|
603
|
+
* The active judgments of `filter`'s domain and expert (when it names them). Judgments imported
|
|
604
|
+
* from packs only with `includeImported`, unconfirmed agent-recorded ones only with
|
|
605
|
+
* `includeAgentRecorded`.
|
|
606
|
+
*/
|
|
607
|
+
export async function activeJudgments(store, grant, filter, now, embeddings = true) {
|
|
608
|
+
const domain = filter.domain === undefined ? undefined : judgmentLabel('domain', filter.domain, DOMAIN);
|
|
609
|
+
return (await store.list(grant, { embeddings })).filter((m) => isActiveJudgment(m, now) &&
|
|
610
|
+
(filter.includeImported === true || m.judgment.pack === undefined) &&
|
|
611
|
+
(filter.includeAgentRecorded === true || !isUnconfirmedAgentJudgment(m.judgment)) &&
|
|
612
|
+
(domain === undefined || m.judgment.domain === domain) &&
|
|
613
|
+
(filter.by === undefined || m.judgment.by === filter.by));
|
|
614
|
+
}
|
|
615
|
+
/** The label offered beside a single voted verdict: "not that one". */
|
|
616
|
+
const OTHER = 'other';
|
|
617
|
+
/**
|
|
618
|
+
* Applies the expert's judgments to `item`: finds the nearest active precedents (domain- and
|
|
619
|
+
* expert-filtered), then asks the decider for a choice over the verdicts they voted for, with the
|
|
620
|
+
* precedents and their reasons as state. It answers "needs the expert" instead of guessing when
|
|
621
|
+
* there are no precedents, none is near, fewer than two near precedents would back the answer,
|
|
622
|
+
* the near ones disagree, or the prediction is below `threshold`; and returns precedents only
|
|
623
|
+
* when there is no decider. Never writes anything.
|
|
624
|
+
*/
|
|
625
|
+
export async function judge(item, deps, opts = {}) {
|
|
626
|
+
const now = deps.now ?? new Date();
|
|
627
|
+
const k = Math.max(1, Math.min(opts.k ?? JUDGE_DEFAULTS.k, 20));
|
|
628
|
+
const threshold = opts.threshold ?? JUDGE_DEFAULTS.threshold;
|
|
629
|
+
const minAgreement = opts.minAgreement ?? JUDGE_DEFAULTS.minAgreement;
|
|
630
|
+
const summary = scrub(item.summary).text;
|
|
631
|
+
const details = item.details === undefined ? undefined : scrub(item.details).text;
|
|
632
|
+
const every = (await activeJudgments(deps.store, deps.grant, { ...item, includeImported: true, includeAgentRecorded: true }, now)).filter((m) => opts.exclude === undefined || !opts.exclude(m));
|
|
633
|
+
const notImported = opts.includeImported === true ? every : every.filter((m) => m.judgment.pack === undefined);
|
|
634
|
+
const all = opts.includeAgentRecorded === true
|
|
635
|
+
? notImported
|
|
636
|
+
: notImported.filter((m) => !isUnconfirmedAgentJudgment(m.judgment));
|
|
637
|
+
const importedExcluded = every.length - notImported.length;
|
|
638
|
+
const agentRecordedExcluded = notImported.length - all.length;
|
|
639
|
+
const degraded = [];
|
|
640
|
+
// Similarity: dense when an embedder is wired (precedents recorded without a vector are embedded
|
|
641
|
+
// now, not stored), lexical otherwise or when embedding fails.
|
|
642
|
+
let sims;
|
|
643
|
+
if (deps.embed !== undefined && all.length > 0) {
|
|
644
|
+
try {
|
|
645
|
+
const [q] = await deps.embed.embed([`${summary}\n${details ?? ''}`.trim()], 'query');
|
|
646
|
+
const missing = all.filter((m) => m.embedding?.length !== q?.length);
|
|
647
|
+
const fresh = [];
|
|
648
|
+
for (let i = 0; i < missing.length; i += EMBED_BATCH) {
|
|
649
|
+
fresh.push(...(await deps.embed.embed(missing.slice(i, i + EMBED_BATCH).map((m) => embedText(m.judgment)), 'document')));
|
|
650
|
+
}
|
|
651
|
+
const vec = new Map(missing.map((m, i) => [m.id, fresh[i]]));
|
|
652
|
+
if (q !== undefined) {
|
|
653
|
+
sims = all.map((m) => {
|
|
654
|
+
const v = vec.get(m.id) ?? m.embedding;
|
|
655
|
+
return v === undefined || v.length !== q.length ? 0 : Math.max(0, cosine(q, v));
|
|
656
|
+
});
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
catch {
|
|
660
|
+
degraded.push('embed-failed');
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
const mode = sims === undefined ? 'lexical' : 'dense';
|
|
664
|
+
if (sims === undefined) {
|
|
665
|
+
const q = informativeTokens(`${summary} ${details ?? ''}`);
|
|
666
|
+
sims = all.map((m) => lexicalSimilarity(q, informativeTokens(embedText(m.judgment))));
|
|
667
|
+
}
|
|
668
|
+
const minSimilarity = opts.minSimilarity ??
|
|
669
|
+
(mode === 'dense' ? JUDGE_DEFAULTS.minSimilarityDense : JUDGE_DEFAULTS.minSimilarityLexical);
|
|
670
|
+
const ranked = all
|
|
671
|
+
.map((m, i) => ({ m, similarity: sims[i] ?? 0 }))
|
|
672
|
+
.sort((a, b) => b.similarity - a.similarity ||
|
|
673
|
+
Date.parse(b.m.judgment.decidedAt) - Date.parse(a.m.judgment.decidedAt) ||
|
|
674
|
+
(a.m.id < b.m.id ? -1 : 1));
|
|
675
|
+
const near = ranked.filter((r) => r.similarity >= minSimilarity).slice(0, k);
|
|
676
|
+
const shown = near.length > 0 ? near : ranked.slice(0, k);
|
|
677
|
+
const precedents = shown.map(({ m, similarity }) => precedentOf(m, similarity, minSimilarity));
|
|
678
|
+
// Only verdicts the near precedents voted for are offered; a single one is offered against
|
|
679
|
+
// "other", so the decider can still say the item is not like them.
|
|
680
|
+
const voted = [...new Set(near.map((r) => r.m.judgment.verdict))];
|
|
681
|
+
const verdicts = voted.length === 1 ? [voted[0], voted[0] === OTHER ? 'something-else' : OTHER] : voted;
|
|
682
|
+
const agreement = shares(near.map((r) => [r.m.judgment.verdict, Math.max(r.similarity, 1e-6)]));
|
|
683
|
+
const base = {
|
|
684
|
+
precedents,
|
|
685
|
+
verdicts,
|
|
686
|
+
agreement,
|
|
687
|
+
similarity: mode,
|
|
688
|
+
threshold,
|
|
689
|
+
minSimilarity,
|
|
690
|
+
decisions: [],
|
|
691
|
+
...(degraded.length > 0 && { degraded }),
|
|
692
|
+
...(importedExcluded > 0 && { importedExcluded }),
|
|
693
|
+
...(agentRecordedExcluded > 0 && { agentRecordedExcluded }),
|
|
694
|
+
};
|
|
695
|
+
if (all.length === 0)
|
|
696
|
+
return { status: 'needs-expert', why: 'no-precedents', ...base };
|
|
697
|
+
if (near.length === 0)
|
|
698
|
+
return { status: 'needs-expert', why: 'too-far', ...base };
|
|
699
|
+
if (near.length < JUDGE_DEFAULTS.minAgreeing) {
|
|
700
|
+
return { status: 'needs-expert', why: 'too-few', ...base };
|
|
701
|
+
}
|
|
702
|
+
const top = Math.max(...Object.values(agreement));
|
|
703
|
+
if (top < minAgreement)
|
|
704
|
+
return { status: 'needs-expert', why: 'precedents-disagree', ...base };
|
|
705
|
+
if (deps.decide?.judge === undefined) {
|
|
706
|
+
return { status: 'precedents-only', why: 'no-decider', ...base };
|
|
707
|
+
}
|
|
708
|
+
let choice;
|
|
709
|
+
try {
|
|
710
|
+
choice = await deps.decide.judge({
|
|
711
|
+
item: {
|
|
712
|
+
summary,
|
|
713
|
+
...(details !== undefined && { details }),
|
|
714
|
+
...(item.ref !== undefined && { ref: scrub(item.ref).text }),
|
|
715
|
+
...(item.domain !== undefined && { domain: item.domain }),
|
|
716
|
+
},
|
|
717
|
+
verdicts,
|
|
718
|
+
precedents: near.map(({ m, similarity }) => ({
|
|
719
|
+
subject: m.judgment.subject.summary,
|
|
720
|
+
...(m.judgment.subject.ref !== undefined && { ref: m.judgment.subject.ref }),
|
|
721
|
+
verdict: m.judgment.verdict,
|
|
722
|
+
reason: m.judgment.reason,
|
|
723
|
+
criteria: m.judgment.criteria,
|
|
724
|
+
decidedAt: m.judgment.decidedAt,
|
|
725
|
+
similarity: round(similarity),
|
|
726
|
+
})),
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
catch {
|
|
730
|
+
return {
|
|
731
|
+
status: 'needs-expert',
|
|
732
|
+
why: 'decider-failed',
|
|
733
|
+
...base,
|
|
734
|
+
degraded: [...degraded, 'decider-failed'],
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
if (!verdicts.includes(choice.verdict)) {
|
|
738
|
+
return {
|
|
739
|
+
status: 'needs-expert',
|
|
740
|
+
why: 'decider-failed',
|
|
741
|
+
...base,
|
|
742
|
+
degraded: [...degraded, 'decider-failed'],
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
// Against "other", the prediction is always the voted verdict, at its own probability.
|
|
746
|
+
const verdict = voted.length === 1 ? voted[0] : choice.verdict;
|
|
747
|
+
const probability = Object.hasOwn(choice.probabilities, verdict)
|
|
748
|
+
? (choice.probabilities[verdict] ?? 0)
|
|
749
|
+
: verdict === choice.verdict
|
|
750
|
+
? choice.probability
|
|
751
|
+
: 0;
|
|
752
|
+
for (const p of precedents)
|
|
753
|
+
p.agrees = p.verdict === verdict;
|
|
754
|
+
const agreeing = near.filter((r) => r.m.judgment.verdict === verdict).length;
|
|
755
|
+
const prediction = { verdict, probability, probabilities: choice.probabilities };
|
|
756
|
+
const why = agreeing < JUDGE_DEFAULTS.minAgreeing
|
|
757
|
+
? 'too-few'
|
|
758
|
+
: probability < threshold
|
|
759
|
+
? 'low-confidence'
|
|
760
|
+
: undefined;
|
|
761
|
+
return {
|
|
762
|
+
status: why === undefined ? 'suggested' : 'needs-expert',
|
|
763
|
+
...(why !== undefined && { why }),
|
|
764
|
+
...base,
|
|
765
|
+
prediction,
|
|
766
|
+
decisions: choice.decisions,
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
const round = (x) => Math.round(x * 1000) / 1000;
|
|
770
|
+
function shares(weights) {
|
|
771
|
+
const total = weights.reduce((s, [, w]) => s + w, 0);
|
|
772
|
+
const sums = new Map();
|
|
773
|
+
for (const [v, w] of weights)
|
|
774
|
+
sums.set(v, (sums.get(v) ?? 0) + w);
|
|
775
|
+
return Object.fromEntries([...sums].map(([v, w]) => [v, total === 0 ? 0 : round(w / total)]));
|
|
776
|
+
}
|
|
777
|
+
function precedentOf(m, similarity, minSimilarity) {
|
|
778
|
+
const j = m.judgment;
|
|
779
|
+
return {
|
|
780
|
+
id: m.id,
|
|
781
|
+
subject: { ...j.subject },
|
|
782
|
+
verdict: j.verdict,
|
|
783
|
+
reason: j.reason,
|
|
784
|
+
criteria: [...j.criteria],
|
|
785
|
+
domain: j.domain,
|
|
786
|
+
by: j.by,
|
|
787
|
+
decidedAt: j.decidedAt,
|
|
788
|
+
similarity: round(similarity),
|
|
789
|
+
near: similarity >= minSimilarity,
|
|
790
|
+
via: j.via,
|
|
791
|
+
confirmed: !isUnconfirmedAgentJudgment(j),
|
|
792
|
+
...(j.pack !== undefined && {
|
|
793
|
+
pack: { id: j.pack.id, name: j.pack.name },
|
|
794
|
+
imported: true,
|
|
795
|
+
}),
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
// ---------------------------------------------------------------------------------------------
|
|
799
|
+
// Packs
|
|
800
|
+
// ---------------------------------------------------------------------------------------------
|
|
801
|
+
export const JUDGMENT_PACK_VERSION = 1;
|
|
802
|
+
/** Raised for a file that is not a readable judgment pack. */
|
|
803
|
+
export class JudgmentPackError extends Error {
|
|
804
|
+
name = 'JudgmentPackError';
|
|
805
|
+
}
|
|
806
|
+
/** Length caps for pack metadata. */
|
|
807
|
+
export const PACK_LIMITS = Object.freeze({ name: 120, exportedBy: 128, id: 128, time: 40 });
|
|
808
|
+
const PACK_ORIGIN_KEYS = [
|
|
809
|
+
'id',
|
|
810
|
+
'name',
|
|
811
|
+
'exportedAt',
|
|
812
|
+
'exportedBy',
|
|
813
|
+
'originalId',
|
|
814
|
+
'importedAt',
|
|
815
|
+
];
|
|
816
|
+
/** Plain one-line text: a string, not empty, at most `max`, no control, bidi or invisible characters. */
|
|
817
|
+
function packText(field, v, max) {
|
|
818
|
+
if (typeof v !== 'string' ||
|
|
819
|
+
v === '' ||
|
|
820
|
+
v.length > max ||
|
|
821
|
+
CONTROL.test(v) ||
|
|
822
|
+
/[\n\r]/.test(v) ||
|
|
823
|
+
INVISIBLE.test(v)) {
|
|
824
|
+
throw new JudgmentPackError(`${field} must be one line of plain text, at most ${max} characters`);
|
|
825
|
+
}
|
|
826
|
+
return v;
|
|
827
|
+
}
|
|
828
|
+
function packTime(field, v) {
|
|
829
|
+
const t = packText(field, v, PACK_LIMITS.time);
|
|
830
|
+
if (Number.isNaN(Date.parse(t)))
|
|
831
|
+
throw new JudgmentPackError(`${field} is not an ISO time`);
|
|
832
|
+
return t;
|
|
833
|
+
}
|
|
834
|
+
/** A line's claimed earlier origin: exactly the six fields, each plain text. */
|
|
835
|
+
function claimedOrigin(v, where) {
|
|
836
|
+
if (typeof v !== 'object' || v === null || Array.isArray(v)) {
|
|
837
|
+
throw new JudgmentPackError(`${where}: pack must be an object`);
|
|
838
|
+
}
|
|
839
|
+
const o = v;
|
|
840
|
+
const keys = Object.keys(o).sort();
|
|
841
|
+
if (keys.join(',') !== [...PACK_ORIGIN_KEYS].sort().join(',')) {
|
|
842
|
+
throw new JudgmentPackError(`${where}: pack must have exactly ${PACK_ORIGIN_KEYS.join(', ')}`);
|
|
843
|
+
}
|
|
844
|
+
packTime(`${where}: pack.importedAt`, o['importedAt']);
|
|
845
|
+
return {
|
|
846
|
+
untrusted: true,
|
|
847
|
+
id: packText(`${where}: pack.id`, o['id'], PACK_LIMITS.id),
|
|
848
|
+
name: packText(`${where}: pack.name`, o['name'], PACK_LIMITS.name),
|
|
849
|
+
exportedAt: packTime(`${where}: pack.exportedAt`, o['exportedAt']),
|
|
850
|
+
exportedBy: packText(`${where}: pack.exportedBy`, o['exportedBy'], PACK_LIMITS.exportedBy),
|
|
851
|
+
originalId: packText(`${where}: pack.originalId`, o['originalId'], PACK_LIMITS.id),
|
|
852
|
+
};
|
|
853
|
+
}
|
|
854
|
+
const lineOf = (m) => {
|
|
855
|
+
const { withdrawn: _w, confirmed: _c, purged: _p, tombstone: _t, pack, ...j } = m.judgment;
|
|
856
|
+
// The exporter's own import record travels, field by field; a claim it holds itself does not.
|
|
857
|
+
return {
|
|
858
|
+
id: m.id,
|
|
859
|
+
judgment: {
|
|
860
|
+
...j,
|
|
861
|
+
...(pack !== undefined && {
|
|
862
|
+
pack: {
|
|
863
|
+
id: pack.id,
|
|
864
|
+
name: pack.name,
|
|
865
|
+
exportedAt: pack.exportedAt,
|
|
866
|
+
exportedBy: pack.exportedBy,
|
|
867
|
+
originalId: pack.originalId,
|
|
868
|
+
importedAt: pack.importedAt,
|
|
869
|
+
},
|
|
870
|
+
}),
|
|
871
|
+
},
|
|
872
|
+
};
|
|
873
|
+
};
|
|
874
|
+
/**
|
|
875
|
+
* The digest of a pack: hex sha256 over its metadata (version, name, exportedAt, exportedBy, as a
|
|
876
|
+
* JSON array) and then each judgment line as written. Changing the name, the exporter, the time or
|
|
877
|
+
* any line changes it.
|
|
878
|
+
*/
|
|
879
|
+
export function packDigest(meta, lines) {
|
|
880
|
+
const h = createHash('sha256');
|
|
881
|
+
h.update(JSON.stringify([
|
|
882
|
+
'judgment-pack',
|
|
883
|
+
JUDGMENT_PACK_VERSION,
|
|
884
|
+
meta.name,
|
|
885
|
+
meta.exportedAt,
|
|
886
|
+
meta.exportedBy,
|
|
887
|
+
])).update('\n');
|
|
888
|
+
for (const l of lines)
|
|
889
|
+
h.update(l).update('\n');
|
|
890
|
+
return h.digest('hex');
|
|
891
|
+
}
|
|
892
|
+
const sortedUnique = (xs) => [...new Set(xs)].sort();
|
|
893
|
+
/**
|
|
894
|
+
* A pack of `judgments` (active ones only; withdrawn and corrected ones never travel), as JSON
|
|
895
|
+
* Lines: a manifest, then one line per judgment in id order. Attribution (`by`, `decidedAt`)
|
|
896
|
+
* travels with each judgment; embeddings do not (the importer embeds them with its own model).
|
|
897
|
+
*/
|
|
898
|
+
export function formatJudgmentPack(judgments, meta, now = meta.exportedAt) {
|
|
899
|
+
const active = judgments
|
|
900
|
+
.filter((m) => isActiveJudgment(m, now))
|
|
901
|
+
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
902
|
+
const lines = active.map((m) => JSON.stringify(lineOf(m)));
|
|
903
|
+
const bound = {
|
|
904
|
+
name: packText('name', meta.name, PACK_LIMITS.name),
|
|
905
|
+
exportedAt: meta.exportedAt.toISOString(),
|
|
906
|
+
exportedBy: packText('exportedBy', meta.exportedBy, PACK_LIMITS.exportedBy),
|
|
907
|
+
};
|
|
908
|
+
const manifest = {
|
|
909
|
+
oort: 'judgment-pack',
|
|
910
|
+
version: JUDGMENT_PACK_VERSION,
|
|
911
|
+
...bound,
|
|
912
|
+
count: lines.length,
|
|
913
|
+
digest: packDigest(bound, lines),
|
|
914
|
+
domains: sortedUnique(active.map((m) => m.judgment.domain)),
|
|
915
|
+
experts: sortedUnique(active.map((m) => m.judgment.by)),
|
|
916
|
+
};
|
|
917
|
+
return [JSON.stringify(manifest), ...lines].join('\n') + '\n';
|
|
918
|
+
}
|
|
919
|
+
/**
|
|
920
|
+
* Reads a pack and checks everything a person is shown before importing it: the manifest's
|
|
921
|
+
* metadata (plain text, length caps), the count, the digest over the metadata and the lines, every
|
|
922
|
+
* judgment as `recordJudgment` would check it, any claimed earlier origin (exact shape), and that
|
|
923
|
+
* the manifest's domains and experts are exactly those of the lines. The lines come back rebuilt
|
|
924
|
+
* from their checked fields only. The digest proves integrity, not authorship: a pack is as
|
|
925
|
+
* trustworthy as whoever handed it over. Throws `JudgmentPackError`.
|
|
926
|
+
*/
|
|
927
|
+
export function parseJudgmentPack(textIn) {
|
|
928
|
+
const raw = textIn.split('\n').filter((l) => l.trim() !== '');
|
|
929
|
+
const parse = (l, n) => {
|
|
930
|
+
try {
|
|
931
|
+
return JSON.parse(l);
|
|
932
|
+
}
|
|
933
|
+
catch {
|
|
934
|
+
throw new JudgmentPackError(`line ${n} is not JSON`);
|
|
935
|
+
}
|
|
936
|
+
};
|
|
937
|
+
const head = parse(raw[0] ?? '', 1);
|
|
938
|
+
if (head?.oort !== 'judgment-pack') {
|
|
939
|
+
throw new JudgmentPackError('line 1 is not an Oort judgment pack manifest');
|
|
940
|
+
}
|
|
941
|
+
if (head.version !== JUDGMENT_PACK_VERSION) {
|
|
942
|
+
throw new JudgmentPackError(`judgment pack version ${String(head.version)} is not supported`);
|
|
943
|
+
}
|
|
944
|
+
const meta = {
|
|
945
|
+
name: packText('the manifest name', head.name, PACK_LIMITS.name),
|
|
946
|
+
exportedAt: packTime('the manifest exportedAt', head.exportedAt),
|
|
947
|
+
exportedBy: packText('the manifest exportedBy', head.exportedBy, PACK_LIMITS.exportedBy),
|
|
948
|
+
};
|
|
949
|
+
const body = raw.slice(1);
|
|
950
|
+
if (body.length > JUDGMENT_LIMITS.importRows) {
|
|
951
|
+
throw new JudgmentPackError(`a pack holds at most ${JUDGMENT_LIMITS.importRows} judgments; this one has ${body.length}`);
|
|
952
|
+
}
|
|
953
|
+
if (head.count !== body.length) {
|
|
954
|
+
throw new JudgmentPackError(`the manifest says ${String(head.count)} judgments, the file holds ${body.length}`);
|
|
955
|
+
}
|
|
956
|
+
const digest = packDigest(meta, body);
|
|
957
|
+
if (head.digest !== digest) {
|
|
958
|
+
throw new JudgmentPackError('the digest does not match: the pack was changed or cut short');
|
|
959
|
+
}
|
|
960
|
+
const lines = body.map((l, i) => {
|
|
961
|
+
const where = `line ${i + 2}`;
|
|
962
|
+
const x = parse(l, i + 2);
|
|
963
|
+
if (typeof x !== 'object' ||
|
|
964
|
+
x === null ||
|
|
965
|
+
typeof x.judgment !== 'object' ||
|
|
966
|
+
x.judgment === null) {
|
|
967
|
+
throw new JudgmentPackError(`${where} is not a judgment`);
|
|
968
|
+
}
|
|
969
|
+
const id = packText(`${where}: id`, x.id, PACK_LIMITS.id);
|
|
970
|
+
const j = x.judgment;
|
|
971
|
+
if (typeof j.decidedAt !== 'string')
|
|
972
|
+
throw new JudgmentPackError(`${where}: decidedAt is missing`);
|
|
973
|
+
let n;
|
|
974
|
+
try {
|
|
975
|
+
n = normaliseJudgment(packInput(j), new Date(0));
|
|
976
|
+
}
|
|
977
|
+
catch (e) {
|
|
978
|
+
throw new JudgmentPackError(`${where}: ${e.message}`);
|
|
979
|
+
}
|
|
980
|
+
const claim = j.pack === undefined ? undefined : claimedOrigin(j.pack, where);
|
|
981
|
+
const via = ['cli', 'mcp', 'import', 'pack', 'sdk'].includes(j.via)
|
|
982
|
+
? j.via
|
|
983
|
+
: 'pack';
|
|
984
|
+
return {
|
|
985
|
+
id,
|
|
986
|
+
judgment: {
|
|
987
|
+
...n,
|
|
988
|
+
via,
|
|
989
|
+
...(claim !== undefined && {
|
|
990
|
+
pack: {
|
|
991
|
+
id: claim.id,
|
|
992
|
+
name: claim.name,
|
|
993
|
+
exportedAt: claim.exportedAt,
|
|
994
|
+
exportedBy: claim.exportedBy,
|
|
995
|
+
originalId: claim.originalId,
|
|
996
|
+
importedAt: j.pack.importedAt,
|
|
997
|
+
},
|
|
998
|
+
}),
|
|
999
|
+
},
|
|
1000
|
+
};
|
|
1001
|
+
});
|
|
1002
|
+
const domains = sortedUnique(lines.map((l) => l.judgment.domain));
|
|
1003
|
+
const experts = sortedUnique(lines.map((l) => l.judgment.by));
|
|
1004
|
+
const same = (a, b) => Array.isArray(a) &&
|
|
1005
|
+
a.every((x) => typeof x === 'string') &&
|
|
1006
|
+
JSON.stringify(sortedUnique(a)) === JSON.stringify(b) &&
|
|
1007
|
+
a.length === b.length;
|
|
1008
|
+
if (!same(head.domains, domains) || !same(head.experts, experts)) {
|
|
1009
|
+
throw new JudgmentPackError("the manifest's domains or experts do not match its judgments");
|
|
1010
|
+
}
|
|
1011
|
+
return {
|
|
1012
|
+
manifest: {
|
|
1013
|
+
oort: 'judgment-pack',
|
|
1014
|
+
version: JUDGMENT_PACK_VERSION,
|
|
1015
|
+
...meta,
|
|
1016
|
+
count: lines.length,
|
|
1017
|
+
digest,
|
|
1018
|
+
domains,
|
|
1019
|
+
experts,
|
|
1020
|
+
},
|
|
1021
|
+
lines,
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
/** A pack line's judgment as recording input. */
|
|
1025
|
+
export function packInput(j) {
|
|
1026
|
+
return {
|
|
1027
|
+
subject: j.subject?.summary,
|
|
1028
|
+
...(j.subject?.ref !== undefined && { ref: j.subject.ref }),
|
|
1029
|
+
verdict: j.verdict,
|
|
1030
|
+
verdicts: j.verdicts,
|
|
1031
|
+
reason: j.reason,
|
|
1032
|
+
criteria: j.criteria ?? [],
|
|
1033
|
+
domain: j.domain,
|
|
1034
|
+
by: j.by,
|
|
1035
|
+
decidedAt: j.decidedAt,
|
|
1036
|
+
};
|
|
1037
|
+
}
|
|
1038
|
+
/**
|
|
1039
|
+
* Imports a parsed pack into the grant's home node, reading the store once and embedding in
|
|
1040
|
+
* batches. Each judgment keeps its expert and decision time, and records where it came from:
|
|
1041
|
+
* `judgment.pack`, built only from this pack (its digest id, the digest-bound name, exporter and
|
|
1042
|
+
* time, and the line's id). An earlier origin the line claims is kept apart as `claimedEarlier`,
|
|
1043
|
+
* marked untrusted, and used for nothing.
|
|
1044
|
+
*
|
|
1045
|
+
* A line is skipped when this pack was imported before (same digest and line id) or the same
|
|
1046
|
+
* exact decision is already here, and reported as tombstoned when its decision was withdrawn,
|
|
1047
|
+
* corrected or purged here: a pack never brings back what an expert took back.
|
|
1048
|
+
*/
|
|
1049
|
+
export async function importJudgmentPack(pack, deps) {
|
|
1050
|
+
if (pack.lines.length > JUDGMENT_LIMITS.importRows) {
|
|
1051
|
+
throw new JudgmentPackError(`a pack holds at most ${JUDGMENT_LIMITS.importRows} judgments; this one has ${pack.lines.length}`);
|
|
1052
|
+
}
|
|
1053
|
+
const now = deps.now ?? new Date();
|
|
1054
|
+
const index = await indexJudgments(deps.store, deps.grant, now);
|
|
1055
|
+
const id = pack.manifest.digest.slice(0, 16);
|
|
1056
|
+
const out = { added: [], skipped: [], tombstoned: [] };
|
|
1057
|
+
const planned = [];
|
|
1058
|
+
for (const line of pack.lines) {
|
|
1059
|
+
const claim = line.judgment.pack;
|
|
1060
|
+
const origin = {
|
|
1061
|
+
id,
|
|
1062
|
+
name: pack.manifest.name,
|
|
1063
|
+
exportedAt: pack.manifest.exportedAt,
|
|
1064
|
+
exportedBy: pack.manifest.exportedBy,
|
|
1065
|
+
originalId: line.id,
|
|
1066
|
+
importedAt: now.toISOString(),
|
|
1067
|
+
...(claim !== undefined && {
|
|
1068
|
+
claimedEarlier: {
|
|
1069
|
+
untrusted: true,
|
|
1070
|
+
id: claim.id,
|
|
1071
|
+
name: claim.name,
|
|
1072
|
+
exportedAt: claim.exportedAt,
|
|
1073
|
+
exportedBy: claim.exportedBy,
|
|
1074
|
+
originalId: claim.originalId,
|
|
1075
|
+
},
|
|
1076
|
+
}),
|
|
1077
|
+
};
|
|
1078
|
+
const r = plan(packInput(line.judgment), {
|
|
1079
|
+
deps,
|
|
1080
|
+
via: 'pack',
|
|
1081
|
+
pack: origin,
|
|
1082
|
+
index,
|
|
1083
|
+
now,
|
|
1084
|
+
tombstones: true,
|
|
1085
|
+
importedKey: `${id}:${line.id}`,
|
|
1086
|
+
});
|
|
1087
|
+
if (r.kind === 'write') {
|
|
1088
|
+
planned.push(r.planned);
|
|
1089
|
+
out.added.push(r.planned.memory.id);
|
|
1090
|
+
}
|
|
1091
|
+
else if (r.kind === 'tombstoned')
|
|
1092
|
+
out.tombstoned.push(line.id);
|
|
1093
|
+
else
|
|
1094
|
+
out.skipped.push(line.id);
|
|
1095
|
+
}
|
|
1096
|
+
await commit(planned, deps, now);
|
|
1097
|
+
return out;
|
|
1098
|
+
}
|
|
1099
|
+
// ---------------------------------------------------------------------------------------------
|
|
1100
|
+
// Bulk import: past decisions from CSV or JSON Lines
|
|
1101
|
+
// ---------------------------------------------------------------------------------------------
|
|
1102
|
+
/** Rows of a CSV file (RFC 4180: quoted fields, `""` for a quote, newlines inside quotes). */
|
|
1103
|
+
export function parseCsv(input) {
|
|
1104
|
+
const rows = [];
|
|
1105
|
+
let row = [];
|
|
1106
|
+
let field = '';
|
|
1107
|
+
let quoted = false;
|
|
1108
|
+
let i = 0;
|
|
1109
|
+
const text = input.charCodeAt(0) === 0xfeff ? input.slice(1) : input;
|
|
1110
|
+
while (i < text.length) {
|
|
1111
|
+
const c = text[i];
|
|
1112
|
+
if (quoted) {
|
|
1113
|
+
if (c === '"' && text[i + 1] === '"') {
|
|
1114
|
+
field += '"';
|
|
1115
|
+
i += 2;
|
|
1116
|
+
continue;
|
|
1117
|
+
}
|
|
1118
|
+
if (c === '"')
|
|
1119
|
+
quoted = false;
|
|
1120
|
+
else
|
|
1121
|
+
field += c;
|
|
1122
|
+
i++;
|
|
1123
|
+
continue;
|
|
1124
|
+
}
|
|
1125
|
+
if (c === '"' && field === '')
|
|
1126
|
+
quoted = true;
|
|
1127
|
+
else if (c === ',') {
|
|
1128
|
+
row.push(field);
|
|
1129
|
+
field = '';
|
|
1130
|
+
}
|
|
1131
|
+
else if (c === '\n' || c === '\r') {
|
|
1132
|
+
if (c === '\r' && text[i + 1] === '\n')
|
|
1133
|
+
i++;
|
|
1134
|
+
row.push(field);
|
|
1135
|
+
rows.push(row);
|
|
1136
|
+
row = [];
|
|
1137
|
+
field = '';
|
|
1138
|
+
}
|
|
1139
|
+
else
|
|
1140
|
+
field += c;
|
|
1141
|
+
i++;
|
|
1142
|
+
}
|
|
1143
|
+
if (quoted)
|
|
1144
|
+
throw new JudgmentInputError('the CSV ends inside a quoted field');
|
|
1145
|
+
if (field !== '' || row.length > 0) {
|
|
1146
|
+
row.push(field);
|
|
1147
|
+
rows.push(row);
|
|
1148
|
+
}
|
|
1149
|
+
return rows.filter((r) => r.some((f) => f.trim() !== ''));
|
|
1150
|
+
}
|
|
1151
|
+
// Nothing here writes CSV. A future CSV writer (an export of judgments for a spreadsheet) must
|
|
1152
|
+
// guard every cell against formula injection: prefix a cell that starts with =, +, -, @, a tab or
|
|
1153
|
+
// a carriage return with a single quote, since reasons and subjects are free text.
|
|
1154
|
+
const COLUMN = Object.freeze({
|
|
1155
|
+
subject: 'subject',
|
|
1156
|
+
ref: 'ref',
|
|
1157
|
+
link: 'ref',
|
|
1158
|
+
verdict: 'verdict',
|
|
1159
|
+
decision: 'verdict',
|
|
1160
|
+
verdicts: 'verdicts',
|
|
1161
|
+
reason: 'reason',
|
|
1162
|
+
criteria: 'criteria',
|
|
1163
|
+
domain: 'domain',
|
|
1164
|
+
by: 'by',
|
|
1165
|
+
expert: 'by',
|
|
1166
|
+
decided_at: 'decidedAt',
|
|
1167
|
+
decidedat: 'decidedAt',
|
|
1168
|
+
});
|
|
1169
|
+
/** The field a column or JSON key names, by own property only (never `constructor` and the like). */
|
|
1170
|
+
const columnOf = (name) => {
|
|
1171
|
+
const k = name.trim().toLowerCase();
|
|
1172
|
+
return Object.hasOwn(COLUMN, k) ? COLUMN[k] : undefined;
|
|
1173
|
+
};
|
|
1174
|
+
function list(field, v) {
|
|
1175
|
+
if (v === undefined || v === null || v === '')
|
|
1176
|
+
return undefined;
|
|
1177
|
+
if (Array.isArray(v)) {
|
|
1178
|
+
if (v.length > JUDGMENT_LIMITS.verdicts) {
|
|
1179
|
+
throw new JudgmentInputError(`${field} has more than ${JUDGMENT_LIMITS.verdicts} entries`);
|
|
1180
|
+
}
|
|
1181
|
+
if (!v.every((x) => typeof x === 'string')) {
|
|
1182
|
+
throw new JudgmentInputError(`${field} must be a list of text labels`);
|
|
1183
|
+
}
|
|
1184
|
+
return v;
|
|
1185
|
+
}
|
|
1186
|
+
if (typeof v !== 'string')
|
|
1187
|
+
throw new JudgmentInputError(`${field} must be a list or text`);
|
|
1188
|
+
return v
|
|
1189
|
+
.split(/[;|]/)
|
|
1190
|
+
.map((x) => x.trim())
|
|
1191
|
+
.filter((x) => x !== '');
|
|
1192
|
+
}
|
|
1193
|
+
function rowInput(raw, defaults) {
|
|
1194
|
+
const str = (k) => {
|
|
1195
|
+
const v = raw[k];
|
|
1196
|
+
if (v === undefined || v === null || v === '')
|
|
1197
|
+
return undefined;
|
|
1198
|
+
if (typeof v === 'number' && Number.isFinite(v))
|
|
1199
|
+
return String(v);
|
|
1200
|
+
if (typeof v !== 'string')
|
|
1201
|
+
throw new JudgmentInputError(`${k} must be text`);
|
|
1202
|
+
return v;
|
|
1203
|
+
};
|
|
1204
|
+
const verdicts = list('verdicts', raw.verdicts);
|
|
1205
|
+
const criteria = list('criteria', raw.criteria);
|
|
1206
|
+
const ref = str('ref');
|
|
1207
|
+
const decidedAt = str('decidedAt');
|
|
1208
|
+
return {
|
|
1209
|
+
subject: str('subject') ?? '',
|
|
1210
|
+
verdict: str('verdict') ?? '',
|
|
1211
|
+
reason: str('reason') ?? '',
|
|
1212
|
+
domain: str('domain') ?? defaults.domain ?? '',
|
|
1213
|
+
by: str('by') ?? defaults.by ?? '',
|
|
1214
|
+
...(ref !== undefined && { ref }),
|
|
1215
|
+
...(verdicts !== undefined && { verdicts }),
|
|
1216
|
+
...(criteria !== undefined && { criteria }),
|
|
1217
|
+
...(decidedAt !== undefined && { decidedAt }),
|
|
1218
|
+
};
|
|
1219
|
+
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Past decisions from a CSV file (a header row naming the columns: subject, verdict, reason,
|
|
1222
|
+
* domain, by, and optionally ref, criteria and verdicts as `;`-separated lists, decided_at) or JSON
|
|
1223
|
+
* Lines (one object per line with the same keys; lists may be arrays). `defaults` fills a missing
|
|
1224
|
+
* domain or expert. More than `JUDGMENT_LIMITS.importRows` rows is refused. Each row is checked as
|
|
1225
|
+
* `recordJudgment` checks it; the first bad row throws `JudgmentInputError` naming its line.
|
|
1226
|
+
*/
|
|
1227
|
+
export function parseJudgmentRows(text, format, defaults = {}) {
|
|
1228
|
+
const at = new Date(0);
|
|
1229
|
+
const max = JUDGMENT_LIMITS.importRows;
|
|
1230
|
+
const tooMany = (n) => new JudgmentInputError(`an import holds at most ${max} rows; this one has ${n}`);
|
|
1231
|
+
const check = (raw, line) => {
|
|
1232
|
+
try {
|
|
1233
|
+
const input = rowInput(raw, defaults);
|
|
1234
|
+
normaliseJudgment(input, at);
|
|
1235
|
+
return input;
|
|
1236
|
+
}
|
|
1237
|
+
catch (e) {
|
|
1238
|
+
throw new JudgmentInputError(`line ${line}: ${e.message}`);
|
|
1239
|
+
}
|
|
1240
|
+
};
|
|
1241
|
+
if (format === 'jsonl') {
|
|
1242
|
+
const lines = text.split('\n').flatMap((l, i) => (l.trim() === '' ? [] : [[l, i]]));
|
|
1243
|
+
if (lines.length > max)
|
|
1244
|
+
throw tooMany(lines.length);
|
|
1245
|
+
return lines.map(([l, i]) => {
|
|
1246
|
+
let x;
|
|
1247
|
+
try {
|
|
1248
|
+
x = JSON.parse(l);
|
|
1249
|
+
}
|
|
1250
|
+
catch {
|
|
1251
|
+
throw new JudgmentInputError(`line ${i + 1} is not JSON`);
|
|
1252
|
+
}
|
|
1253
|
+
if (typeof x !== 'object' || x === null || Array.isArray(x)) {
|
|
1254
|
+
throw new JudgmentInputError(`line ${i + 1} is not an object`);
|
|
1255
|
+
}
|
|
1256
|
+
const raw = {};
|
|
1257
|
+
for (const [k, v] of Object.entries(x)) {
|
|
1258
|
+
const key = columnOf(k);
|
|
1259
|
+
if (key !== undefined)
|
|
1260
|
+
raw[key] = v;
|
|
1261
|
+
}
|
|
1262
|
+
return check(raw, i + 1);
|
|
1263
|
+
});
|
|
1264
|
+
}
|
|
1265
|
+
const rows = parseCsv(text);
|
|
1266
|
+
if (rows.length - 1 > max)
|
|
1267
|
+
throw tooMany(rows.length - 1);
|
|
1268
|
+
const header = (rows[0] ?? []).map((h) => columnOf(h));
|
|
1269
|
+
for (const need of ['subject', 'verdict', 'reason']) {
|
|
1270
|
+
if (!header.includes(need))
|
|
1271
|
+
throw new JudgmentInputError(`the CSV header has no ${need} column`);
|
|
1272
|
+
}
|
|
1273
|
+
return rows.slice(1).map((r, i) => {
|
|
1274
|
+
const raw = {};
|
|
1275
|
+
header.forEach((k, j) => {
|
|
1276
|
+
if (k !== undefined)
|
|
1277
|
+
raw[k] = r[j];
|
|
1278
|
+
});
|
|
1279
|
+
return check(raw, i + 2);
|
|
1280
|
+
});
|
|
1281
|
+
}
|
|
1282
|
+
// ---------------------------------------------------------------------------------------------
|
|
1283
|
+
// Evaluation: agreement with the expert on held-out decisions
|
|
1284
|
+
// ---------------------------------------------------------------------------------------------
|
|
1285
|
+
export const EVAL_THRESHOLDS = Object.freeze([0.5, 0.6, 0.7, 0.8, 0.9]);
|
|
1286
|
+
/**
|
|
1287
|
+
* Judges each held-out decision against the store (never recording it, and never using the item
|
|
1288
|
+
* itself as a precedent: leave-one-out by item ref and by exact decision) and compares the
|
|
1289
|
+
* prediction with the expert's verdict: agreement and coverage per domain at each threshold.
|
|
1290
|
+
* An item counts as predicted at threshold t when judging would say `suggested` with t as the
|
|
1291
|
+
* threshold. Needs a decider; without one nothing is ever predicted.
|
|
1292
|
+
*/
|
|
1293
|
+
export async function evaluateJudgments(heldOut, deps, opts = {}) {
|
|
1294
|
+
const thresholds = [...(opts.thresholds ?? EVAL_THRESHOLDS)].sort((a, b) => a - b);
|
|
1295
|
+
const now = deps.now ?? new Date();
|
|
1296
|
+
const rows = [];
|
|
1297
|
+
const modes = new Set();
|
|
1298
|
+
for (const input of heldOut) {
|
|
1299
|
+
const n = normaliseJudgment(input, now);
|
|
1300
|
+
const r = await judge({
|
|
1301
|
+
summary: n.subject.summary,
|
|
1302
|
+
domain: n.domain,
|
|
1303
|
+
by: n.by,
|
|
1304
|
+
...(n.subject.ref !== undefined && { ref: n.subject.ref }),
|
|
1305
|
+
}, deps, {
|
|
1306
|
+
...opts,
|
|
1307
|
+
threshold: 0,
|
|
1308
|
+
// Leave-one-out: the item itself, if the store holds it (same item ref in the domain, or
|
|
1309
|
+
// the very same decision), never counts as its own precedent.
|
|
1310
|
+
exclude: (m) => (opts.exclude?.(m) ?? false) ||
|
|
1311
|
+
fingerprint(m.judgment) === fingerprint(n) ||
|
|
1312
|
+
(n.subject.ref !== undefined &&
|
|
1313
|
+
m.judgment.subject.ref === n.subject.ref &&
|
|
1314
|
+
m.judgment.domain === n.domain),
|
|
1315
|
+
});
|
|
1316
|
+
modes.add(r.similarity);
|
|
1317
|
+
rows.push({ domain: n.domain, actual: n.verdict, r });
|
|
1318
|
+
}
|
|
1319
|
+
const summarise = (rs) => {
|
|
1320
|
+
const why = {};
|
|
1321
|
+
for (const x of rs)
|
|
1322
|
+
if (x.r.why !== undefined)
|
|
1323
|
+
why[x.r.why] = (why[x.r.why] ?? 0) + 1;
|
|
1324
|
+
return {
|
|
1325
|
+
items: rs.length,
|
|
1326
|
+
why,
|
|
1327
|
+
points: thresholds.map((t) => {
|
|
1328
|
+
const hit = rs.filter((x) => x.r.prediction !== undefined &&
|
|
1329
|
+
x.r.status === 'suggested' &&
|
|
1330
|
+
x.r.prediction.probability >= t);
|
|
1331
|
+
const agreed = hit.filter((x) => x.r.prediction.verdict === x.actual).length;
|
|
1332
|
+
return {
|
|
1333
|
+
threshold: t,
|
|
1334
|
+
coverage: rs.length === 0 ? 0 : round(hit.length / rs.length),
|
|
1335
|
+
agreement: hit.length === 0 ? null : round(agreed / hit.length),
|
|
1336
|
+
predicted: hit.length,
|
|
1337
|
+
agreed,
|
|
1338
|
+
};
|
|
1339
|
+
}),
|
|
1340
|
+
};
|
|
1341
|
+
};
|
|
1342
|
+
const domains = {};
|
|
1343
|
+
for (const d of [...new Set(rows.map((x) => x.domain))].sort()) {
|
|
1344
|
+
domains[d] = summarise(rows.filter((x) => x.domain === d));
|
|
1345
|
+
}
|
|
1346
|
+
domains['(all)'] = summarise(rows);
|
|
1347
|
+
return {
|
|
1348
|
+
domains,
|
|
1349
|
+
thresholds,
|
|
1350
|
+
similarity: modes.size === 1
|
|
1351
|
+
? [...modes][0]
|
|
1352
|
+
: modes.size === 0
|
|
1353
|
+
? 'lexical'
|
|
1354
|
+
: 'mixed',
|
|
1355
|
+
};
|
|
1356
|
+
}
|
|
1357
|
+
//# sourceMappingURL=judgment.js.map
|