@framers/agentos-ext-citation-check 0.2.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/claims.js ADDED
@@ -0,0 +1,792 @@
1
+ /**
2
+ * @file claims.ts
3
+ * @description A spoken claim checked against passages from any source: an encyclopedia article, a web page, a paper's
4
+ * abstract, a person's own notes or documents. The grounding guard's checker keeps the sentences about the claim;
5
+ * AgentOS's `CitationVerifier` names the best of them by cosine; one judge call, whose instruction, schema and quote
6
+ * rule are this pack's, says whether that sentence matches the claim, differs from it, or neither; the judge's quote is
7
+ * held to the sentence word for word (`verifyQuotes`); and the grounding guard decides by code over every passage
8
+ * found, the ones never sent to a model among them. Passages that arrive after the first decision are decided by code
9
+ * alone against it (`agree`), and a held "differs" is read again over every passage found by then (`revisit`).
10
+ *
11
+ * The host passes its embeddings call and its judge (`claimJudge` builds one on AgentOS's `generateObject`); the pack
12
+ * reads no key from the environment.
13
+ *
14
+ * @module @framers/agentos-ext-citation-check/claims
15
+ */
16
+ import { createHash, createHmac, hkdfSync } from 'node:crypto';
17
+ import { existsSync, readFileSync } from 'node:fs';
18
+ import { createRequire } from 'node:module';
19
+ import { join } from 'node:path';
20
+ import { collapse, fold, numberOf, words } from '@framers/agentos-ext-grounding-guard/values';
21
+ import { z } from 'zod';
22
+ /** The most words a judge's quote may hold. */
23
+ export const CLAIM_QUOTE_MAX_WORDS = 20;
24
+ /** How long a read of the public vectors may take before it counts as empty, in milliseconds. */
25
+ const VECTOR_READ_MS = 200;
26
+ /** The salt of the verdict key's derivation, which names what the key is for. */
27
+ const VERDICT_KEY_SALT = 'agentos-ext-citation-check claim verdict key';
28
+ /** This pack's version and the grounding guard's, read from their package files once. */
29
+ let packVersions;
30
+ /**
31
+ * This pack's version and the grounding guard's, which the verdict key reads, so an upgrade of either checks again.
32
+ * The grounding guard is found as Node would find it from this module; a version that cannot be read is `unknown`.
33
+ */
34
+ function versions() {
35
+ if (packVersions !== undefined)
36
+ return packVersions;
37
+ const read = (file) => {
38
+ try {
39
+ const version = JSON.parse(readFileSync(file, 'utf8')).version;
40
+ return typeof version === 'string' ? version : 'unknown';
41
+ }
42
+ catch {
43
+ return 'unknown';
44
+ }
45
+ };
46
+ let citationCheck = 'unknown';
47
+ let groundingGuard = 'unknown';
48
+ try {
49
+ const require = createRequire(import.meta.url);
50
+ citationCheck = read(require.resolve('../package.json'));
51
+ // The guard's package exports no package.json, so its file is found by Node's own search path from here.
52
+ for (const dir of require.resolve.paths('@framers/agentos-ext-grounding-guard') ?? []) {
53
+ const file = join(dir, '@framers', 'agentos-ext-grounding-guard', 'package.json');
54
+ if (existsSync(file)) {
55
+ groundingGuard = read(file);
56
+ break;
57
+ }
58
+ }
59
+ }
60
+ catch {
61
+ // A host that bundles the pack may give no module URL to search from.
62
+ }
63
+ packVersions = { citationCheck, groundingGuard };
64
+ return packVersions;
65
+ }
66
+ /** The SHA-256 hex of a text. */
67
+ function sha256(text) {
68
+ return createHash('sha256').update(text).digest('hex');
69
+ }
70
+ /**
71
+ * The hash a sentence's vector is stored under: the SHA-256 hex of the model, NUL, the dimensions, NUL and the text, as
72
+ * AgentOS's `embeddingInputHash` gives it.
73
+ */
74
+ function inputHashOf(embedding, text) {
75
+ return sha256(`${embedding.model}\u0000${embedding.dimensions}\u0000${text}`);
76
+ }
77
+ /** `work`'s value, or `undefined` when it rejects or has not settled within `ms`. */
78
+ function within(work, ms) {
79
+ return new Promise((resolve) => {
80
+ const timer = setTimeout(() => resolve(undefined), ms);
81
+ work.then((value) => {
82
+ clearTimeout(timer);
83
+ resolve(value);
84
+ }, () => {
85
+ clearTimeout(timer);
86
+ resolve(undefined);
87
+ });
88
+ });
89
+ }
90
+ /** Whether a vector holds `dimensions` finite numbers. */
91
+ function vectorOfSize(vector, dimensions) {
92
+ return Array.isArray(vector) && (dimensions === undefined || vector.length === dimensions) && vector.every((value) => Number.isFinite(value));
93
+ }
94
+ /** The judge's instruction's words for a kind of source: what the sentence is from, and the title above it. */
95
+ function sourceWords(kind) {
96
+ switch (kind) {
97
+ case 'encyclopedia':
98
+ return { from: 'an encyclopedia article', title: "its article's title" };
99
+ case 'official':
100
+ case 'news':
101
+ case 'page':
102
+ case 'forum':
103
+ return { from: 'a web page', title: "its page's title" };
104
+ case 'paper':
105
+ return { from: "a paper's abstract", title: "the paper's title" };
106
+ case 'note':
107
+ case 'document':
108
+ return { from: "the person's own notes or documents", title: "its note's or document's title" };
109
+ default:
110
+ return { from: 'a source', title: "its source's title" };
111
+ }
112
+ }
113
+ /**
114
+ * The judge's instruction for a sentence from a source of this kind: whether the sentence states the same fact as the
115
+ * claim (the same value, holder, place, maker or date), a different one, or neither, with the words it compared quoted
116
+ * from the sentence. Every instruction opens with "You compare one sentence heard in a room with one sentence from",
117
+ * then what the sentence is from: "an encyclopedia article" for `encyclopedia`; "a web page" for `official`, `news`,
118
+ * `page` and `forum`; "a paper's abstract" for `paper`; "the person's own notes or documents" for `note` and
119
+ * `document`; "a source" for any other word.
120
+ *
121
+ * @param kind - The named passage's kind ({@link ClaimPassage.kind}).
122
+ */
123
+ export function claimJudgeSystem(kind) {
124
+ const source = sourceWords(kind);
125
+ return [
126
+ `You compare one sentence heard in a room with one sentence from ${source.from}.`,
127
+ `HEARD is the sentence, transcribed by machine; it may be misheard. CHECK is the part being checked, rewritten to stand alone. SOURCE is that sentence, after ${source.title}.`,
128
+ 'relation "matches": SOURCE states the same fact for the same thing as CHECK: the same value, holder, place, maker or date, holding at the same time.',
129
+ 'relation "differs": SOURCE states a different value, holder, place, maker or date for the same thing as CHECK, or says it held only in the past while CHECK says it holds now.',
130
+ 'relation "none": everything else, including a sentence about something else, and a HEARD that is an opinion, a plan, a question, a joke or a hypothetical. When unsure, "none".',
131
+ `quote: copied character for character from SOURCE, at most ${CLAIM_QUOTE_MAX_WORDS} words, holding the words that state the fact you compared; empty for "none".`,
132
+ 'Use no knowledge beyond SOURCE and HEARD. Never judge the speaker.',
133
+ ].join('\n');
134
+ }
135
+ /**
136
+ * The judge's prompt: the heard sentence, the claim, and the one sentence with its source's title above it.
137
+ *
138
+ * @param heard - The whole sentence heard.
139
+ * @param statement - The claim rewritten to stand alone.
140
+ * @param title - The source's title; empty when it should not be sent.
141
+ * @param sentence - The source's sentence.
142
+ */
143
+ export function claimJudgePrompt(heard, statement, title, sentence) {
144
+ return `HEARD: ${heard}\nCHECK: ${statement}\nSOURCE: ${title}\n${sentence}`;
145
+ }
146
+ /** The judge's answer's schema. */
147
+ const CLAIM_JUDGEMENT = z.object({
148
+ relation: z.enum(['matches', 'differs', 'none']),
149
+ quote: z.string().describe(`Copied from SOURCE, at most ${CLAIM_QUOTE_MAX_WORDS} words; empty for none.`),
150
+ });
151
+ /** `work`'s outcome, or a rejection with the signal's reason the moment the signal aborts first. */
152
+ function untilAbort(work, signal) {
153
+ if (signal.aborted) {
154
+ // Nobody waits for `work` now, so its own rejection is handled here.
155
+ work.catch(() => undefined);
156
+ return Promise.reject(signal.reason);
157
+ }
158
+ return new Promise((resolve, reject) => {
159
+ const onAbort = () => reject(signal.reason);
160
+ signal.addEventListener('abort', onAbort, { once: true });
161
+ work.then((value) => {
162
+ signal.removeEventListener('abort', onAbort);
163
+ resolve(value);
164
+ }, (error) => {
165
+ signal.removeEventListener('abort', onAbort);
166
+ reject(error);
167
+ });
168
+ });
169
+ }
170
+ /**
171
+ * A judge on AgentOS's `generateObject`, with this pack's schema: one call, sent once (`maxAttempts: 1`), with no
172
+ * second call for an answer that is not the schema (`maxRetries: 0`) and no other provider (`fallbackProviders: []`),
173
+ * ended at `deadlineMs` or by the caller's signal, whichever comes first, the body's read included.
174
+ *
175
+ * @param options - The model (`provider:model`, such as `openai:gpt-6-luna`), the key, the base address (a stand-in in
176
+ * a host's tests), each call's deadline, the most tokens it may answer, and the reasoning effort for a model that takes
177
+ * one.
178
+ * @returns The judge. It rejects with the caller's reason when the caller's signal aborts, with a timeout when the
179
+ * deadline passes, and with the provider's error otherwise.
180
+ */
181
+ export function claimJudge(options) {
182
+ return async ({ system, prompt, signal }) => {
183
+ signal.throwIfAborted();
184
+ const bounded = AbortSignal.any([signal, AbortSignal.timeout(options.deadlineMs)]);
185
+ const call = (async () => {
186
+ const { generateObject } = await import('@framers/agentos');
187
+ return generateObject({
188
+ model: options.model,
189
+ schema: CLAIM_JUDGEMENT,
190
+ schemaName: 'judgement',
191
+ system,
192
+ prompt,
193
+ maxTokens: options.maxOutputTokens,
194
+ maxRetries: 0,
195
+ maxAttempts: 1,
196
+ fallbackProviders: [],
197
+ requestTimeout: options.deadlineMs,
198
+ abortSignal: bounded,
199
+ apiKey: options.apiKey,
200
+ ...(options.baseUrl === undefined ? {} : { baseUrl: options.baseUrl }),
201
+ ...(options.effort === undefined ? {} : { effort: options.effort }),
202
+ });
203
+ })();
204
+ const result = await untilAbort(call, bounded);
205
+ return {
206
+ value: { relation: result.object.relation, quote: result.object.quote },
207
+ usage: { in: result.usage.promptTokens ?? 0, out: result.usage.completionTokens ?? 0 },
208
+ };
209
+ };
210
+ }
211
+ /**
212
+ * One call's sentences: each passage read once, when the call first meets it, so everything the call sends or decides on
213
+ * is the passage as it was then, whatever another call or the host does to it meanwhile, and one passage is one sentence
214
+ * within the call.
215
+ */
216
+ function snapshot() {
217
+ const sentences = new Map();
218
+ return (passage) => {
219
+ let sentence = sentences.get(passage);
220
+ if (sentence === undefined) {
221
+ sentence = {
222
+ text: passage.content,
223
+ group: passage.group,
224
+ title: passage.title,
225
+ kind: passage.kind,
226
+ url: passage.url,
227
+ ref: passage.ref,
228
+ vector: passage.vector === undefined ? undefined : { ...passage.vector, embedding: [...passage.vector.embedding] },
229
+ publicKey: passage.publicKey === true,
230
+ passage,
231
+ };
232
+ sentences.set(passage, sentence);
233
+ }
234
+ return sentence;
235
+ };
236
+ }
237
+ /** The first sentence of each text, in order. */
238
+ function firstOfEachText(sentences) {
239
+ const seen = new Set();
240
+ return sentences.filter((sentence) => {
241
+ if (seen.has(sentence.text))
242
+ return false;
243
+ seen.add(sentence.text);
244
+ return true;
245
+ });
246
+ }
247
+ /** `a`, then each of `b` not in it. */
248
+ function unite(a, b) {
249
+ const seen = new Set(a);
250
+ return [...a, ...b.filter((item) => !seen.has(item))];
251
+ }
252
+ /** The characters of a span, or empty for none or one out of range. */
253
+ function spanText(text, span) {
254
+ if (span === null || span === undefined)
255
+ return '';
256
+ const { start, end } = span;
257
+ return Number.isInteger(start) && Number.isInteger(end) && start >= 0 && end <= text.length && start < end ? text.slice(start, end) : '';
258
+ }
259
+ /** Two marked values the same as written: their characters equal once white space and thousands separators are read alike. */
260
+ function sameAsWritten(a, b) {
261
+ return a !== '' && b !== '' && numberOf(collapse(a)) === numberOf(collapse(b));
262
+ }
263
+ /** Whether a judgement's quote may stand: a relation, one to {@link CLAIM_QUOTE_MAX_WORDS} words, word for word in the sentence. */
264
+ function quoteHolds(value, sentence, verifyQuotes) {
265
+ if (value.relation === 'none')
266
+ return false;
267
+ const count = words(value.quote).length;
268
+ return count >= 1 && count <= CLAIM_QUOTE_MAX_WORDS && verifyQuotes([{ text: value.quote, sourceIndex: 0 }], [sentence], { normalize: fold })[0]?.found === true;
269
+ }
270
+ /**
271
+ * Checks spoken claims against passages from any source, as `check` says, and decides later passages by code against a
272
+ * first decision. One checker serves many checks: it keeps nothing of a check but what `agree` and `revisit` read
273
+ * again, held weakly by the outcome `check` answered.
274
+ *
275
+ * @example
276
+ * ```ts
277
+ * import { ClaimChecker, claimJudge } from '@framers/agentos-ext-citation-check';
278
+ * import { ValueAgreementChecker } from '@framers/agentos-ext-grounding-guard';
279
+ *
280
+ * const checker = new ClaimChecker({
281
+ * embed: async (texts, signal) => myEmbeddings(texts, signal), // { vectors, usage: { in } }
282
+ * judge: claimJudge({ model: 'openai:gpt-6-luna', apiKey, deadlineMs: 8_000, maxOutputTokens: 400 }),
283
+ * checker: new ValueAgreementChecker({ names: true }),
284
+ * });
285
+ * const outcome = await checker.check(
286
+ * { heard, heardSentence, statement },
287
+ * passages, // ClaimPassage[]: { content, title, url?, group, kind }
288
+ * { signal },
289
+ * );
290
+ * ```
291
+ */
292
+ export class ClaimChecker {
293
+ embed;
294
+ judge;
295
+ checker;
296
+ maxPassages;
297
+ publicVectors;
298
+ grader;
299
+ verdictSecret;
300
+ vectorReadMs;
301
+ /**
302
+ * What a decided check keeps for `agree` and `revisit`, by the outcome `check` answered: every passage it found, the
303
+ * held-back ones among them.
304
+ */
305
+ decided = new WeakMap();
306
+ /**
307
+ * The texts each decision this checker answered was decided on (`check`'s and `revisit`'s alike), which the host's
308
+ * copy in `decision.texts` may since have been changed from. Kept by the decision's `pairs`, the offsets that index
309
+ * them: every decision answered holds its own copy of that array, which a host's shallow copy of the decision keeps.
310
+ */
311
+ decisionTexts = new WeakMap();
312
+ /**
313
+ * @param options - The embeddings call, the judge, the grounding guard's checker, the most passages compared, and
314
+ * the store of public vectors, the grader and the verdict secret.
315
+ * @throws {RangeError} For a `maxPassages` that is not a whole number of at least 1, a `vectorReadMs` that is not a
316
+ * positive number, or a grader whose dimensions are not a whole number of at least 1.
317
+ * @throws {TypeError} For `publicVectors` or `verdictSecret` without `grader`.
318
+ */
319
+ constructor(options) {
320
+ const maxPassages = options.maxPassages ?? 24;
321
+ if (!Number.isInteger(maxPassages) || maxPassages < 1)
322
+ throw new RangeError(`maxPassages must be a whole number of at least 1, not ${maxPassages}`);
323
+ const vectorReadMs = options.vectorReadMs ?? VECTOR_READ_MS;
324
+ if (!Number.isFinite(vectorReadMs) || vectorReadMs <= 0)
325
+ throw new RangeError(`vectorReadMs must be a positive number, not ${vectorReadMs}`);
326
+ const dimensions = options.grader?.embedding.dimensions;
327
+ if (dimensions !== undefined && (!Number.isInteger(dimensions) || dimensions < 1)) {
328
+ throw new RangeError(`the grader's dimensions must be a whole number of at least 1, not ${dimensions}`);
329
+ }
330
+ if (options.grader === undefined && options.publicVectors !== undefined)
331
+ throw new TypeError('publicVectors needs grader, whose embedding names the hash');
332
+ if (options.grader === undefined && options.verdictSecret !== undefined)
333
+ throw new TypeError('verdictSecret needs grader, which the verdict key reads');
334
+ this.embed = options.embed;
335
+ this.judge = options.judge;
336
+ this.checker = options.checker;
337
+ this.maxPassages = maxPassages;
338
+ this.publicVectors = options.publicVectors;
339
+ this.grader =
340
+ options.grader === undefined
341
+ ? undefined
342
+ : {
343
+ judgeModel: options.grader.judgeModel,
344
+ embedding: { ...options.grader.embedding },
345
+ ...(options.grader.rules === undefined ? {} : { rules: options.grader.rules }),
346
+ };
347
+ this.verdictSecret =
348
+ options.verdictSecret === undefined
349
+ ? undefined
350
+ : typeof options.verdictSecret === 'string'
351
+ ? new TextEncoder().encode(options.verdictSecret)
352
+ : Uint8Array.from(options.verdictSecret);
353
+ this.vectorReadMs = vectorReadMs;
354
+ }
355
+ /** A decision with the host's passages in place of the grounding guard's sentences, and their texts as read. */
356
+ toHost(decision) {
357
+ const texts = decision.sentences.map((sentence) => sentence.text);
358
+ const host = {
359
+ ...decision,
360
+ // Its own array, whatever the checker given reuses from one decision to the next.
361
+ ...(Array.isArray(decision.pairs) ? { pairs: [...decision.pairs] } : {}),
362
+ sentences: decision.sentences.map((sentence) => sentence.passage),
363
+ texts: [...texts],
364
+ };
365
+ if (Array.isArray(host.pairs))
366
+ this.decisionTexts.set(host.pairs, texts);
367
+ return host;
368
+ }
369
+ /** Every passage found for a decided check; the decision's own passages when the outcome is not one `check` answered. */
370
+ foundOf(first) {
371
+ const kept = this.decided.get(first)?.found;
372
+ return kept ?? unite(first.decision.sentences, [first.named]);
373
+ }
374
+ /**
375
+ * The verdict key: an HMAC-SHA-256, under a key derived by HKDF-SHA-256 from the secret with the context, over a
376
+ * canonical encoding of everything that decides the verdict. That is the grader (this pack's and the grounding guard's
377
+ * versions, the judge's model, the embedding model and dimensions, `maxPassages`), the claim, each compared sentence
378
+ * in order with its text, title, kind, group and `ref` and, for one ranked by a passage vector, the passage's id, its
379
+ * `inputHash` and a hash of the vector, and each sentence found with its text, group, `ref` and whether it is held
380
+ * back. `undefined` without a secret.
381
+ */
382
+ verdictKey(claim, compared, found, sendable, context) {
383
+ const secret = this.verdictSecret;
384
+ const grader = this.grader;
385
+ if (secret === undefined || grader === undefined)
386
+ return undefined;
387
+ const { citationCheck, groundingGuard } = versions();
388
+ const canonical = JSON.stringify([
389
+ 'claim-verdict/1',
390
+ [citationCheck, groundingGuard, grader.judgeModel, grader.embedding.model, grader.embedding.dimensions, this.maxPassages, grader.rules ?? null],
391
+ [claim.heard, claim.heardSentence, claim.statement],
392
+ compared.map((sentence) => [
393
+ sentence.text,
394
+ sentence.title,
395
+ sentence.kind,
396
+ sentence.group,
397
+ sentence.ref ?? null,
398
+ sentence.vector === undefined ? null : [sentence.vector.passageId, sentence.vector.inputHash, sha256(JSON.stringify(sentence.vector.embedding))],
399
+ ]),
400
+ found.map((sentence) => [sentence.text, sentence.group, sentence.ref ?? null, !sendable.has(sentence.passage)]),
401
+ ]);
402
+ const key = new Uint8Array(hkdfSync('sha256', secret, VERDICT_KEY_SALT, context, 32));
403
+ return createHmac('sha256', key).update(canonical).digest('base64url');
404
+ }
405
+ /**
406
+ * The sentences whose vectors may be read from and written to the public store: those marked `publicKey` that the
407
+ * host's `clean` passes. A `clean` that throws refuses its sentence.
408
+ */
409
+ async storable(sentences, signal) {
410
+ const store = this.publicVectors;
411
+ const marked = sentences.filter((sentence) => sentence.publicKey);
412
+ if (store === undefined || marked.length === 0)
413
+ return new Set();
414
+ const passed = await untilAbort(Promise.all(marked.map(async (sentence) => {
415
+ try {
416
+ return (await store.clean(sentence.text)) === true;
417
+ }
418
+ catch {
419
+ return false;
420
+ }
421
+ })), signal);
422
+ return new Set(marked.filter((_, index) => passed[index]));
423
+ }
424
+ /** The stored vectors of `storable`, by sentence; a store that rejects or answers after `vectorReadMs` gives none. */
425
+ async readPublic(storable, signal) {
426
+ const stored = new Map();
427
+ const store = this.publicVectors;
428
+ const grader = this.grader;
429
+ if (store === undefined || grader === undefined || storable.size === 0)
430
+ return stored;
431
+ const hashes = new Map([...storable].map((sentence) => [sentence, inputHashOf(grader.embedding, sentence.text)]));
432
+ const answered = await untilAbort(within(Promise.resolve().then(() => store.get([...new Set(hashes.values())])), this.vectorReadMs), signal);
433
+ if (answered === undefined)
434
+ return stored;
435
+ for (const [sentence, hash] of hashes) {
436
+ const vector = answered.get(hash);
437
+ if (vectorOfSize(vector, grader.embedding.dimensions))
438
+ stored.set(sentence, [...vector]);
439
+ }
440
+ return stored;
441
+ }
442
+ /** Writes the new public vectors, waiting at most `vectorReadMs`; a failed or late write changes nothing. */
443
+ async writePublic(rows) {
444
+ const store = this.publicVectors;
445
+ if (store === undefined || rows.length === 0)
446
+ return;
447
+ await within(Promise.resolve().then(() => store.put(rows)), this.vectorReadMs);
448
+ }
449
+ /**
450
+ * Checks a claim against passages, in order:
451
+ *
452
+ * 1. the grounding guard's `passages` keeps the sentences of `given` about the claim; none ends `no_passage`, with
453
+ * nothing sent anywhere. The first `maxPassages` kept, one a text (a text two sources share is compared as the
454
+ * first of them, whose title and kind the judge then reads), are the ones compared;
455
+ * 2. with `verdictSecret`, the verdict key is made; a key in `options.known` answers `known`, with nothing sent;
456
+ * 3. `options.beforeSend` is awaited once, before anything is sent; unless it answers true the check ends `limited`;
457
+ * 4. one sentence compared is named with no embeddings call (AgentOS's `CitationVerifier` with `pickOnly`), its
458
+ * verdict's `confidence` 0 and `scored` false;
459
+ * 5. otherwise the claim is embedded in one call with every compared sentence that carries no passage vector and has
460
+ * no stored public vector. A sentence marked `publicKey` that the host's `clean` passes is looked up in
461
+ * `publicVectors` first; a store that rejects or answers after `vectorReadMs` counts as empty. Sentence vectors and
462
+ * passage vectors are not on one scale, so the verifier ranks each class by its own cosine, and of the two classes'
463
+ * best the one earlier among the compared sentences is named; a sentence whose cosine is 0 or less is never named,
464
+ * and none ends `no_best`. The public vectors made are then stored, a failed or late write changing nothing;
465
+ * 6. the judge is asked once about the named sentence, under {@link claimJudgeSystem} for its kind, with
466
+ * {@link claimJudgePrompt}; `none` ends `judge_none`;
467
+ * 7. the judge's quote must hold one to {@link CLAIM_QUOTE_MAX_WORDS} words and stand in the sentence word for word
468
+ * (`verifyQuotes` with `fold`); else it ends `quote`;
469
+ * 8. the grounding guard's `decide` takes the judge's relation as its proposal, the named sentence, the quote, and as
470
+ * `found` every sentence kept in step 1 and every one of `options.found`; `null` ends `undecided`.
471
+ *
472
+ * `options.found` is every passage about the claim the host holds, the ones it may not send included (a sentence that
473
+ * holds personal data, for example): they are never embedded, judged or sent, and take part in step 8 alone, where a
474
+ * passage that agrees with every heard value stops a "differs". Left out, it is `given`'s sentences about the claim.
475
+ *
476
+ * @param heard - The claim heard, read once when the call begins.
477
+ * @param given - The passages that may be sent to the embeddings and the judge.
478
+ * @param options - The check's signal; every passage found; the verdict keys the host holds; the host's say before
479
+ * anything is sent; and the context the verdict key is derived for, such as the account's and the run's ids.
480
+ * @throws The signal's reason when it aborts, at once, whatever is pending; the embeddings' or the judge's error as it
481
+ * came, which the verifier alone would have swallowed; `beforeSend`'s error; a `RangeError` for a passage vector of
482
+ * another size than the claim's.
483
+ */
484
+ async check(heard, given, options) {
485
+ const { signal } = options;
486
+ signal.throwIfAborted();
487
+ // The claim as it is now: the key, the embeddings and the judge read this copy, whatever the host changes later.
488
+ const claim = { ...heard };
489
+ const usage = { judgeIn: 0, judgeOut: 0, embedIn: 0, vectorHits: 0 };
490
+ const state = {
491
+ judged: 0,
492
+ judgement: null,
493
+ failure: null,
494
+ key: undefined,
495
+ };
496
+ const ended = (reason) => ({
497
+ kind: 'ended',
498
+ reason,
499
+ judged: state.judged,
500
+ usage: { ...usage },
501
+ ...(state.key === undefined ? {} : { key: state.key }),
502
+ });
503
+ // The passages as they are now: this call reads nothing of them later.
504
+ const sentenceOf = snapshot();
505
+ // 1. The sentences about the claim that may be sent.
506
+ const kept = this.checker.passages(claim.heard, given.map((passage) => sentenceOf(passage))).found;
507
+ if (kept.length === 0)
508
+ return ended('no_passage');
509
+ // One passage a text, the first found: the judge reads the title and the kind of the very passage the outcome names.
510
+ // Every passage found still meets the rule in step 8.
511
+ const compared = firstOfEachText(kept).slice(0, this.maxPassages);
512
+ const found = unite(kept, (options.found ?? []).map((passage) => sentenceOf(passage)));
513
+ // 2. The verdict key, and a verdict the host holds.
514
+ state.key = this.verdictKey(claim, compared, found, new Set(given), options.verdictContext ?? '');
515
+ if (state.key !== undefined && options.known !== undefined && new Set(options.known).has(state.key)) {
516
+ return { kind: 'known', key: state.key, judged: 0, usage: { ...usage } };
517
+ }
518
+ // 3. The host's say, once, before anything is sent.
519
+ if (options.beforeSend !== undefined) {
520
+ const go = await untilAbort(Promise.resolve().then(() => options.beforeSend?.()), signal);
521
+ if (go !== true)
522
+ return ended('limited');
523
+ }
524
+ const [{ CitationVerifier }, { verifyQuotes }] = await untilAbort(Promise.all([import('@framers/agentos/cognition/rag/citation/index'), import('@framers/agentos-ext-grounding-guard')]), signal);
525
+ let named;
526
+ let verdict;
527
+ if (compared.length === 1) {
528
+ // 4. One sentence compared: named with no embeddings call, the judge deciding.
529
+ const about = new Map(compared.map((sentence) => [sentence.text, { title: sentence.title, kind: sentence.kind }]));
530
+ const verifier = new CitationVerifier({
531
+ pickOnly: true,
532
+ supportThreshold: 0,
533
+ unverifiableThreshold: 0,
534
+ embedFn: async (texts) => {
535
+ signal.throwIfAborted();
536
+ const embedded = await this.embed(texts, signal);
537
+ usage.embedIn += embedded.usage.in;
538
+ if (embedded.vectors.length !== texts.length)
539
+ throw new Error(`the embeddings answered ${embedded.vectors.length} vectors for ${texts.length} texts`);
540
+ return embedded.vectors;
541
+ },
542
+ // The verifier keeps its verdict when its NLI function throws, so a judge that failed is held here and thrown
543
+ // after `verify`.
544
+ nliFn: async (premise, hypothesis) => {
545
+ try {
546
+ signal.throwIfAborted();
547
+ const source = about.get(premise) ?? { title: '', kind: '' };
548
+ state.judged += 1;
549
+ const answer = await this.judge({
550
+ system: claimJudgeSystem(source.kind),
551
+ prompt: claimJudgePrompt(claim.heardSentence, hypothesis, source.title, premise),
552
+ signal,
553
+ });
554
+ usage.judgeIn += answer.usage.in;
555
+ usage.judgeOut += answer.usage.out;
556
+ const holds = quoteHolds(answer.value, premise, verifyQuotes);
557
+ state.judgement = { value: answer.value, holds };
558
+ if (answer.value.relation === 'none' || !holds)
559
+ return { label: 'neutral', score: 0 };
560
+ return { label: answer.value.relation === 'matches' ? 'entailment' : 'contradiction', score: 1 };
561
+ }
562
+ catch (error) {
563
+ state.failure = { error };
564
+ throw error;
565
+ }
566
+ },
567
+ });
568
+ const verified = await untilAbort(verifier.verify([claim.statement], compared.map((sentence) => ({ content: sentence.text, title: sentence.title, ...(sentence.url === undefined ? {} : { url: sentence.url }) }))), signal);
569
+ if (state.failure !== null)
570
+ throw state.failure.error;
571
+ signal.throwIfAborted();
572
+ verdict = verified.claims[0];
573
+ named = verdict?.sourceIndex === undefined ? undefined : compared[verdict.sourceIndex];
574
+ if (verdict === undefined || named === undefined)
575
+ return ended('no_best');
576
+ }
577
+ else {
578
+ // 5. The claim and every sentence without a vector, in one call; each class ranked by its own cosine.
579
+ const dimensions = this.grader?.embedding.dimensions;
580
+ const byPassage = compared.filter((sentence) => sentence.vector !== undefined && vectorOfSize(sentence.vector.embedding, dimensions));
581
+ const passageSet = new Set(byPassage);
582
+ const bySentence = compared.filter((sentence) => !passageSet.has(sentence));
583
+ const storable = await this.storable(bySentence, signal);
584
+ const stored = await this.readPublic(storable, signal);
585
+ usage.vectorHits = stored.size;
586
+ const toEmbed = bySentence.filter((sentence) => !stored.has(sentence));
587
+ signal.throwIfAborted();
588
+ const texts = [claim.statement, ...toEmbed.map((sentence) => sentence.text)];
589
+ const embedded = await untilAbort(this.embed(texts, signal), signal);
590
+ usage.embedIn += embedded.usage.in;
591
+ if (embedded.vectors.length !== texts.length)
592
+ throw new Error(`the embeddings answered ${embedded.vectors.length} vectors for ${texts.length} texts`);
593
+ // Every vector the embeddings answered is of the grader's size, or the claim's with no grader, and finite.
594
+ const size = this.grader?.embedding.dimensions ?? embedded.vectors[0]?.length;
595
+ for (const vector of embedded.vectors) {
596
+ if (!vectorOfSize(vector, size) || vector.length === 0) {
597
+ throw new Error(`the embeddings answered a vector that is not ${size} finite numbers`);
598
+ }
599
+ }
600
+ const [statementVector] = embedded.vectors;
601
+ // With no grader, a passage vector of another size than the claim's cannot be ranked beside it.
602
+ for (const sentence of byPassage) {
603
+ const length = sentence.vector.embedding.length;
604
+ if (length !== statementVector.length) {
605
+ throw new RangeError(`a passage vector holds ${length} numbers, and the claim's ${statementVector.length}`);
606
+ }
607
+ }
608
+ const vectorOf = new Map();
609
+ toEmbed.forEach((sentence, index) => vectorOf.set(sentence, embedded.vectors[index + 1]));
610
+ for (const [sentence, vector] of stored)
611
+ vectorOf.set(sentence, vector);
612
+ for (const sentence of byPassage)
613
+ vectorOf.set(sentence, sentence.vector.embedding);
614
+ // Every vector is known, so the verifier's one embeddings call, the claim's, is answered from the call above.
615
+ const rank = async (sentences) => {
616
+ if (sentences.length === 0)
617
+ return undefined;
618
+ const verifier = new CitationVerifier({
619
+ supportThreshold: 0,
620
+ unverifiableThreshold: 0,
621
+ embedFn: async (claims) => claims.map(() => [...statementVector]),
622
+ });
623
+ const ranked = await verifier.verify([claim.statement], sentences.map((sentence) => ({
624
+ content: sentence.text,
625
+ title: sentence.title,
626
+ ...(sentence.url === undefined ? {} : { url: sentence.url }),
627
+ embedding: vectorOf.get(sentence),
628
+ })));
629
+ const best = ranked.claims[0];
630
+ const sentence = best?.sourceIndex === undefined ? undefined : sentences[best.sourceIndex];
631
+ return best === undefined || sentence === undefined ? undefined : { sentence, verdict: best };
632
+ };
633
+ const ranked = [await rank(bySentence), await rank(byPassage)].filter((one) => one !== undefined);
634
+ ranked.sort((a, b) => compared.indexOf(a.sentence) - compared.indexOf(b.sentence));
635
+ const best = ranked[0];
636
+ // The public vectors made, stored once the claim is ranked.
637
+ const grader = this.grader;
638
+ const rows = grader === undefined
639
+ ? []
640
+ : toEmbed
641
+ .filter((sentence) => storable.has(sentence) && vectorOfSize(vectorOf.get(sentence), grader.embedding.dimensions))
642
+ .map((sentence) => ({ hash: inputHashOf(grader.embedding, sentence.text), vector: [...vectorOf.get(sentence)] }));
643
+ if (best === undefined) {
644
+ await untilAbort(this.writePublic(rows), signal);
645
+ signal.throwIfAborted();
646
+ return ended('no_best');
647
+ }
648
+ // 6. The judge on the named sentence.
649
+ signal.throwIfAborted();
650
+ named = best.sentence;
651
+ state.judged += 1;
652
+ const answer = await untilAbort(this.judge({
653
+ system: claimJudgeSystem(named.kind),
654
+ prompt: claimJudgePrompt(claim.heardSentence, claim.statement, named.title, named.text),
655
+ signal,
656
+ }), signal);
657
+ usage.judgeIn += answer.usage.in;
658
+ usage.judgeOut += answer.usage.out;
659
+ const holds = quoteHolds(answer.value, named.text, verifyQuotes);
660
+ state.judgement = { value: answer.value, holds };
661
+ // The verdict the verifier gives with this judge as its NLI function: a "differs" whose quote holds contradicts.
662
+ verdict = {
663
+ ...best.verdict,
664
+ sourceIndex: compared.indexOf(named),
665
+ verdict: answer.value.relation === 'differs' && holds ? 'contradicted' : best.verdict.verdict,
666
+ };
667
+ await untilAbort(this.writePublic(rows), signal);
668
+ signal.throwIfAborted();
669
+ }
670
+ const judgement = state.judgement;
671
+ if (judgement === null || judgement.value.relation === 'none')
672
+ return ended('judge_none');
673
+ // 7. The quote.
674
+ if (!judgement.holds)
675
+ return ended('quote');
676
+ // 8. The rule, by code, over every passage found.
677
+ const decision = this.checker.decide({
678
+ heardWords: claim.heard,
679
+ heardSentence: claim.heardSentence,
680
+ proposed: judgement.value.relation,
681
+ named: [named],
682
+ found,
683
+ quote: judgement.value.quote,
684
+ });
685
+ if (decision === null)
686
+ return ended('undecided');
687
+ const outcome = {
688
+ kind: 'decided',
689
+ decision: this.toHost(decision),
690
+ named: named.passage,
691
+ quote: judgement.value.quote,
692
+ verdict,
693
+ judged: state.judged,
694
+ usage: { ...usage },
695
+ ...(state.key === undefined ? {} : { key: state.key }),
696
+ };
697
+ this.decided.set(outcome, { found: found.map((sentence) => sentence.passage) });
698
+ return outcome;
699
+ }
700
+ /**
701
+ * The passages that arrived after a first decision and agree with it, decided by code alone, with no model call. Each
702
+ * source of `more` (by `group`) other than the first decision's is read once, by its passage the grounding guard
703
+ * ranks first about the claim, which joins when:
704
+ *
705
+ * 1. it holds every content stem of the heard words;
706
+ * 2. the grounding guard decides the first decision's relation for it, on the values, over every passage found (the
707
+ * first check's and every one of `more`, whatever its length);
708
+ * 3. its first value pair pairs the same heard value as the first decision's;
709
+ * 4. for a "differs", its marked value is the first decision's as written; for a "matches", each of the first
710
+ * decision's pairs finds its partner written alike, the decision's sentence read as the call that answered it
711
+ * (`check`, or `revisit` for a decision the host put in the outcome's place) read it, whatever the host has
712
+ * written into `decision.texts` since.
713
+ *
714
+ * A "disagree", a decision on the judge's quote alone, and a decision with no value pair take no passage.
715
+ *
716
+ * @param first - The decided outcome `check` answered.
717
+ * @param claim - The claim it checked.
718
+ * @param more - The later passages.
719
+ * @returns The passages that join, in the grounding guard's order.
720
+ */
721
+ agree(first, claim, more) {
722
+ const { decision } = first;
723
+ if (decision.relation === 'disagree' || decision.basis !== 'values')
724
+ return [];
725
+ const firstPairs = Array.isArray(decision.pairs) ? decision.pairs : [];
726
+ const lead = firstPairs[0];
727
+ if (lead === undefined)
728
+ return [];
729
+ // The decision's marks and pairs are offsets into its sentence as it was decided: the text kept with those pairs for
730
+ // a decision this checker answered (`check`'s, or `revisit`'s the host put in its place, a shallow copy of either
731
+ // included), whatever the host has written into `decision.texts` since.
732
+ const firstSentence = this.decisionTexts.get(firstPairs)?.[0] ?? decision.texts?.[0] ?? (decision.sentences[0] ?? first.named).content;
733
+ // Every passage as it is now, the first check's included (a host may have changed one since).
734
+ const sentenceOf = snapshot();
735
+ const later = more.map((passage) => sentenceOf(passage));
736
+ // The passages that may join are those about the claim; the veto reads every passage, as `check`'s `found`.
737
+ const candidates = this.checker.passages(claim.heard, later).found;
738
+ const found = unite(this.foundOf(first).map((passage) => sentenceOf(passage)), later);
739
+ const heardStems = this.checker.stems(claim.heard);
740
+ const considered = new Set(decision.sentences.map((passage) => passage.group));
741
+ const joined = [];
742
+ for (const sentence of candidates) {
743
+ if (considered.has(sentence.group))
744
+ continue;
745
+ // Each other source once: its best-ranked passage stands for it.
746
+ considered.add(sentence.group);
747
+ const own = this.checker.stems(sentence.text);
748
+ if (![...heardStems].every((stem) => own.has(stem)))
749
+ continue;
750
+ const decided = this.checker.decide({ heardWords: claim.heard, heardSentence: claim.heardSentence, proposed: decision.relation, named: [sentence], found });
751
+ if (decided === null || decided.relation !== decision.relation || decided.basis !== 'values')
752
+ continue;
753
+ const pairs = Array.isArray(decided.pairs) ? decided.pairs : [];
754
+ const other = pairs[0];
755
+ if (other === undefined || other.heard.start !== lead.heard.start || other.heard.end !== lead.heard.end)
756
+ continue;
757
+ const same = decision.relation === 'differs'
758
+ ? sameAsWritten(spanText(firstSentence, decision.marks[0]), spanText(sentence.text, decided.marks[0]))
759
+ : firstPairs.every((pair) => {
760
+ const at = pairs.find((candidate) => candidate.heard.start === pair.heard.start && candidate.heard.end === pair.heard.end);
761
+ return at !== undefined && sameAsWritten(spanText(firstSentence, pair.source), spanText(sentence.text, at.source));
762
+ });
763
+ if (same)
764
+ joined.push(sentence.passage);
765
+ }
766
+ return joined;
767
+ }
768
+ /**
769
+ * The first decision read again over every passage found by then: the grounding guard's rule with the same proposal,
770
+ * named passage and quote, its `found` every passage the check found (held-back ones among them) and every passage of
771
+ * `found`, as `check` reads its own `found`, whatever its length. A "differs" a later passage stops (one that agrees with every heard value) answers `null`,
772
+ * so a host that holds a "differs" until its slower sources have answered shows nothing.
773
+ *
774
+ * @param first - The decided outcome `check` answered.
775
+ * @param claim - The claim it checked.
776
+ * @param found - Every passage found by then.
777
+ * @returns The decision, its relation the first's, or `null`.
778
+ */
779
+ revisit(first, claim, found) {
780
+ const { decision } = first;
781
+ // Every passage as it is now, the first check's included; none is held to the sentence limits a passage the judge
782
+ // reads is, as `check`'s `found`.
783
+ const sentenceOf = snapshot();
784
+ const named = decision.relation === 'disagree' ? decision.sentences.map((passage) => sentenceOf(passage)) : [sentenceOf(first.named)];
785
+ const all = unite(unite(named, this.foundOf(first).map((passage) => sentenceOf(passage))), found.map((passage) => sentenceOf(passage)));
786
+ const again = this.checker.decide({ heardWords: claim.heard, heardSentence: claim.heardSentence, proposed: decision.relation, named, found: all, quote: first.quote });
787
+ if (again === null || again.relation !== decision.relation)
788
+ return null;
789
+ return this.toHost(again);
790
+ }
791
+ }
792
+ //# sourceMappingURL=claims.js.map