@crewhaus/provability 0.0.0-stage → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +300 -2
- package/dist/artifact.d.ts +9 -0
- package/dist/artifact.js +38 -0
- package/dist/builtins.d.ts +32 -0
- package/dist/builtins.js +46 -0
- package/dist/evidence.d.ts +326 -0
- package/dist/evidence.js +1391 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +22 -0
- package/dist/levels.d.ts +19 -0
- package/dist/levels.js +42 -0
- package/dist/origin.d.ts +83 -0
- package/dist/origin.js +296 -0
- package/dist/private-address.d.ts +34 -0
- package/dist/private-address.js +209 -0
- package/dist/purity.d.ts +18 -0
- package/dist/purity.js +15 -0
- package/dist/render.d.ts +47 -0
- package/dist/render.js +315 -0
- package/dist/store.d.ts +59 -0
- package/dist/store.js +319 -0
- package/dist/summary.d.ts +8 -0
- package/dist/summary.js +22 -0
- package/dist/types.d.ts +459 -0
- package/dist/types.js +2 -0
- package/dist/vacuous.d.ts +50 -0
- package/dist/vacuous.js +423 -0
- package/dist/verify.d.ts +176 -0
- package/dist/verify.js +3608 -0
- package/package.json +47 -4
package/dist/verify.js
ADDED
|
@@ -0,0 +1,3608 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `verifyDocument` — the four verifiers over one answer (design §2, §6).
|
|
3
|
+
*
|
|
4
|
+
* The gate, the `ProvabilityPreflight` tool, `crewhaus provability check`
|
|
5
|
+
* and the research report gate all call this one function, so a draft the
|
|
6
|
+
* model preflights is graded exactly as the gate will grade it. Every rung
|
|
7
|
+
* is deterministic: the citation grammar, bodies the run itself read, tool
|
|
8
|
+
* results the log recorded verbatim, re-execution of pure tools, and a proof
|
|
9
|
+
* checker's recorded verdict. The three things that reach outside — the HEAD
|
|
10
|
+
* request, the replay and the paraphrase-waiver judge — are INJECTED, and
|
|
11
|
+
* each is called only when the floor needs it (see {@link verifyDocument}).
|
|
12
|
+
* The package imports no model adapter: the judge is a callback.
|
|
13
|
+
*/
|
|
14
|
+
import { existsSync, readFileSync, realpathSync, statSync } from "node:fs";
|
|
15
|
+
import * as path from "node:path";
|
|
16
|
+
import { MIN_CLAIM_TOKENS, NEGATIONS, STOPWORDS, citationDefinitions, claimFigureKeys, claimQuery, claimWords, classifyDefinition, compoundValues, containsQuote, containsValue, contentTokens, coverage, definitionGloss, extractClaims, extractNumbers, figureKey, findSequence, foldTypography, governingNegation, indexSource, isWaivableMiss, locateClaim, normalizeNumber, quotedSpans, tightestWindows, tokenize, writtenSpan, } from "@crewhaus/claim-ledger";
|
|
17
|
+
import { resolveContained } from "@crewhaus/tool-safety/fs";
|
|
18
|
+
import { isProofArtifact, isProofId } from "./artifact";
|
|
19
|
+
import { FACT_CROSS_CHECK, PROOF_CHECK, PROVABILITY_PREFLIGHT, RETRIEVE, builtinKey, isMcpTool, } from "./builtins";
|
|
20
|
+
import { ProvabilityError, groundingBody, inputPlaces, inputStrings, isReader, originFieldValues, restrictToTurns, servedFrom, servesPage, sha256Of, writtenPlaces, writtenText, } from "./evidence";
|
|
21
|
+
import { isProvabilityLevel, levelRank, maxRung, meetsFloor } from "./levels";
|
|
22
|
+
import { doiUrl, hasUserinfo, hostMatches, hostOf, isPrivateHost, normalizePath, normalizeUrl, registrableDomain, } from "./origin";
|
|
23
|
+
import { PROVABILITY_LEVELS, } from "./types";
|
|
24
|
+
import { backstopClaims } from "./vacuous";
|
|
25
|
+
/** A claim's text as the bundle carries it (design §5.3). */
|
|
26
|
+
const MAX_CLAIM_CHARS = 400;
|
|
27
|
+
/** A frozen excerpt (design §5.3, the FrozenProof discipline). */
|
|
28
|
+
const MAX_EXCERPT_CHARS = 240;
|
|
29
|
+
/** A formal statement as the bundle shows it beside the prose. */
|
|
30
|
+
const MAX_STATEMENT_CHARS = 1_000;
|
|
31
|
+
/**
|
|
32
|
+
* Distinct URLs one verification sends a HEAD to under `sources.resolve:
|
|
33
|
+
* head` (design §6.1). The answer is model-written, so without a ceiling it
|
|
34
|
+
* decides how many outbound requests the gate makes and how long it waits
|
|
35
|
+
* on them; a URL past the ceiling is `unresolved` — not checked is not
|
|
36
|
+
* reachable — and the detail names the cap.
|
|
37
|
+
*/
|
|
38
|
+
export const MAX_HEAD_REQUESTS = 50;
|
|
39
|
+
/**
|
|
40
|
+
* How many of those HEADs are in flight at once, and how long one
|
|
41
|
+
* verification waits on them in all (design §6.1, §18: the gate's cost is
|
|
42
|
+
* bounded). Sent one at a time under a per-request timeout alone, fifty
|
|
43
|
+
* URLs on hosts that never answer held one pass for minutes, and again on
|
|
44
|
+
* every retry; past the deadline a URL is `unresolved` — not checked is not
|
|
45
|
+
* reachable — whatever is still pending.
|
|
46
|
+
*/
|
|
47
|
+
export const HEAD_CONCURRENCY = 6;
|
|
48
|
+
export const HEAD_DEADLINE_MS = 15_000;
|
|
49
|
+
/**
|
|
50
|
+
* Fetched text one verification reads while grounding, in tokens: each
|
|
51
|
+
* claim reads every body it cites end to end, so a long answer citing a
|
|
52
|
+
* large page costs claims × body — 200 claims over a 4 MB page held the
|
|
53
|
+
* harness thread for twelve seconds per pass. Past this a source binding
|
|
54
|
+
* is not read for grounding: it reaches `cited` and, at a higher floor,
|
|
55
|
+
* reads `below_floor` with a detail naming the cap (§18: bounded gate
|
|
56
|
+
* cost). The verifier also yields to the event loop between claims, so
|
|
57
|
+
* the work it does do never holds the process for long at a stretch.
|
|
58
|
+
*/
|
|
59
|
+
export const MAX_GROUNDING_TOKENS = 40_000_000;
|
|
60
|
+
let groundingBudget = MAX_GROUNDING_TOKENS;
|
|
61
|
+
/** Test seam: a smaller grounding budget, or (no argument) {@link MAX_GROUNDING_TOKENS} again. */
|
|
62
|
+
export function _setGroundingBudgetForTest(tokens) {
|
|
63
|
+
groundingBudget = tokens ?? MAX_GROUNDING_TOKENS;
|
|
64
|
+
}
|
|
65
|
+
/** How long the verifier works between yields to the event loop. */
|
|
66
|
+
const YIELD_EVERY_MS = 10;
|
|
67
|
+
/** How much of a span `echoCheck` re-reads when the excerpt was cut at 240. */
|
|
68
|
+
const MAX_ECHO_SPAN_CHARS = 64 * 1024;
|
|
69
|
+
/**
|
|
70
|
+
* Whitespace collapsed, cut to `max` with an ellipsis. Only a prefix of a
|
|
71
|
+
* long text is collapsed — a 2 MB tool result clipped to 240 characters
|
|
72
|
+
* must not be rewritten whole — unless that prefix is so full of
|
|
73
|
+
* whitespace that it collapses to fewer than `max` characters.
|
|
74
|
+
*/
|
|
75
|
+
export function clip(text, max) {
|
|
76
|
+
const bound = max * 8 + 1_024;
|
|
77
|
+
if (text.length > bound) {
|
|
78
|
+
const head = text.slice(0, bound).replace(/\s+/g, " ").trimStart();
|
|
79
|
+
if (head.length > max)
|
|
80
|
+
return `${head.slice(0, max - 1).trimEnd()}…`;
|
|
81
|
+
}
|
|
82
|
+
const t = text.replace(/\s+/g, " ").trim();
|
|
83
|
+
return t.length > max ? `${t.slice(0, max - 1).trimEnd()}…` : t;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* At most `max` characters of `text` around `[from, to)`, whitespace
|
|
87
|
+
* collapsed, an ellipsis marking each cut: the frozen excerpt of a long
|
|
88
|
+
* body or result, centred on what met the floor (design §5.3: a bundle
|
|
89
|
+
* stays human-checkable after the body is gone, so its excerpt must show
|
|
90
|
+
* the figure or the words that grounded the claim, not the text's first
|
|
91
|
+
* 240 characters). When `[from, to)` itself is longer, its start is kept.
|
|
92
|
+
*/
|
|
93
|
+
export function clipAround(text, from, to, max) {
|
|
94
|
+
const reach = max * 4;
|
|
95
|
+
const lo = Math.max(0, from - reach);
|
|
96
|
+
const hi = Math.min(text.length, to + reach);
|
|
97
|
+
const squash = (s) => s.replace(/\s+/g, " ");
|
|
98
|
+
let before = squash(text.slice(lo, from));
|
|
99
|
+
const focus = squash(text.slice(from, to)).trim();
|
|
100
|
+
let after = squash(text.slice(to, hi));
|
|
101
|
+
if (lo === 0)
|
|
102
|
+
before = before.trimStart();
|
|
103
|
+
if (hi === text.length)
|
|
104
|
+
after = after.trimEnd();
|
|
105
|
+
if (focus.length >= max - 1) {
|
|
106
|
+
const lead = before !== "" ? "…" : "";
|
|
107
|
+
return `${lead}${focus.slice(0, max - 1 - lead.length).trimEnd()}…`;
|
|
108
|
+
}
|
|
109
|
+
const room = max - focus.length;
|
|
110
|
+
let b = Math.min(before.length, Math.floor(room / 2));
|
|
111
|
+
let a = Math.min(after.length, room - b);
|
|
112
|
+
b = Math.min(before.length, room - a);
|
|
113
|
+
const fits = () => b + a + (b < before.length ? 1 : 0) + (a < after.length ? 1 : 0) <= room;
|
|
114
|
+
while (!fits()) {
|
|
115
|
+
if (a >= b && a > 0)
|
|
116
|
+
a -= 1;
|
|
117
|
+
else
|
|
118
|
+
b -= 1;
|
|
119
|
+
}
|
|
120
|
+
const left = `${b < before.length ? "…" : ""}${before.slice(before.length - b).trimStart()}`;
|
|
121
|
+
const right = `${after.slice(0, a).trimEnd()}${a < after.length ? "…" : ""}`;
|
|
122
|
+
return `${left}${focus}${right}`;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* The figures a text states, by the keys the figure rule compares them under
|
|
126
|
+
* (`claimFigureKeys`): `thirty` and `30` are one, "-500" is `-500` and not
|
|
127
|
+
* `500`, the `M` of "$5M" is `1000000`. A date or a time of day is also
|
|
128
|
+
* its whole value (`compoundValues`), so `2024-03-05` and `2024-05-03` — the
|
|
129
|
+
* same three numbers — differ (design §6.2, acceptance A: a wrong figure
|
|
130
|
+
* never binds).
|
|
131
|
+
*/
|
|
132
|
+
function figureKeys(text) {
|
|
133
|
+
const keys = new Set([...claimFigureKeys(text).values()].flat());
|
|
134
|
+
for (const value of compoundValues(text))
|
|
135
|
+
keys.add(value.key);
|
|
136
|
+
return keys;
|
|
137
|
+
}
|
|
138
|
+
/** sha256 of a result with its whitespace collapsed: what a replay is compared under. */
|
|
139
|
+
function outputDigest(content) {
|
|
140
|
+
return sha256Of(content.replace(/\s+/g, " ").trim());
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The fewest of `of` claims that clears `threshold` — `met / of ≥ threshold`
|
|
144
|
+
* asked in integers, so `0.7 × 10` (7.000000000000001 in floating point)
|
|
145
|
+
* needs 7, not 8.
|
|
146
|
+
*/
|
|
147
|
+
function needed(threshold, of) {
|
|
148
|
+
return Math.max(0, Math.ceil(threshold * of - 1e-9));
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The reasons a claim is unmet: `no_citation` for a claim with no marker at
|
|
152
|
+
* all (it has zero bindings — the representation design §5.1's nudge reads),
|
|
153
|
+
* else the reason of every binding that is not `ok`, in order, each once. An
|
|
154
|
+
* empty list for a met or assumption claim.
|
|
155
|
+
*/
|
|
156
|
+
export function claimReasons(claim) {
|
|
157
|
+
if (claim.status !== "unmet")
|
|
158
|
+
return [];
|
|
159
|
+
if (claim.bindings.length === 0)
|
|
160
|
+
return ["no_citation"];
|
|
161
|
+
const out = [];
|
|
162
|
+
for (const binding of claim.bindings) {
|
|
163
|
+
if (binding.reason !== undefined && !out.includes(binding.reason))
|
|
164
|
+
out.push(binding.reason);
|
|
165
|
+
}
|
|
166
|
+
return out;
|
|
167
|
+
}
|
|
168
|
+
function belowFloorDetail(kind, rung, floor) {
|
|
169
|
+
const most = kind === "source"
|
|
170
|
+
? "a source reaches grounded at most"
|
|
171
|
+
: kind === "execution"
|
|
172
|
+
? "a tool result reaches checked at most"
|
|
173
|
+
: `this binding reaches ${rung}`;
|
|
174
|
+
return `${most}; the floor is ${floor} — cite ${floor === "proven" ? "a proof:prf_… artifact" : "a tool:<toolUseId> or proof:prf_… binding"}`;
|
|
175
|
+
}
|
|
176
|
+
/** A reached state measured against the floor. */
|
|
177
|
+
function finish(ref, kind, reached, floor) {
|
|
178
|
+
const ok = meetsFloor(reached.rung, floor);
|
|
179
|
+
const reason = ok ? undefined : (reached.failure?.reason ?? "below_floor");
|
|
180
|
+
const detail = ok
|
|
181
|
+
? undefined
|
|
182
|
+
: (reached.failure?.detail ?? belowFloorDetail(kind, reached.rung, floor));
|
|
183
|
+
return {
|
|
184
|
+
ref,
|
|
185
|
+
kind,
|
|
186
|
+
rung: reached.rung,
|
|
187
|
+
ok,
|
|
188
|
+
...(reason !== undefined ? { reason } : {}),
|
|
189
|
+
...(detail !== undefined ? { detail } : {}),
|
|
190
|
+
...(reached.origin !== undefined ? { origin: reached.origin } : {}),
|
|
191
|
+
...(reached.sha256 !== undefined ? { sha256: reached.sha256 } : {}),
|
|
192
|
+
...(reached.excerpt !== undefined ? { excerpt: reached.excerpt } : {}),
|
|
193
|
+
...(reached.replayed !== undefined ? { replayed: reached.replayed } : {}),
|
|
194
|
+
...(reached.judgeWaived === true ? { judgeWaived: true } : {}),
|
|
195
|
+
...(reached.writtenBy !== undefined ? { writtenBy: reached.writtenBy } : {}),
|
|
196
|
+
...(reached.checker !== undefined ? { checker: reached.checker } : {}),
|
|
197
|
+
...(reached.statement !== undefined ? { statement: reached.statement } : {}),
|
|
198
|
+
...(reached.assumptions !== undefined && reached.assumptions.length > 0
|
|
199
|
+
? { assumptions: reached.assumptions }
|
|
200
|
+
: {}),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
function fail(rung, reason, detail, extra = {}) {
|
|
204
|
+
return { ...extra, rung, failure: { reason, detail } };
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* One guarded HEAD through the caller's resolver: a status, `"egress"` when
|
|
208
|
+
* the run's egress fabric refused to send it, or `undefined` for no answer.
|
|
209
|
+
*/
|
|
210
|
+
async function sendHead(run, url, signal) {
|
|
211
|
+
const resolve = run.opts.resolveHead;
|
|
212
|
+
if (resolve === undefined)
|
|
213
|
+
return undefined;
|
|
214
|
+
try {
|
|
215
|
+
const status = await resolve(url, signal);
|
|
216
|
+
if (typeof status === "object" && status !== null && status.refused === "egress") {
|
|
217
|
+
return "egress";
|
|
218
|
+
}
|
|
219
|
+
return typeof status === "number" && Number.isFinite(status) ? status : undefined;
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
return undefined;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Send the HEADs of `urls` — the distinct URLs this verification will ask
|
|
227
|
+
* about, in the order it asks, at most {@link MAX_HEAD_REQUESTS} — before
|
|
228
|
+
* any is awaited: {@link HEAD_CONCURRENCY} in flight at once, every one
|
|
229
|
+
* under one deadline for the whole verification (`headDeadlineMs`). A URL
|
|
230
|
+
* still unanswered, or not yet sent, when the deadline passes comes to
|
|
231
|
+
* `"deadline"`; the resolver's signal is aborted then, so one that honours
|
|
232
|
+
* it stops. The deadline timer never holds the process open.
|
|
233
|
+
*/
|
|
234
|
+
function sendHeads(run, urls) {
|
|
235
|
+
if (urls.length === 0)
|
|
236
|
+
return;
|
|
237
|
+
const deadline = run.opts.headDeadlineMs ?? HEAD_DEADLINE_MS;
|
|
238
|
+
const controller = new AbortController();
|
|
239
|
+
const settle = new Map();
|
|
240
|
+
for (const url of urls) {
|
|
241
|
+
run.headsUsed += 1;
|
|
242
|
+
run.heads.set(url, new Promise((resolve) => settle.set(url, resolve)));
|
|
243
|
+
}
|
|
244
|
+
const queue = [...urls];
|
|
245
|
+
let open = urls.length;
|
|
246
|
+
let active = 0;
|
|
247
|
+
let expired = false;
|
|
248
|
+
const timer = setTimeout(() => {
|
|
249
|
+
expired = true;
|
|
250
|
+
controller.abort();
|
|
251
|
+
for (const resolve of settle.values())
|
|
252
|
+
resolve("deadline");
|
|
253
|
+
}, deadline);
|
|
254
|
+
timer.unref?.();
|
|
255
|
+
const pump = () => {
|
|
256
|
+
while (!expired && active < HEAD_CONCURRENCY && queue.length > 0) {
|
|
257
|
+
const url = queue.shift();
|
|
258
|
+
active += 1;
|
|
259
|
+
void sendHead(run, url, controller.signal).then((status) => {
|
|
260
|
+
active -= 1;
|
|
261
|
+
open -= 1;
|
|
262
|
+
// A settled promise ignores a second settlement: one the deadline
|
|
263
|
+
// already decided stays "deadline".
|
|
264
|
+
settle.get(url)?.(status);
|
|
265
|
+
if (open === 0)
|
|
266
|
+
clearTimeout(timer);
|
|
267
|
+
pump();
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
pump();
|
|
272
|
+
}
|
|
273
|
+
async function headStatus(run, url) {
|
|
274
|
+
let pending = run.heads.get(url);
|
|
275
|
+
if (pending === undefined) {
|
|
276
|
+
// Every URL the verification asks about was sent up front (`sendHeads`);
|
|
277
|
+
// one that was not is past the cap.
|
|
278
|
+
if (run.headsUsed >= MAX_HEAD_REQUESTS)
|
|
279
|
+
return "budget";
|
|
280
|
+
sendHeads(run, [url]);
|
|
281
|
+
pending = run.heads.get(url);
|
|
282
|
+
}
|
|
283
|
+
return pending;
|
|
284
|
+
}
|
|
285
|
+
/** Why a URL fails the `cited` host checks (deny, allow, private address), or `undefined`. */
|
|
286
|
+
function hostRefusal(run, url) {
|
|
287
|
+
const { sources } = run.config;
|
|
288
|
+
const host = hostOf(url) ?? "";
|
|
289
|
+
if (sources.deny.length > 0 && hostMatches(host, sources.deny)) {
|
|
290
|
+
return fail("none", "host_denied", `${host} matches provability.sources.deny`, { origin: url });
|
|
291
|
+
}
|
|
292
|
+
if (sources.allow.length > 0 && !hostMatches(host, sources.allow)) {
|
|
293
|
+
return fail("none", "host_denied", `${host} is not on provability.sources.allow`, {
|
|
294
|
+
origin: url,
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
if (isPrivateHost(host)) {
|
|
298
|
+
return fail("none", "private_address", `${host} is a loopback, private or link-local address, not a public source`, { origin: url });
|
|
299
|
+
}
|
|
300
|
+
return undefined;
|
|
301
|
+
}
|
|
302
|
+
/**
|
|
303
|
+
* Does this verification send HEADs? Reachability matters only where a
|
|
304
|
+
* source can meet the floor: at `checked` and `proven` no source binding
|
|
305
|
+
* does, so no request is made.
|
|
306
|
+
*/
|
|
307
|
+
function sendsHeads(run) {
|
|
308
|
+
return (run.config.sources.resolve === "head" &&
|
|
309
|
+
levelRank(run.floor) <= levelRank("grounded") &&
|
|
310
|
+
run.opts.resolveHead !== undefined);
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* The URL a source definition names, normalized (a DOI as its resolver
|
|
314
|
+
* URL), or `undefined` for any other definition.
|
|
315
|
+
*/
|
|
316
|
+
function definitionUrl(source) {
|
|
317
|
+
const ref = classifyDefinition(source);
|
|
318
|
+
// A URL with userinfo is refused before it is checked (`sourceBinding`).
|
|
319
|
+
if (ref.kind === "url")
|
|
320
|
+
return hasUserinfo(ref.url) ? undefined : normalizeUrl(ref.url);
|
|
321
|
+
if (ref.kind === "doi")
|
|
322
|
+
return doiUrl(ref.doi);
|
|
323
|
+
return undefined;
|
|
324
|
+
}
|
|
325
|
+
/** The `cited` checks for a URL (or a DOI's resolver URL). */
|
|
326
|
+
async function citedUrl(run, url) {
|
|
327
|
+
const { sources } = run.config;
|
|
328
|
+
const refused = hostRefusal(run, url);
|
|
329
|
+
if (refused !== undefined)
|
|
330
|
+
return refused;
|
|
331
|
+
if (sources.resolve === "head" && levelRank(run.floor) <= levelRank("grounded")) {
|
|
332
|
+
if (run.opts.resolveHead === undefined) {
|
|
333
|
+
return fail("none", "unresolved", "provability.sources.resolve is head, but no reachability check is wired here", { origin: url });
|
|
334
|
+
}
|
|
335
|
+
const status = await headStatus(run, url);
|
|
336
|
+
if (status === "budget") {
|
|
337
|
+
return fail("none", "unresolved", `not checked: this answer cites more than ${MAX_HEAD_REQUESTS} distinct URLs, the most one verification sends a HEAD to`, { origin: url });
|
|
338
|
+
}
|
|
339
|
+
if (status === "deadline") {
|
|
340
|
+
const ms = run.opts.headDeadlineMs ?? HEAD_DEADLINE_MS;
|
|
341
|
+
return fail("none", "unresolved", `not checked: this verification's HEAD requests ran past ${ms >= 1000 && ms % 1000 === 0 ? `${ms / 1000} s` : `${ms} ms`}, the longest it waits on them`, { origin: url });
|
|
342
|
+
}
|
|
343
|
+
if (status === "egress") {
|
|
344
|
+
return fail("none", "unresolved", "not checked: the run's egress fabric refused to send a HEAD to this URL — it carries content a tool or another boundary handed the run (a secret it read, say), which the request would have taken out", { origin: url });
|
|
345
|
+
}
|
|
346
|
+
if (status === undefined || status < 200 || status >= 400) {
|
|
347
|
+
return fail("none", "unresolved", status === undefined
|
|
348
|
+
? `HEAD ${url} got no answer`
|
|
349
|
+
: `HEAD ${url} answered ${status}, not 2xx/3xx`, { origin: url });
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return undefined;
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Directories under the workspace that are always the harness's own state
|
|
356
|
+
* ({@link stateRootOf}); the caller names any others (`VerifyOptions.stateRoots`).
|
|
357
|
+
*/
|
|
358
|
+
const STATE_DIRS = [".crewhaus", ".crewhaus-shared"];
|
|
359
|
+
/** A directory's real path, or its resolved path when it does not resolve. */
|
|
360
|
+
function realDir(dir) {
|
|
361
|
+
try {
|
|
362
|
+
return realpathSync(dir);
|
|
363
|
+
}
|
|
364
|
+
catch {
|
|
365
|
+
return path.resolve(dir);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* The state directories one verification refuses path sources under: the
|
|
370
|
+
* workspace's {@link STATE_DIRS} and the caller's `stateRoots`, each as
|
|
371
|
+
* resolved and as its real path, so a link into one is caught too.
|
|
372
|
+
*/
|
|
373
|
+
function stateRootsOf(workspaceRoot, extra = []) {
|
|
374
|
+
const named = [
|
|
375
|
+
...STATE_DIRS.map((dir) => [`${dir}/`, path.join(workspaceRoot, dir)]),
|
|
376
|
+
...extra
|
|
377
|
+
.filter((dir) => dir.trim() !== "")
|
|
378
|
+
.map((dir) => [dir, path.resolve(workspaceRoot, dir)]),
|
|
379
|
+
];
|
|
380
|
+
const out = [];
|
|
381
|
+
const seen = new Set();
|
|
382
|
+
for (const [shown, dir] of named) {
|
|
383
|
+
for (const candidate of [dir, realDir(dir)]) {
|
|
384
|
+
if (seen.has(candidate))
|
|
385
|
+
continue;
|
|
386
|
+
seen.add(candidate);
|
|
387
|
+
out.push({ dir: candidate, shown });
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
return out;
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* The state directory a path source lies under, as the refusal names it, or
|
|
394
|
+
* `undefined` (design §1: evidence is a source the run read, never what the
|
|
395
|
+
* model said). A session log holds every answer verbatim (an
|
|
396
|
+
* `assistant_message` line), a proof bundle every claim, memory what a
|
|
397
|
+
* model stored: a `Read` of one returns the model's own earlier text, so a
|
|
398
|
+
* claim it wrote once would ground in its own log — under default
|
|
399
|
+
* permissions, with no write. The offline gate never reads a cited path
|
|
400
|
+
* under these (`crewhaus provability check`); the run's gate must not
|
|
401
|
+
* either. Read by the workspace-relative name and by the real path, so a
|
|
402
|
+
* link into one is refused too.
|
|
403
|
+
*/
|
|
404
|
+
function stateRootOf(run, rel, real) {
|
|
405
|
+
const first = rel.split("/")[0] ?? "";
|
|
406
|
+
if (STATE_DIRS.includes(first))
|
|
407
|
+
return `${first}/`;
|
|
408
|
+
// The path as the filesystem spells it: on a case-insensitive volume
|
|
409
|
+
// (macOS by default) `.CREWHAUS/sessions/…` is the session log, and the
|
|
410
|
+
// resolved path keeps the case the model typed. `realpathSync` returns
|
|
411
|
+
// the on-disk spelling, so it is compared as well as the path given.
|
|
412
|
+
const canonical = realDir(real);
|
|
413
|
+
for (const root of run.stateRoots) {
|
|
414
|
+
const under = root.dir.endsWith(path.sep) ? root.dir : `${root.dir}${path.sep}`;
|
|
415
|
+
for (const candidate of canonical === real ? [real] : [real, canonical]) {
|
|
416
|
+
if (candidate === root.dir || candidate.startsWith(under))
|
|
417
|
+
return root.shown;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
return undefined;
|
|
421
|
+
}
|
|
422
|
+
/** The `cited` checks for a workspace path: inside the workspace, links included, and there. */
|
|
423
|
+
function citedPath(run, given) {
|
|
424
|
+
const rel = normalizePath(given, { workspaceRoot: run.workspaceRoot, baseDir: run.baseDir });
|
|
425
|
+
if (rel === undefined) {
|
|
426
|
+
return fail("none", "path_outside_workspace", `${given} resolves outside the workspace`);
|
|
427
|
+
}
|
|
428
|
+
// The lexical check above cannot see a symlink inside the workspace that
|
|
429
|
+
// leads out of it; the contained resolution follows every link the way the
|
|
430
|
+
// kernel would and refuses one that leaves.
|
|
431
|
+
const located = resolveContained(run.workspaceRoot, rel);
|
|
432
|
+
if (!located.ok) {
|
|
433
|
+
return located.code === "escapes-root"
|
|
434
|
+
? fail("none", "path_outside_workspace", `${given} leads outside the workspace`, {
|
|
435
|
+
origin: rel,
|
|
436
|
+
})
|
|
437
|
+
: fail("none", "path_missing", `${given} cannot be resolved in the workspace`, {
|
|
438
|
+
origin: rel,
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
if (!existsSync(located.real)) {
|
|
442
|
+
return fail("none", "path_missing", `${rel} does not exist in the workspace`, { origin: rel });
|
|
443
|
+
}
|
|
444
|
+
return { origin: rel, real: located.real };
|
|
445
|
+
}
|
|
446
|
+
function bodyOf(run, record) {
|
|
447
|
+
let body = run.bodies.get(record);
|
|
448
|
+
if (body === undefined) {
|
|
449
|
+
const text = groundingBody(record);
|
|
450
|
+
body = { index: indexSource(text), sha256: sha256Of(text) };
|
|
451
|
+
run.bodies.set(record, body);
|
|
452
|
+
}
|
|
453
|
+
return body;
|
|
454
|
+
}
|
|
455
|
+
function callInputs(run, record) {
|
|
456
|
+
let inputs = run.inputs.get(record);
|
|
457
|
+
if (inputs === undefined) {
|
|
458
|
+
const values = originFieldValues(record, {
|
|
459
|
+
workspaceRoot: run.workspaceRoot,
|
|
460
|
+
}).filter((value) => value.trim() !== "");
|
|
461
|
+
const pending = [...values];
|
|
462
|
+
const plain = [];
|
|
463
|
+
// A builtin reader is asked only where its address is printed back:
|
|
464
|
+
// the rest of its input (WebFetch's `prompt`, which `groundingBody`
|
|
465
|
+
// already takes off) is not text the page could be reflecting, and a
|
|
466
|
+
// page that states the claim in the words the model asked about must
|
|
467
|
+
// still ground.
|
|
468
|
+
if (!isReader(record.toolName)) {
|
|
469
|
+
for (const value of inputStrings(record.input)) {
|
|
470
|
+
const at = pending.indexOf(value);
|
|
471
|
+
if (at !== -1)
|
|
472
|
+
pending.splice(at, 1);
|
|
473
|
+
else if (value.trim() !== "")
|
|
474
|
+
plain.push(indexSource(value));
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
inputs = { plain, named: values.map((value) => indexSource(value)) };
|
|
478
|
+
run.inputs.set(record, inputs);
|
|
479
|
+
}
|
|
480
|
+
return inputs;
|
|
481
|
+
}
|
|
482
|
+
/** The locate options `grounded` reads a body with (see there). */
|
|
483
|
+
function locateOptions(run) {
|
|
484
|
+
const { coverage: bar, windowTokens } = run.config.grounded;
|
|
485
|
+
return { coverage: bar, windowTokens, strictFigures: true };
|
|
486
|
+
}
|
|
487
|
+
/** The claim at its tightest ({@link tightestRuns}) in the span `location` reports of `index`. */
|
|
488
|
+
function spanRuns(run, query, index, location) {
|
|
489
|
+
let span = location.excerpt ?? "";
|
|
490
|
+
if (span.endsWith("…")) {
|
|
491
|
+
// The excerpt is cut at 240 characters; the test needs the span itself.
|
|
492
|
+
const whole = locateClaim(query, index, {
|
|
493
|
+
...locateOptions(run),
|
|
494
|
+
maxExcerptChars: MAX_ECHO_SPAN_CHARS,
|
|
495
|
+
}).excerpt;
|
|
496
|
+
if (whole !== undefined)
|
|
497
|
+
span = whole;
|
|
498
|
+
}
|
|
499
|
+
const words = tokenize(foldTypography(span)).map((token) => token.norm);
|
|
500
|
+
// A span still cut ends on a word that may be partial.
|
|
501
|
+
if (span.endsWith("…"))
|
|
502
|
+
words.pop();
|
|
503
|
+
return tightestRuns(words, new Set(query.need.map(wordKey)));
|
|
504
|
+
}
|
|
505
|
+
/** `text` as a case-insensitive pattern of itself. */
|
|
506
|
+
function caseless(text, flags = "") {
|
|
507
|
+
return new RegExp(text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), `i${flags}`);
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* What a result prints of an address the call was pointed at, where that
|
|
511
|
+
* printing can carry the claim's words (`need`): the whole address, found
|
|
512
|
+
* as written, and the piece of it that spells two or more of the claim's
|
|
513
|
+
* words at their tightest — a slug ("rust-is-faster-than-go", the path a
|
|
514
|
+
* 404 page repeats). A slug spells words as no prose does; a piece spelled
|
|
515
|
+
* with spaces (a path may have them) reads as prose too, so for one of
|
|
516
|
+
* those only the whole address counts. Compared without case: a tool may
|
|
517
|
+
* print the host as it normalized it.
|
|
518
|
+
*/
|
|
519
|
+
function printsOf(index, need, body) {
|
|
520
|
+
const shown = [index.text];
|
|
521
|
+
for (const [from, to] of tightestWindows(index.tokens.map((token) => wordKey(token.norm)), need)) {
|
|
522
|
+
if (to === from)
|
|
523
|
+
continue;
|
|
524
|
+
const first = index.tokens[from];
|
|
525
|
+
const last = index.tokens[to];
|
|
526
|
+
const piece = index.text.slice(first.at, last.at + last.raw.length);
|
|
527
|
+
if (!/\s/.test(piece) && !shown.includes(piece))
|
|
528
|
+
shown.push(piece);
|
|
529
|
+
}
|
|
530
|
+
return shown.filter((text) => text.trim() !== "" && caseless(text).test(body));
|
|
531
|
+
}
|
|
532
|
+
/** A body with every printing of `prints` blanked out, offsets kept: indexed once per run. */
|
|
533
|
+
function maskedIndex(run, index, prints) {
|
|
534
|
+
let byPrints = run.masked.get(index);
|
|
535
|
+
if (byPrints === undefined) {
|
|
536
|
+
byPrints = new Map();
|
|
537
|
+
run.masked.set(index, byPrints);
|
|
538
|
+
}
|
|
539
|
+
const key = prints.join("\u0000");
|
|
540
|
+
let masked = byPrints.get(key);
|
|
541
|
+
if (masked === undefined) {
|
|
542
|
+
let text = index.text;
|
|
543
|
+
for (const shown of prints) {
|
|
544
|
+
text = text.replace(caseless(shown, "g"), (match) => " ".repeat(match.length));
|
|
545
|
+
}
|
|
546
|
+
masked = indexSource(text);
|
|
547
|
+
byPrints.set(key, masked);
|
|
548
|
+
}
|
|
549
|
+
return masked;
|
|
550
|
+
}
|
|
551
|
+
/**
|
|
552
|
+
* Is the span of `record` that carries the claim the call's own input
|
|
553
|
+
* handed back (design §10)? `"echo"` when it is; otherwise the location to
|
|
554
|
+
* read the record by — `location` itself, or the claim located again with
|
|
555
|
+
* an address the result prints blanked out (see below).
|
|
556
|
+
*
|
|
557
|
+
* The evidence index drops a result, or a JSON hit, that is WHOLLY an echo
|
|
558
|
+
* of its input (`echoesInput`). A tool the gate cannot see into (an MCP
|
|
559
|
+
* server's, a plugin's) can still reflect the model's text padded with
|
|
560
|
+
* anything else — a SELECT that appends computed hex or a real row to a
|
|
561
|
+
* literal sentence, a linter that repeats a `<title>` inside a report —
|
|
562
|
+
* and the claim then "grounds" in words the model wrote, under an origin it
|
|
563
|
+
* named and never read. So the claim at its tightest inside the span the
|
|
564
|
+
* locate kernel found ({@link tightestRuns}) is looked for, word for word,
|
|
565
|
+
* in each of the call's input strings; found, it is the model's text, not
|
|
566
|
+
* the source's.
|
|
567
|
+
*
|
|
568
|
+
* The URL or path the call was pointed at (`originFieldValues`) is read
|
|
569
|
+
* apart. Its slug can spell the claim the page states ("…/rust-is-faster-
|
|
570
|
+
* than-go"), and a fetcher returns the page, not its address, so the
|
|
571
|
+
* address counts only where the result prints it ({@link printsOf}): a
|
|
572
|
+
* 404 page that echoes the requested path, a report headed with the URL.
|
|
573
|
+
* Then the claim is looked for again with that printing blanked out: a
|
|
574
|
+
* page that carries the claim in its own text beside its printed address
|
|
575
|
+
* still grounds there, and one that carries it only in the address is the
|
|
576
|
+
* address handed back.
|
|
577
|
+
*
|
|
578
|
+
* The builtin readers (`WebFetch`, `Fetch`, `Read`) return the content of
|
|
579
|
+
* what they were pointed at, so only their address is asked about: a page
|
|
580
|
+
* whose slug spells the claim grounds in its own text, and one that holds
|
|
581
|
+
* the claim only where it prints its address — a 404 page that repeats the
|
|
582
|
+
* path, a page headed with its URL — does not. Other builtins are not asked
|
|
583
|
+
* at all (their input names no origin), nor is Retrieve (tool-categories
|
|
584
|
+
* does not list it), which returns corpus text, never its query. A search
|
|
585
|
+
* hit whose claim-bearing run is a phrase the model searched for word for
|
|
586
|
+
* word is refused as well — the gate cannot tell that from a reflection,
|
|
587
|
+
* and fetching the page grounds it. What stays open is reflected text the
|
|
588
|
+
* tool rewrites on the way out (an SQL function applied to the literal, an
|
|
589
|
+
* address printed re-spelled): closing that needs the spec to say which
|
|
590
|
+
* tools' hits are trusted — a design decision, not made here.
|
|
591
|
+
*
|
|
592
|
+
* An MCP server's result is asked once more ({@link storedText}): a server
|
|
593
|
+
* with storage hands back what an EARLIER call wrote to it.
|
|
594
|
+
*/
|
|
595
|
+
function echoCheck(run, record, query, index, location) {
|
|
596
|
+
const kept = { location, index };
|
|
597
|
+
if (record.toolName === RETRIEVE)
|
|
598
|
+
return kept;
|
|
599
|
+
if (builtinKey(record.toolName) !== undefined && !isReader(record.toolName))
|
|
600
|
+
return kept;
|
|
601
|
+
const read = sameCallEcho(run, record, query, index, location);
|
|
602
|
+
return read === "echo" || !isMcpTool(record.toolName)
|
|
603
|
+
? read
|
|
604
|
+
: storedText(run, record, query, read);
|
|
605
|
+
}
|
|
606
|
+
/** {@link echoCheck}'s test of the call's own input (see there). */
|
|
607
|
+
function sameCallEcho(run, record, query, index, location) {
|
|
608
|
+
const kept = { location, index };
|
|
609
|
+
const { plain, named } = callInputs(run, record);
|
|
610
|
+
const need = new Set(query.need.map(wordKey));
|
|
611
|
+
// Nothing to ask when no input string was handed over and no address
|
|
612
|
+
// the call was pointed at holds a word of the claim (a reader's usual
|
|
613
|
+
// case: the page is read once, not again for its address).
|
|
614
|
+
if (plain.length === 0 &&
|
|
615
|
+
!named.some((one) => one.tokens.some((token) => need.has(wordKey(token.norm))))) {
|
|
616
|
+
return kept;
|
|
617
|
+
}
|
|
618
|
+
const handedBack = (words) => plain.some((input) => findSequence(input, words) !== -1);
|
|
619
|
+
const runs = spanRuns(run, query, index, location);
|
|
620
|
+
if (runs.length === 0)
|
|
621
|
+
return kept;
|
|
622
|
+
if (runs.every(handedBack))
|
|
623
|
+
return "echo";
|
|
624
|
+
// An address counts only where the result prints it, and only when the
|
|
625
|
+
// claim at its tightest is in the address at all.
|
|
626
|
+
const prints = named
|
|
627
|
+
.filter((one) => runs.some((words) => findSequence(one, words) !== -1))
|
|
628
|
+
.flatMap((one) => printsOf(one, need, index.text));
|
|
629
|
+
if (prints.length === 0)
|
|
630
|
+
return kept;
|
|
631
|
+
// Blank the printing out and read the result again: what carries the
|
|
632
|
+
// claim now is the page's own text, or the claim was only ever in the
|
|
633
|
+
// address handed back.
|
|
634
|
+
const masked = maskedIndex(run, index, prints);
|
|
635
|
+
const again = locateClaim(query, masked, locateOptions(run));
|
|
636
|
+
if (again.status === "polarityConflict")
|
|
637
|
+
return { location: again, index: masked };
|
|
638
|
+
if (again.status !== "supported" && !isWaivableMiss(again))
|
|
639
|
+
return "echo";
|
|
640
|
+
const rest = spanRuns(run, query, masked, again);
|
|
641
|
+
return rest.length > 0 && rest.every(handedBack) ? "echo" : { location: again, index: masked };
|
|
642
|
+
}
|
|
643
|
+
/** A call's session and id: one call, whichever view of it (a hit's included) is asked. */
|
|
644
|
+
const callOf = (record) => `${record.sessionId ?? ""}\u0000${record.toolUseId}`;
|
|
645
|
+
function sentText(run) {
|
|
646
|
+
if (run.sent === undefined) {
|
|
647
|
+
// Every call the index holds, whatever `turnNumbers` keeps: text sent in
|
|
648
|
+
// an earlier turn and read back in this one is still the run's own.
|
|
649
|
+
const calls = run.opts.evidence.all();
|
|
650
|
+
const position = new Map();
|
|
651
|
+
calls.forEach((call, i) => {
|
|
652
|
+
if (!position.has(callOf(call)))
|
|
653
|
+
position.set(callOf(call), i);
|
|
654
|
+
});
|
|
655
|
+
run.sent = { position, calls, strings: new Map() };
|
|
656
|
+
}
|
|
657
|
+
return run.sent;
|
|
658
|
+
}
|
|
659
|
+
/**
|
|
660
|
+
* One call's sent strings, indexed the first time any claim asks — and
|
|
661
|
+
* counted against the grounding budget then, once, as a body read is.
|
|
662
|
+
*/
|
|
663
|
+
function sentStrings(run, sent, at) {
|
|
664
|
+
let strings = sent.strings.get(at);
|
|
665
|
+
if (strings === undefined) {
|
|
666
|
+
const call = sent.calls[at];
|
|
667
|
+
const pending = originFieldValues(call, {
|
|
668
|
+
workspaceRoot: run.workspaceRoot,
|
|
669
|
+
});
|
|
670
|
+
const out = [];
|
|
671
|
+
for (const value of inputStrings(call.input)) {
|
|
672
|
+
const named = pending.indexOf(value);
|
|
673
|
+
if (named !== -1)
|
|
674
|
+
pending.splice(named, 1);
|
|
675
|
+
else if (value.trim() !== "") {
|
|
676
|
+
const index = indexSource(value);
|
|
677
|
+
run.groundingTokens += index.tokens.length;
|
|
678
|
+
out.push({ index, words: new Set(index.tokens.map((token) => token.norm)) });
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
strings = out;
|
|
682
|
+
sent.strings.set(at, strings);
|
|
683
|
+
}
|
|
684
|
+
return strings;
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Is the claim, in an MCP server's result, text an EARLIER call of the run
|
|
688
|
+
* sent (design §5.2: MCP bodies are evidence because they were fetched; §0
|
|
689
|
+
* A)? A server with storage — SQL, a key-value or notes store, a vector
|
|
690
|
+
* index — hands back what was written to it, and the same-call test cannot
|
|
691
|
+
* see that: `write_query` stores `('https://acme.example/policy', 'The
|
|
692
|
+
* refund window is 30 days …')`, `read_query` returns it as a `{url, text}`
|
|
693
|
+
* hit, and the model's own sentence grounds under a page nobody fetched. So
|
|
694
|
+
* the claim at its tightest in the span that carries it
|
|
695
|
+
* ({@link tightestRuns}) is looked for, word for word, in what every
|
|
696
|
+
* earlier call sent; found, every run of it, it is the model's text stored
|
|
697
|
+
* and read back — `{ storedBy }`, the call that sent it. Only earlier
|
|
698
|
+
* calls: a page the run read and then summarised in its notes is still the
|
|
699
|
+
* page. Which earlier calls depends on how the claim reached its URL:
|
|
700
|
+
*
|
|
701
|
+
* - under a URL or path the call's input named (`mcp__fetch__fetch {url}`),
|
|
702
|
+
* only text sent to the same server (`mcp__<server>__`): a store hands
|
|
703
|
+
* back what was written to IT, and text the run merely sent elsewhere —
|
|
704
|
+
* the draft given to `ProvabilityPreflight`, an exact phrase searched for,
|
|
705
|
+
* a comment posted to another server — is not what the page says.
|
|
706
|
+
* Counting those refused an honest page fetched over MCP as "never read",
|
|
707
|
+
* in the designed preflight-then-fetch flow. Text another tool WROTE to
|
|
708
|
+
* that very URL or path — a PUT to it, a `Write` of the file a file
|
|
709
|
+
* server then reads — is the page now, and the run's own: {@link writtenBy}
|
|
710
|
+
* marks the binding, as for a builtin reader;
|
|
711
|
+
* - under a hit the result carried (a JSON row's `url`), text sent by any
|
|
712
|
+
* call but the preflight, which writes nowhere: the row is whatever the
|
|
713
|
+
* server keeps, and a file `Write` saved is what a filesystem server reads.
|
|
714
|
+
*
|
|
715
|
+
* What stays open is a row stored before the evidence this verification is
|
|
716
|
+
* handed begins — by an earlier session, or by an earlier turn when the
|
|
717
|
+
* caller hands a turn-only index rather than the session's with
|
|
718
|
+
* `turnNumbers` (the gate and its preflight hand the latter): no call in it
|
|
719
|
+
* sent the text. Closing that needs the spec to say
|
|
720
|
+
* which MCP tools return sources (a design decision; the README names the
|
|
721
|
+
* residual). The words read here count against the grounding budget
|
|
722
|
+
* ({@link MAX_GROUNDING_TOKENS}); past it the record is `"unread"`, not
|
|
723
|
+
* grounded.
|
|
724
|
+
*/
|
|
725
|
+
function storedText(run, record, query, read) {
|
|
726
|
+
const sent = sentText(run);
|
|
727
|
+
const at = sent.position.get(callOf(record)) ?? 0;
|
|
728
|
+
if (at === 0)
|
|
729
|
+
return read;
|
|
730
|
+
const server = mcpServerOf(record.toolName);
|
|
731
|
+
// A hit is a view of the call (`hitView`): its own text, under an origin
|
|
732
|
+
// the result carried rather than one the input named.
|
|
733
|
+
const hit = isHitView(run, record);
|
|
734
|
+
const sentHere = (call) => hit ? call.toolName !== PROVABILITY_PREFLIGHT : mcpServerOf(call.toolName) === server;
|
|
735
|
+
const runs = spanRuns(run, query, read.index, read.location);
|
|
736
|
+
if (runs.length === 0)
|
|
737
|
+
return read;
|
|
738
|
+
let by;
|
|
739
|
+
for (const words of runs) {
|
|
740
|
+
let found = false;
|
|
741
|
+
for (let i = 0; i < at && !found; i++) {
|
|
742
|
+
if (!sentHere(sent.calls[i]))
|
|
743
|
+
continue;
|
|
744
|
+
for (const held of sentStrings(run, sent, i)) {
|
|
745
|
+
if (run.groundingTokens >= groundingBudget)
|
|
746
|
+
return "unread";
|
|
747
|
+
// A string missing one of the run's words cannot hold it: only a
|
|
748
|
+
// string that has them all is read through.
|
|
749
|
+
if (!words.every((word) => held.words.has(word)))
|
|
750
|
+
continue;
|
|
751
|
+
run.groundingTokens += held.index.tokens.length;
|
|
752
|
+
if (findSequence(held.index, words) !== -1) {
|
|
753
|
+
found = true;
|
|
754
|
+
by ??= sent.calls[i];
|
|
755
|
+
break;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
if (!found)
|
|
760
|
+
return read;
|
|
761
|
+
}
|
|
762
|
+
return by !== undefined ? { storedBy: by } : read;
|
|
763
|
+
}
|
|
764
|
+
/** How many copy or move steps {@link provenanceOf} follows back from the places a body was read from. */
|
|
765
|
+
const MAX_WRITE_HOPS = 4;
|
|
766
|
+
/** How many places one read's text may be followed back through. */
|
|
767
|
+
const MAX_WRITE_ORIGINS = 32;
|
|
768
|
+
/** The places call `at` of {@link SentText} may have written, or copied from (`writtenPlaces`), read once. */
|
|
769
|
+
function writesOf(run, sent, at) {
|
|
770
|
+
run.writes ??= new Map();
|
|
771
|
+
let named = run.writes.get(at);
|
|
772
|
+
if (named === undefined) {
|
|
773
|
+
const call = sent.calls[at];
|
|
774
|
+
// Measured as the evidence index measures a reader's path: from the
|
|
775
|
+
// workspace root, where the tools resolve one.
|
|
776
|
+
named = writtenPlaces(call.toolName, call.input, { workspaceRoot: run.workspaceRoot });
|
|
777
|
+
run.writes.set(at, named);
|
|
778
|
+
}
|
|
779
|
+
return named;
|
|
780
|
+
}
|
|
781
|
+
/** A word that carries content, for the introduced-words test: a figure, or a non-stopword of three letters or more. */
|
|
782
|
+
const carriesContent = (word) => figureKey(word) !== null || (word.length >= 3 && !STOPWORDS.has(word) && /\p{L}/u.test(word));
|
|
783
|
+
/** The words of a text as the locate kernel normalizes them. */
|
|
784
|
+
const wordsOf = (text) => tokenize(foldTypography(text)).map((token) => token.norm);
|
|
785
|
+
/** What call `at` of {@link SentText} put where it wrote, read once and counted against the grounding budget. */
|
|
786
|
+
function wroteOf(run, sent, at) {
|
|
787
|
+
run.wrote ??= new Map();
|
|
788
|
+
let wrote = run.wrote.get(at);
|
|
789
|
+
if (wrote === undefined) {
|
|
790
|
+
const call = sent.calls[at];
|
|
791
|
+
const places = new Set(writesOf(run, sent, at).map((one) => one.place));
|
|
792
|
+
const strings = [];
|
|
793
|
+
for (const value of inputStrings(call.input)) {
|
|
794
|
+
if (value.trim() === "")
|
|
795
|
+
continue;
|
|
796
|
+
const trimmed = value.trim();
|
|
797
|
+
const place = /^https?:\/\//i.test(trimmed)
|
|
798
|
+
? normalizeUrl(trimmed)
|
|
799
|
+
: normalizePath(trimmed, { workspaceRoot: run.workspaceRoot });
|
|
800
|
+
if (place !== undefined && places.has(place))
|
|
801
|
+
continue;
|
|
802
|
+
const index = indexSource(value);
|
|
803
|
+
run.groundingTokens += index.tokens.length;
|
|
804
|
+
strings.push(index);
|
|
805
|
+
}
|
|
806
|
+
const text = writtenText(call.input);
|
|
807
|
+
const added = text.added.map(wordsOf);
|
|
808
|
+
const removedWords = text.removed.map(wordsOf);
|
|
809
|
+
const removed = new Set(removedWords.flat().map(wordKey));
|
|
810
|
+
const introduced = new Set();
|
|
811
|
+
for (const words of added) {
|
|
812
|
+
run.groundingTokens += words.length;
|
|
813
|
+
for (const word of words) {
|
|
814
|
+
const key = wordKey(word);
|
|
815
|
+
if (carriesContent(word) && !removed.has(key))
|
|
816
|
+
introduced.add(key);
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
const lines = [];
|
|
820
|
+
for (const one of text.added) {
|
|
821
|
+
for (const line of one.split(/\r\n|[\n\r\u2028\u2029\u0085]/)) {
|
|
822
|
+
const words = wordsOf(line);
|
|
823
|
+
if (words.some(carriesContent))
|
|
824
|
+
lines.push(words.map(wordKey));
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
wrote = { strings, added, removed: removedWords, lines, introduced };
|
|
828
|
+
run.wrote.set(at, wrote);
|
|
829
|
+
}
|
|
830
|
+
return wrote;
|
|
831
|
+
}
|
|
832
|
+
/** `origin` relative to `place` (`""` when they are one, `/rest` when it lies beneath it), else `undefined`. */
|
|
833
|
+
function beneathOf(origin, place) {
|
|
834
|
+
if (origin === place)
|
|
835
|
+
return "";
|
|
836
|
+
return origin.startsWith(`${place}/`) ? origin.slice(place.length) : undefined;
|
|
837
|
+
}
|
|
838
|
+
/** Why a body under `origin` that a copy brought out of the harness's own state is no source (§1). */
|
|
839
|
+
function stateCopyDetail(origin, state) {
|
|
840
|
+
const brought = state.copied
|
|
841
|
+
? `${state.via.toolUseId} (${state.via.toolName}) copied ${state.place} there`
|
|
842
|
+
: `${state.via.toolUseId} (${state.via.toolName}) was pointed at ${state.place}`;
|
|
843
|
+
return `${origin} is no source: what this run read under it came from ${state.place}, under ${state.shown}, the harness's own state — ${brought}; session logs, proof bundles, spilled results and memory hold the model's own text, so nothing there, or copied from there, is a source; cite the source itself`;
|
|
844
|
+
}
|
|
845
|
+
/** Is `record` a view of one hit of a call (`hitView`): its own text, under an origin the result carried? */
|
|
846
|
+
function isHitView(run, record) {
|
|
847
|
+
const sent = sentText(run);
|
|
848
|
+
const at = sent.position.get(callOf(record));
|
|
849
|
+
if (at === undefined)
|
|
850
|
+
return false;
|
|
851
|
+
const whole = sent.calls[at];
|
|
852
|
+
return (record !== whole &&
|
|
853
|
+
(record.content !== whole.content ||
|
|
854
|
+
record.origins.some((origin) => !whole.origins.includes(origin))));
|
|
855
|
+
}
|
|
856
|
+
/**
|
|
857
|
+
* The places a body under `origin` was read from: the origin itself and,
|
|
858
|
+
* for a call an MCP server answered, every place its input pointed at,
|
|
859
|
+
* declared or not (`inputPlaces`) — a file server handed the session log's
|
|
860
|
+
* path read the session log, whatever address the body was then indexed
|
|
861
|
+
* under.
|
|
862
|
+
*/
|
|
863
|
+
function readPlaces(run, record, origin) {
|
|
864
|
+
const places = [origin];
|
|
865
|
+
if (isMcpTool(record.toolName) && !isHitView(run, record)) {
|
|
866
|
+
for (const place of inputPlaces(record.toolName, record.input, {
|
|
867
|
+
workspaceRoot: run.workspaceRoot,
|
|
868
|
+
})) {
|
|
869
|
+
if (!places.includes(place))
|
|
870
|
+
places.push(place);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
return places;
|
|
874
|
+
}
|
|
875
|
+
/**
|
|
876
|
+
* What the run did to `places` before `record` read them ({@link Provenance}),
|
|
877
|
+
* read once per read and places. The calls that may have written
|
|
878
|
+
* (`writtenPlaces`: not a reader, not the preflight, not a pure tool —
|
|
879
|
+
* `VerifyOptions.isPureTool` — and not one that errored) are read back from
|
|
880
|
+
* the read, through every copy, move, concatenation and archive that
|
|
881
|
+
* brought text to one of the places from somewhere else (a directory copied
|
|
882
|
+
* whole maps each file beneath it; at most {@link MAX_WRITE_HOPS} steps):
|
|
883
|
+
*
|
|
884
|
+
* - a place the trail reaches under a state root (`stateRootOf`) is a copy
|
|
885
|
+
* of the harness's own state — a session log copied into the workspace
|
|
886
|
+
* holds every answer verbatim, the one the gate just refused among them —
|
|
887
|
+
* and is `state`, with the call that brought it here;
|
|
888
|
+
* - a call that wrote into a place on the trail (or into a directory one
|
|
889
|
+
* lies beneath) is `into`, for the self-written test ({@link selfWriter}).
|
|
890
|
+
*/
|
|
891
|
+
function provenanceOf(run, record, places) {
|
|
892
|
+
const key = `${callOf(record)}\u0000${places.join("\u0000")}`;
|
|
893
|
+
run.provenances ??= new Map();
|
|
894
|
+
const known = run.provenances.get(key);
|
|
895
|
+
if (known !== undefined)
|
|
896
|
+
return known;
|
|
897
|
+
const sent = sentText(run);
|
|
898
|
+
const at = sent.position.get(callOf(record)) ?? 0;
|
|
899
|
+
if (run.writers === undefined) {
|
|
900
|
+
const all = [];
|
|
901
|
+
sent.calls.forEach((call, i) => {
|
|
902
|
+
if (call.isError || isPure(run, call.toolName))
|
|
903
|
+
return;
|
|
904
|
+
if (writesOf(run, sent, i).some((one) => !one.source))
|
|
905
|
+
all.push(i);
|
|
906
|
+
});
|
|
907
|
+
run.writers = all;
|
|
908
|
+
}
|
|
909
|
+
const writers = run.writers.filter((i) => i < at);
|
|
910
|
+
// Every place the text could have come here from, and the copy that brought it.
|
|
911
|
+
const trail = new Map(places.map((place) => [place, undefined]));
|
|
912
|
+
for (let hop = 0; hop < MAX_WRITE_HOPS && trail.size < MAX_WRITE_ORIGINS; hop++) {
|
|
913
|
+
const before = trail.size;
|
|
914
|
+
for (const i of writers) {
|
|
915
|
+
const named = writesOf(run, sent, i);
|
|
916
|
+
const sources = named.filter((one) => one.source);
|
|
917
|
+
if (sources.length === 0)
|
|
918
|
+
continue;
|
|
919
|
+
for (const place of [...trail.keys()]) {
|
|
920
|
+
for (const target of named) {
|
|
921
|
+
if (target.source)
|
|
922
|
+
continue;
|
|
923
|
+
const rest = beneathOf(place, target.place);
|
|
924
|
+
if (rest === undefined)
|
|
925
|
+
continue;
|
|
926
|
+
for (const from of sources) {
|
|
927
|
+
const copied = `${from.place}${rest}`;
|
|
928
|
+
if (!trail.has(copied) && trail.size < MAX_WRITE_ORIGINS)
|
|
929
|
+
trail.set(copied, i);
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
if (trail.size === before)
|
|
935
|
+
break;
|
|
936
|
+
}
|
|
937
|
+
let state;
|
|
938
|
+
for (const [place, via] of trail) {
|
|
939
|
+
if (/^https?:\/\//i.test(place))
|
|
940
|
+
continue;
|
|
941
|
+
const shown = stateRootOf(run, place, path.resolve(run.workspaceRoot, place));
|
|
942
|
+
if (shown === undefined)
|
|
943
|
+
continue;
|
|
944
|
+
// The record's own place is the citation's question (`sourceBinding`
|
|
945
|
+
// refuses it there); here, a copy of one, or a place an MCP call was
|
|
946
|
+
// pointed at beside the origin it is indexed under.
|
|
947
|
+
state =
|
|
948
|
+
via !== undefined
|
|
949
|
+
? { place, shown, via: sent.calls[via], copied: true }
|
|
950
|
+
: { place, shown, via: record, copied: false };
|
|
951
|
+
break;
|
|
952
|
+
}
|
|
953
|
+
const into = writers.filter((i) => writesOf(run, sent, i).some((one) => !one.source && [...trail.keys()].some((place) => beneathOf(place, one.place) !== undefined)));
|
|
954
|
+
const provenance = { into, ...(state !== undefined ? { state } : {}) };
|
|
955
|
+
run.provenances.set(key, provenance);
|
|
956
|
+
return provenance;
|
|
957
|
+
}
|
|
958
|
+
/** Does `words` hold `run` as one contiguous stretch? */
|
|
959
|
+
function holdsRun(words, run) {
|
|
960
|
+
if (run.length === 0 || run.length > words.length)
|
|
961
|
+
return false;
|
|
962
|
+
outer: for (let i = 0; i + run.length <= words.length; i++) {
|
|
963
|
+
for (let j = 0; j < run.length; j++)
|
|
964
|
+
if (words[i + j] !== run[j])
|
|
965
|
+
continue outer;
|
|
966
|
+
return true;
|
|
967
|
+
}
|
|
968
|
+
return false;
|
|
969
|
+
}
|
|
970
|
+
/** Where each word of a body stands, by {@link wordKey}: read once per body. */
|
|
971
|
+
const KEY_POSITIONS = new WeakMap();
|
|
972
|
+
function keyPositions(index) {
|
|
973
|
+
let positions = KEY_POSITIONS.get(index);
|
|
974
|
+
if (positions === undefined) {
|
|
975
|
+
positions = new Map();
|
|
976
|
+
index.tokens.forEach((token, at) => {
|
|
977
|
+
const key = wordKey(token.norm);
|
|
978
|
+
const list = positions.get(key);
|
|
979
|
+
if (list === undefined)
|
|
980
|
+
positions.set(key, [at]);
|
|
981
|
+
else
|
|
982
|
+
list.push(at);
|
|
983
|
+
});
|
|
984
|
+
KEY_POSITIONS.set(index, positions);
|
|
985
|
+
}
|
|
986
|
+
return positions;
|
|
987
|
+
}
|
|
988
|
+
/** How many places {@link startsOf} reports, and how many first-word hits it tries. */
|
|
989
|
+
const MAX_STARTS = 64;
|
|
990
|
+
const MAX_START_TRIES = 4_096;
|
|
991
|
+
/** The token offsets in `index` where `keys` stands whole, at most {@link MAX_STARTS}. */
|
|
992
|
+
function startsOf(index, keys) {
|
|
993
|
+
const first = keys[0];
|
|
994
|
+
if (first === undefined)
|
|
995
|
+
return [];
|
|
996
|
+
const out = [];
|
|
997
|
+
const at = keyPositions(index).get(first) ?? [];
|
|
998
|
+
for (const start of at.slice(0, MAX_START_TRIES)) {
|
|
999
|
+
if (start + keys.length > index.tokens.length)
|
|
1000
|
+
break;
|
|
1001
|
+
let whole = true;
|
|
1002
|
+
for (let j = 1; j < keys.length && whole; j++) {
|
|
1003
|
+
whole = wordKey(index.tokens[start + j].norm) === keys[j];
|
|
1004
|
+
}
|
|
1005
|
+
if (whole)
|
|
1006
|
+
out.push(start);
|
|
1007
|
+
if (out.length >= MAX_STARTS)
|
|
1008
|
+
break;
|
|
1009
|
+
}
|
|
1010
|
+
return out;
|
|
1011
|
+
}
|
|
1012
|
+
/**
|
|
1013
|
+
* Do `run` and `line` share a stretch of at least `min(|line|, 3)` words
|
|
1014
|
+
* that holds a word `introduced` has? For a line no longer in the body as
|
|
1015
|
+
* written (a later edit changed it, a read of part of the file): a word it
|
|
1016
|
+
* shares with the claim in passing ("refund" in an appended "For any refund
|
|
1017
|
+
* question, email support.") is not where it put the claim.
|
|
1018
|
+
*/
|
|
1019
|
+
function sharesStretch(run, line, introduced) {
|
|
1020
|
+
const need = Math.min(line.length, 3);
|
|
1021
|
+
for (let i = 0; i < run.length; i++) {
|
|
1022
|
+
for (let j = 0; j < line.length; j++) {
|
|
1023
|
+
let k = 0;
|
|
1024
|
+
let added = false;
|
|
1025
|
+
while (i + k < run.length && j + k < line.length && run[i + k] === line[j + k]) {
|
|
1026
|
+
if (introduced.has(run[i + k]))
|
|
1027
|
+
added = true;
|
|
1028
|
+
k++;
|
|
1029
|
+
}
|
|
1030
|
+
if (k >= need && added)
|
|
1031
|
+
return true;
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
return false;
|
|
1035
|
+
}
|
|
1036
|
+
/**
|
|
1037
|
+
* Which calls of `into` INTRODUCED a word of the claim's run `keys` where
|
|
1038
|
+
* the run stands in `index` (the body it grounded in). Located, not
|
|
1039
|
+
* counted: a place the run stands in the body is a call's when it overlaps
|
|
1040
|
+
* a line the call added there with one of its introduced words inside the
|
|
1041
|
+
* overlap — so an `Edit` of `14` to `30` in the sentence is the run's, and
|
|
1042
|
+
* an unrelated line it appended that shares a topic word with the claim
|
|
1043
|
+
* ("For any refund question, email support." under "The refund window is
|
|
1044
|
+
* 14 days…") is not. A line not in the body as written counts by
|
|
1045
|
+
* {@link sharesStretch}; so does every line where the run itself is not
|
|
1046
|
+
* found in `index`. `every`: the calls that put it at every place it
|
|
1047
|
+
* stands; `union`: the first call of those that, together, did — or
|
|
1048
|
+
* `undefined` when a place it stands is text no call wrote.
|
|
1049
|
+
*/
|
|
1050
|
+
function introducedBy(run, sent, into, index, keys) {
|
|
1051
|
+
const places = startsOf(index, keys);
|
|
1052
|
+
const covers = (i, place) => {
|
|
1053
|
+
const wrote = wroteOf(run, sent, i);
|
|
1054
|
+
// Only a line holding a word of the run the call introduced can overlap it with one.
|
|
1055
|
+
const wanted = new Set(keys.filter((key) => wrote.introduced.has(key)));
|
|
1056
|
+
if (wanted.size === 0)
|
|
1057
|
+
return false;
|
|
1058
|
+
for (const line of wrote.lines) {
|
|
1059
|
+
if (!line.some((key) => wanted.has(key)))
|
|
1060
|
+
continue;
|
|
1061
|
+
const at = place === undefined ? [] : startsOf(index, line);
|
|
1062
|
+
if (at.length === 0) {
|
|
1063
|
+
if (sharesStretch(keys, line, wrote.introduced))
|
|
1064
|
+
return true;
|
|
1065
|
+
continue;
|
|
1066
|
+
}
|
|
1067
|
+
for (const start of at) {
|
|
1068
|
+
const from = Math.max(place, start);
|
|
1069
|
+
const to = Math.min(place + keys.length, start + line.length);
|
|
1070
|
+
for (let k = from; k < to; k++) {
|
|
1071
|
+
if (wanted.has(wordKey(index.tokens[k].norm)))
|
|
1072
|
+
return true;
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
return false;
|
|
1077
|
+
};
|
|
1078
|
+
const every = new Set(into);
|
|
1079
|
+
let union;
|
|
1080
|
+
for (const place of places.length > 0 ? places : [undefined]) {
|
|
1081
|
+
const by = into.filter((i) => covers(i, place));
|
|
1082
|
+
for (const i of [...every])
|
|
1083
|
+
if (!by.includes(i))
|
|
1084
|
+
every.delete(i);
|
|
1085
|
+
if (by.length === 0)
|
|
1086
|
+
return { every, union: undefined };
|
|
1087
|
+
union = Math.min(union ?? Number.POSITIVE_INFINITY, ...by);
|
|
1088
|
+
}
|
|
1089
|
+
return { every, union };
|
|
1090
|
+
}
|
|
1091
|
+
/**
|
|
1092
|
+
* The first call of `into` that wrote the claim's words where they were
|
|
1093
|
+
* read, or `undefined` (design §10): each tightest run of the claim
|
|
1094
|
+
* (`runs`, found in `index`, the body that grounded it) must be text one of
|
|
1095
|
+
* them put there — by one of three tests, so that the commonest way a run
|
|
1096
|
+
* changes a fact in a file, an `Edit` of part of a sentence, is the run's
|
|
1097
|
+
* own:
|
|
1098
|
+
*
|
|
1099
|
+
* - one string the call sent holds the run whole (a `Write` of the
|
|
1100
|
+
* sentence, a PUT of the page), and no text it replaced did (an `Edit`
|
|
1101
|
+
* whose `oldString` already said it kept the sentence, not wrote it);
|
|
1102
|
+
* - where the run stands in the body, it overlaps a line the call added
|
|
1103
|
+
* with a word the call INTRODUCED ({@link Wrote}, {@link introducedBy}):
|
|
1104
|
+
* an `Edit` of `14` to `30`, a placeholder later filled in, a sentence
|
|
1105
|
+
* finished by an `AppendFile`, a diff's `+` line;
|
|
1106
|
+
* - the run holds, whole, a text the call put there that carries content:
|
|
1107
|
+
* an `Edit` that only took a word out ("is not refundable" → "is
|
|
1108
|
+
* refundable").
|
|
1109
|
+
*
|
|
1110
|
+
* A claim that also stands, as tightly, in text no call put there (a tie
|
|
1111
|
+
* the run did not write) is not the run's. Past the grounding budget,
|
|
1112
|
+
* nothing is.
|
|
1113
|
+
*/
|
|
1114
|
+
function selfWriter(run, into, runs, index) {
|
|
1115
|
+
if (into.length === 0 || runs.length === 0)
|
|
1116
|
+
return undefined;
|
|
1117
|
+
const sent = sentText(run);
|
|
1118
|
+
let by;
|
|
1119
|
+
for (const words of runs) {
|
|
1120
|
+
const keys = words.map(wordKey);
|
|
1121
|
+
const introduced = introducedBy(run, sent, into, index, keys);
|
|
1122
|
+
let found;
|
|
1123
|
+
for (const i of into) {
|
|
1124
|
+
if (run.groundingTokens >= groundingBudget)
|
|
1125
|
+
return undefined;
|
|
1126
|
+
const wrote = wroteOf(run, sent, i);
|
|
1127
|
+
const put = (wrote.strings.some((held) => findSequence(held, words) !== -1) &&
|
|
1128
|
+
!wrote.removed.some((gone) => holdsRun(gone, words))) ||
|
|
1129
|
+
introduced.every.has(i) ||
|
|
1130
|
+
wrote.added.some((added) => added.some(carriesContent) && holdsRun(words, added));
|
|
1131
|
+
if (put) {
|
|
1132
|
+
found = i;
|
|
1133
|
+
break;
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
found ??= introduced.union;
|
|
1137
|
+
if (run.groundingTokens >= groundingBudget)
|
|
1138
|
+
return undefined;
|
|
1139
|
+
if (found === undefined)
|
|
1140
|
+
return undefined;
|
|
1141
|
+
by = by === undefined ? found : Math.min(by, found);
|
|
1142
|
+
}
|
|
1143
|
+
return by !== undefined ? sent.calls[by] : undefined;
|
|
1144
|
+
}
|
|
1145
|
+
/**
|
|
1146
|
+
* Did the run write, itself, the text that grounds a claim in a body read
|
|
1147
|
+
* from `origin` (design §10's residual: "a model that fetches a page it
|
|
1148
|
+
* wrote itself")? The call that did, or `undefined`.
|
|
1149
|
+
*
|
|
1150
|
+
* A `Read` returns the file as it is when read, so a claim the model put
|
|
1151
|
+
* there with `Write` or `Edit` grounds like any other — and should: an
|
|
1152
|
+
* agent that sets `timeout: 30` in a config and then reports it reports the
|
|
1153
|
+
* truth. But the run is then its own source, and the bundle says so rather
|
|
1154
|
+
* than letting the ✓ read as an outside source's (`ProofBinding.writtenBy`;
|
|
1155
|
+
* the footer counts it, `provability audit` lists it). Decided per origin,
|
|
1156
|
+
* not per call: the calls that wrote into the places the body was read from
|
|
1157
|
+
* before it was read ({@link provenanceOf}) are asked whether the claim at
|
|
1158
|
+
* its tightest in the span that grounded it ({@link spanRuns}) is text they
|
|
1159
|
+
* put there ({@link selfWriter}). A read made before any of them that
|
|
1160
|
+
* grounds the claim does so unmarked (`grounded` reads the origin's records
|
|
1161
|
+
* in call order and stops at the first unmarked one).
|
|
1162
|
+
*
|
|
1163
|
+
* Asked of the builtin readers and of an MCP call whose own input named the
|
|
1164
|
+
* origin (`mcp__fetch__fetch {url}` after a PUT to that URL, a file server's
|
|
1165
|
+
* `read_file` of a file `Write` saved); a hit an MCP result carried is
|
|
1166
|
+
* {@link storedText}'s (refused there, not marked), and Retrieve returns the
|
|
1167
|
+
* operator's corpus. What stays open: text written by `Bash` or
|
|
1168
|
+
* `RunCommand` (a command line names no field this reads), by a call that
|
|
1169
|
+
* names the place in a form this cannot map to the cited one (a repository
|
|
1170
|
+
* path behind a raw-file URL), or by an earlier session; and past the
|
|
1171
|
+
* grounding budget nothing is marked.
|
|
1172
|
+
*/
|
|
1173
|
+
function writtenBy(run, record, provenance, query, index, location) {
|
|
1174
|
+
if (provenance.into.length === 0)
|
|
1175
|
+
return undefined;
|
|
1176
|
+
if (!isReader(record.toolName) && !(isMcpTool(record.toolName) && !isHitView(run, record))) {
|
|
1177
|
+
return undefined;
|
|
1178
|
+
}
|
|
1179
|
+
return selfWriter(run, provenance.into, spanRuns(run, query, index, location), index);
|
|
1180
|
+
}
|
|
1181
|
+
/** The server an MCP tool is served by (`mcp__<server>__<tool>`), lowercased; `undefined` for any other tool. */
|
|
1182
|
+
function mcpServerOf(name) {
|
|
1183
|
+
return /^mcp__(.+?)__/i.exec(name)?.[1]?.toLowerCase();
|
|
1184
|
+
}
|
|
1185
|
+
/** A word as the figure rule compares it: a figure by its value (`thirty` is `30`). */
|
|
1186
|
+
const wordKey = (word) => figureKey(word) ?? word;
|
|
1187
|
+
/**
|
|
1188
|
+
* The claim at its tightest inside the span the locate kernel reported:
|
|
1189
|
+
* every shortest run of `words` holding each of the claim's words (`need`,
|
|
1190
|
+
* by {@link wordKey}) that `words` holds at all (claim-ledger's
|
|
1191
|
+
* `tightestWindows`). The reported span runs from the first claim word in
|
|
1192
|
+
* the winning window to the last, so it can open on a stray copy of one
|
|
1193
|
+
* word (the "flu" of a URL a report prints before echoing a `<title>`) and
|
|
1194
|
+
* no longer be the reflection it mostly is; the tightest run is the
|
|
1195
|
+
* reflection itself. Every tie is returned, so a claim that also stands, as
|
|
1196
|
+
* tightly, in text the tool did not reflect is not refused for the copy it
|
|
1197
|
+
* did.
|
|
1198
|
+
*/
|
|
1199
|
+
function tightestRuns(words, need) {
|
|
1200
|
+
return tightestWindows(words.map(wordKey), need).map(([from, to]) => words.slice(from, to + 1));
|
|
1201
|
+
}
|
|
1202
|
+
/**
|
|
1203
|
+
* The excerpt a grounded binding freezes (design §5.3): the claim at its
|
|
1204
|
+
* tightest ({@link tightestWindows}) in the span the locate kernel reported,
|
|
1205
|
+
* with what is around it — the span itself when it fits in
|
|
1206
|
+
* {@link MAX_EXCERPT_CHARS}. The kernel's span runs from the first claim
|
|
1207
|
+
* word in the winning window to the last and is cut from its start, so a
|
|
1208
|
+
* span that opens on a stray early copy of one word ("Refund policy" in a
|
|
1209
|
+
* heading, the figure 200 tokens on) froze the heading and lost the
|
|
1210
|
+
* figure. Where even the tightest run is too long, the excerpt is centred
|
|
1211
|
+
* on the claim's first figure in it, which is what a reader checks first.
|
|
1212
|
+
*/
|
|
1213
|
+
function groundedExcerpt(run, query, index, location) {
|
|
1214
|
+
let span = location.excerpt;
|
|
1215
|
+
if (span === undefined)
|
|
1216
|
+
return undefined;
|
|
1217
|
+
if (!span.endsWith("…") && span.length <= MAX_EXCERPT_CHARS)
|
|
1218
|
+
return span;
|
|
1219
|
+
if (span.endsWith("…")) {
|
|
1220
|
+
span =
|
|
1221
|
+
locateClaim(query, index, { ...locateOptions(run), maxExcerptChars: MAX_ECHO_SPAN_CHARS })
|
|
1222
|
+
.excerpt ?? span;
|
|
1223
|
+
}
|
|
1224
|
+
const words = indexSource(span);
|
|
1225
|
+
const tokens = words.tokens;
|
|
1226
|
+
const window = tightestWindows(tokens.map((token) => wordKey(token.norm)), new Set(query.need.map(wordKey)))[0];
|
|
1227
|
+
if (window === undefined)
|
|
1228
|
+
return clip(span, MAX_EXCERPT_CHARS);
|
|
1229
|
+
const [first, last] = window;
|
|
1230
|
+
let from = tokens[first].at;
|
|
1231
|
+
let to = tokens[last].at + tokens[last].raw.length;
|
|
1232
|
+
if (to - from > MAX_EXCERPT_CHARS) {
|
|
1233
|
+
const figures = new Set(query.figures.map(wordKey));
|
|
1234
|
+
for (let i = first; i <= last; i++) {
|
|
1235
|
+
const token = tokens[i];
|
|
1236
|
+
if (!figures.has(wordKey(token.norm)))
|
|
1237
|
+
continue;
|
|
1238
|
+
from = token.at;
|
|
1239
|
+
to = token.at + token.raw.length;
|
|
1240
|
+
break;
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
// Cut from the span as written, not its folded twin the tokens index: an
|
|
1244
|
+
// excerpt of "… هي ٣٠ يوما" says ٣٠ (design §5.3).
|
|
1245
|
+
const cut = writtenSpan(words, from, to);
|
|
1246
|
+
return clipAround(cut.text, cut.from, cut.to, MAX_EXCERPT_CHARS);
|
|
1247
|
+
}
|
|
1248
|
+
/** The URL `record`'s body was served from (`servedFrom`), read once per verification. */
|
|
1249
|
+
function servedOf(run, record) {
|
|
1250
|
+
if (!run.served.has(record))
|
|
1251
|
+
run.served.set(record, servedFrom(record));
|
|
1252
|
+
return run.served.get(record);
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* Why a body found under `origin` may not stand for it: the fetch was
|
|
1256
|
+
* redirected (`servedFrom`), and the host that SERVED the body fails the
|
|
1257
|
+
* checks a redirect hop is held to — `sources.deny` and the private-address
|
|
1258
|
+
* rule, as `resolve: head` holds every hop (evaluation-service `hopPolicy`).
|
|
1259
|
+
* Not `sources.allow`: that names what may be cited, so a `doi.org`
|
|
1260
|
+
* citation still grounds in the publisher it redirects to (design §3, §10:
|
|
1261
|
+
* the host lists are a security allowlist). `undefined` when it may.
|
|
1262
|
+
*/
|
|
1263
|
+
function servedRefusal(run, origin, record) {
|
|
1264
|
+
const served = servedOf(run, record);
|
|
1265
|
+
if (served === undefined || served === origin)
|
|
1266
|
+
return undefined;
|
|
1267
|
+
const host = hostOf(served) ?? "";
|
|
1268
|
+
const { deny } = run.config.sources;
|
|
1269
|
+
if (deny.length > 0 && hostMatches(host, deny)) {
|
|
1270
|
+
return {
|
|
1271
|
+
reason: "host_denied",
|
|
1272
|
+
detail: `${origin} redirected to ${host} when ${record.toolUseId} fetched it, and ${host} matches provability.sources.deny — the body is that host's`,
|
|
1273
|
+
};
|
|
1274
|
+
}
|
|
1275
|
+
if (isPrivateHost(host)) {
|
|
1276
|
+
return {
|
|
1277
|
+
reason: "private_address",
|
|
1278
|
+
detail: `${origin} redirected to ${host} when ${record.toolUseId} fetched it — a loopback, private or link-local address, not a public source`,
|
|
1279
|
+
};
|
|
1280
|
+
}
|
|
1281
|
+
return undefined;
|
|
1282
|
+
}
|
|
1283
|
+
/** A grounded binding's place in {@link Run.bodyKeys}: the body it read, under the origin it cited. */
|
|
1284
|
+
const groundedAs = (sha256, origin) => `${sha256}\u0000${origin}`;
|
|
1285
|
+
/**
|
|
1286
|
+
* What `grounded.independent` counts a body found under `origin` as (design
|
|
1287
|
+
* §6.2: n distinct origins): the {@link independenceKey} of the URL that
|
|
1288
|
+
* served it, or of `origin` itself when nothing redirected — kept by the
|
|
1289
|
+
* body's digest and `origin`, which every binding that grounds there
|
|
1290
|
+
* carries. A page cited once directly and once through a shortener that
|
|
1291
|
+
* redirects to it, or a DOI and its publisher's URL, is one origin, not
|
|
1292
|
+
* two: both bodies were served by one host.
|
|
1293
|
+
*/
|
|
1294
|
+
function bodyKey(run, record, origin, sha256) {
|
|
1295
|
+
const at = groundedAs(sha256, origin);
|
|
1296
|
+
let key = run.bodyKeys.get(at);
|
|
1297
|
+
if (key === undefined) {
|
|
1298
|
+
key = independenceKey(servedOf(run, record) ?? origin);
|
|
1299
|
+
run.bodyKeys.set(at, key);
|
|
1300
|
+
}
|
|
1301
|
+
return key;
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* The `grounded` check: the origin was read in this run, and the claim's
|
|
1305
|
+
* words are in it. A body is a record that is not an error and whose
|
|
1306
|
+
* fetch answered 2xx (`servesPage`): an error page carries the words of
|
|
1307
|
+
* whatever path it repeats. A body served from a host the hop rule
|
|
1308
|
+
* refuses is not one either ({@link servedRefusal}).
|
|
1309
|
+
*/
|
|
1310
|
+
async function grounded(run, claim, origin) {
|
|
1311
|
+
const records = run.evidence.bySource(origin);
|
|
1312
|
+
const served = records.filter((record) => !record.isError && servesPage(record.status));
|
|
1313
|
+
let redirected;
|
|
1314
|
+
const usable = served.filter((record) => {
|
|
1315
|
+
const refused = servedRefusal(run, origin, record);
|
|
1316
|
+
redirected ??= refused;
|
|
1317
|
+
return refused === undefined;
|
|
1318
|
+
});
|
|
1319
|
+
if (usable.length === 0 && redirected !== undefined) {
|
|
1320
|
+
return { reached: fail("none", redirected.reason, redirected.detail, { origin }) };
|
|
1321
|
+
}
|
|
1322
|
+
if (usable.length === 0) {
|
|
1323
|
+
const statuses = [
|
|
1324
|
+
...new Set(records
|
|
1325
|
+
.filter((record) => !record.isError && !servesPage(record.status))
|
|
1326
|
+
.map((record) => record.status)),
|
|
1327
|
+
];
|
|
1328
|
+
return {
|
|
1329
|
+
reached: fail("cited", "not_fetched", statuses.length > 0
|
|
1330
|
+
? `${origin} answered ${statuses.join(", ")} when fetched in this run: that body is the server's error page, not the page`
|
|
1331
|
+
: records.length > 0
|
|
1332
|
+
? `every read of ${origin} in this run returned an error`
|
|
1333
|
+
: `${origin} was never fetched or read in this run`, { origin }),
|
|
1334
|
+
};
|
|
1335
|
+
}
|
|
1336
|
+
const { windowTokens } = run.config.grounded;
|
|
1337
|
+
// The figure rule on at every bar (claim-ledger `strictFigures`): it is
|
|
1338
|
+
// on by default only below full coverage, so a spec that tightened
|
|
1339
|
+
// `coverage` to 1 would otherwise switch it off — and with it the check
|
|
1340
|
+
// that a quotation's framing figures agree with the source ("the window
|
|
1341
|
+
// is 30 days, since \"customers may request a refund\"" against a source
|
|
1342
|
+
// saying 14), and figures compared as values (`thirty` and `30`).
|
|
1343
|
+
const options = locateOptions(run);
|
|
1344
|
+
const query = claimQuery(claim.text);
|
|
1345
|
+
let polarity;
|
|
1346
|
+
let waivable;
|
|
1347
|
+
let miss;
|
|
1348
|
+
let echoed;
|
|
1349
|
+
let stored;
|
|
1350
|
+
let addressOnly;
|
|
1351
|
+
// A body that grounds the claim in text the run wrote there itself: kept
|
|
1352
|
+
// while a read that grounds it otherwise is looked for ({@link writtenBy}).
|
|
1353
|
+
let selfGrounded;
|
|
1354
|
+
// A body read from a copy of the harness's own state ({@link provenanceOf}).
|
|
1355
|
+
let laundered;
|
|
1356
|
+
let unread = false;
|
|
1357
|
+
for (const record of usable) {
|
|
1358
|
+
if (run.groundingTokens >= groundingBudget) {
|
|
1359
|
+
unread = true;
|
|
1360
|
+
break;
|
|
1361
|
+
}
|
|
1362
|
+
// A copy of a session log is the session log: what the run copied out
|
|
1363
|
+
// of a state root is the model's own text wherever it now lies (§1).
|
|
1364
|
+
const provenance = provenanceOf(run, record, readPlaces(run, record, origin));
|
|
1365
|
+
if (provenance.state !== undefined) {
|
|
1366
|
+
laundered ??= { record, state: provenance.state };
|
|
1367
|
+
continue;
|
|
1368
|
+
}
|
|
1369
|
+
const body = bodyOf(run, record);
|
|
1370
|
+
run.groundingTokens += body.index.tokens.length;
|
|
1371
|
+
let location = locateClaim(query, body.index, options);
|
|
1372
|
+
let index = body.index;
|
|
1373
|
+
if (location.status === "supported" || isWaivableMiss(location)) {
|
|
1374
|
+
// Before it grounds — or reaches the judge, who would be asked whether
|
|
1375
|
+
// the model's own words support it — is it the call's input handed back?
|
|
1376
|
+
const read = echoCheck(run, record, query, body.index, location);
|
|
1377
|
+
if (read === "unread") {
|
|
1378
|
+
unread = true;
|
|
1379
|
+
break;
|
|
1380
|
+
}
|
|
1381
|
+
if (read === "echo") {
|
|
1382
|
+
// A reader's echo is its own address printed back: the page was
|
|
1383
|
+
// read, and its own text does not carry the claim.
|
|
1384
|
+
if (isReader(record.toolName))
|
|
1385
|
+
addressOnly ??= record;
|
|
1386
|
+
else
|
|
1387
|
+
echoed ??= record;
|
|
1388
|
+
continue;
|
|
1389
|
+
}
|
|
1390
|
+
if ("storedBy" in read) {
|
|
1391
|
+
stored ??= { record, by: read.storedBy };
|
|
1392
|
+
continue;
|
|
1393
|
+
}
|
|
1394
|
+
location = read.location;
|
|
1395
|
+
index = read.index;
|
|
1396
|
+
}
|
|
1397
|
+
if (location.status === "supported") {
|
|
1398
|
+
const writer = writtenBy(run, record, provenance, query, index, location);
|
|
1399
|
+
if (writer !== undefined && selfGrounded !== undefined)
|
|
1400
|
+
continue;
|
|
1401
|
+
const excerpt = groundedExcerpt(run, query, index, location);
|
|
1402
|
+
bodyKey(run, record, origin, body.sha256);
|
|
1403
|
+
const reached = {
|
|
1404
|
+
rung: "grounded",
|
|
1405
|
+
origin,
|
|
1406
|
+
sha256: body.sha256,
|
|
1407
|
+
...(excerpt !== undefined ? { excerpt } : {}),
|
|
1408
|
+
...(writer !== undefined
|
|
1409
|
+
? { writtenBy: { toolUseId: writer.toolUseId, toolName: writer.toolName } }
|
|
1410
|
+
: {}),
|
|
1411
|
+
};
|
|
1412
|
+
if (writer === undefined)
|
|
1413
|
+
return { reached };
|
|
1414
|
+
selfGrounded = { reached };
|
|
1415
|
+
continue;
|
|
1416
|
+
}
|
|
1417
|
+
if (location.status === "polarityConflict")
|
|
1418
|
+
polarity ??= { location, sha256: body.sha256 };
|
|
1419
|
+
else if (isWaivableMiss(location)) {
|
|
1420
|
+
if (waivable === undefined ||
|
|
1421
|
+
location.matched / location.total > waivable.location.matched / waivable.location.total) {
|
|
1422
|
+
waivable = {
|
|
1423
|
+
location,
|
|
1424
|
+
index,
|
|
1425
|
+
sha256: body.sha256,
|
|
1426
|
+
key: bodyKey(run, record, origin, body.sha256),
|
|
1427
|
+
};
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
else
|
|
1431
|
+
miss ??= { location, sha256: body.sha256 };
|
|
1432
|
+
}
|
|
1433
|
+
// Grounded, only in text the run wrote there itself: it still grounds,
|
|
1434
|
+
// and the binding says whose text it is.
|
|
1435
|
+
if (selfGrounded !== undefined)
|
|
1436
|
+
return selfGrounded;
|
|
1437
|
+
if (polarity !== undefined) {
|
|
1438
|
+
return {
|
|
1439
|
+
reached: fail("cited", "polarity_conflict", `${origin} carries the claim's words with a negation the claim lacks`, { origin, sha256: polarity.sha256 }),
|
|
1440
|
+
};
|
|
1441
|
+
}
|
|
1442
|
+
if (unread) {
|
|
1443
|
+
return {
|
|
1444
|
+
reached: fail("cited", "below_floor", `not checked: grounding this answer would read more than ${groundingBudget.toLocaleString("en-US")} tokens of fetched text, the most one verification reads — cite fewer or shorter sources, or split the answer`, { origin }),
|
|
1445
|
+
};
|
|
1446
|
+
}
|
|
1447
|
+
if (laundered !== undefined &&
|
|
1448
|
+
waivable === undefined &&
|
|
1449
|
+
miss === undefined &&
|
|
1450
|
+
echoed === undefined &&
|
|
1451
|
+
stored === undefined &&
|
|
1452
|
+
addressOnly === undefined) {
|
|
1453
|
+
// Everything under this origin is a copy of the harness's own state.
|
|
1454
|
+
return {
|
|
1455
|
+
reached: fail("cited", "not_fetched", stateCopyDetail(origin, laundered.state), { origin }),
|
|
1456
|
+
};
|
|
1457
|
+
}
|
|
1458
|
+
if ((echoed !== undefined || stored !== undefined || addressOnly !== undefined) &&
|
|
1459
|
+
waivable === undefined &&
|
|
1460
|
+
miss === undefined) {
|
|
1461
|
+
if (echoed === undefined && stored !== undefined) {
|
|
1462
|
+
// Everything under this origin that carries the claim is text the run
|
|
1463
|
+
// itself sent earlier, handed back by a store.
|
|
1464
|
+
return {
|
|
1465
|
+
reached: fail("cited", "not_fetched", `${origin} was never read in this run: the only text under it that carries this claim is what ${stored.by.toolUseId} (${stored.by.toolName}) sent earlier, handed back by ${stored.record.toolUseId} (${stored.record.toolName}) — a store returns what was written to it; fetch the page itself`, { origin }),
|
|
1466
|
+
};
|
|
1467
|
+
}
|
|
1468
|
+
if (echoed === undefined && addressOnly !== undefined) {
|
|
1469
|
+
// The page was read; the claim is only in the address it prints.
|
|
1470
|
+
return {
|
|
1471
|
+
reached: fail("cited", "not_found_in_source", `the claim's words are in ${origin} only where it prints the address ${addressOnly.toolUseId} (${addressOnly.toolName}) fetched, not in the page's own text`, { origin }),
|
|
1472
|
+
};
|
|
1473
|
+
}
|
|
1474
|
+
const handed = echoed;
|
|
1475
|
+
// Everything under this origin that carries the claim is text the run
|
|
1476
|
+
// sent itself: the origin was named, never read.
|
|
1477
|
+
return {
|
|
1478
|
+
reached: fail("cited", "not_fetched", `${origin} was never read in this run: the only text under it that carries this claim is the input ${handed.toolUseId} (${handed.toolName}) was given, handed back — fetch the page itself`, { origin }),
|
|
1479
|
+
};
|
|
1480
|
+
}
|
|
1481
|
+
const reported = waivable ?? miss;
|
|
1482
|
+
const location = reported?.location;
|
|
1483
|
+
const missing = location?.missing ?? [];
|
|
1484
|
+
const reached = fail("cited", "not_found_in_source", location?.status === "misquoted"
|
|
1485
|
+
? `${origin} does not carry the quotation word for word`
|
|
1486
|
+
: `the claim's words are not together in one ${windowTokens}-token window of ${origin}${missing.length > 0 ? ` (missing: ${missing.slice(0, 8).join(", ")})` : ""}`, { origin, ...(reported !== undefined ? { sha256: reported.sha256 } : {}) });
|
|
1487
|
+
// The waiver judge: ONLY for a paraphrase miss on coverage alone (design
|
|
1488
|
+
// §6.2), only where `grounded` is the floor — at `cited` the binding meets
|
|
1489
|
+
// the floor without it, and at `checked`/`proven` no source binding can —
|
|
1490
|
+
// and never for a missing figure or negation (`isWaivableMiss`). It is
|
|
1491
|
+
// asked later, and only while the claim still needs it (`waive`).
|
|
1492
|
+
if (waivable !== undefined && run.floor === "grounded" && run.opts.judgeWaiver !== undefined) {
|
|
1493
|
+
return {
|
|
1494
|
+
reached,
|
|
1495
|
+
waivable: {
|
|
1496
|
+
claim,
|
|
1497
|
+
origin,
|
|
1498
|
+
span: waivable.location.excerpt ?? "",
|
|
1499
|
+
excerpt: groundedExcerpt(run, query, waivable.index, waivable.location),
|
|
1500
|
+
sha256: waivable.sha256,
|
|
1501
|
+
key: waivable.key,
|
|
1502
|
+
},
|
|
1503
|
+
};
|
|
1504
|
+
}
|
|
1505
|
+
return { reached };
|
|
1506
|
+
}
|
|
1507
|
+
/** Ask the waiver judge about one paraphrase miss; the waived state, or `undefined` for a no. */
|
|
1508
|
+
async function askJudge(run, waivable) {
|
|
1509
|
+
const judge = run.opts.judgeWaiver;
|
|
1510
|
+
if (judge === undefined)
|
|
1511
|
+
return undefined;
|
|
1512
|
+
try {
|
|
1513
|
+
run.judge.calls += 1;
|
|
1514
|
+
const verdict = await judge({
|
|
1515
|
+
claim: waivable.claim.text,
|
|
1516
|
+
excerpt: waivable.span,
|
|
1517
|
+
source: waivable.origin,
|
|
1518
|
+
});
|
|
1519
|
+
if (typeof verdict.model === "string")
|
|
1520
|
+
run.judge.model ??= verdict.model;
|
|
1521
|
+
// The spend is the judge's report: a NaN or a negative would poison
|
|
1522
|
+
// the bundle's total (and serialize as null), so only a real cost counts.
|
|
1523
|
+
const cost = verdict.costUsdMicros;
|
|
1524
|
+
if (typeof cost === "number" && Number.isFinite(cost) && cost >= 0) {
|
|
1525
|
+
run.judge.costUsdMicros += cost;
|
|
1526
|
+
}
|
|
1527
|
+
if (verdict.supported !== true)
|
|
1528
|
+
return undefined;
|
|
1529
|
+
return {
|
|
1530
|
+
rung: "grounded",
|
|
1531
|
+
origin: waivable.origin,
|
|
1532
|
+
sha256: waivable.sha256,
|
|
1533
|
+
...(waivable.excerpt !== undefined && waivable.excerpt !== ""
|
|
1534
|
+
? { excerpt: waivable.excerpt }
|
|
1535
|
+
: {}),
|
|
1536
|
+
judgeWaived: true,
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
catch {
|
|
1540
|
+
// A judge that fails waives nothing; the binding keeps its miss.
|
|
1541
|
+
return undefined;
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
async function sourceBinding(run, claim, ref) {
|
|
1545
|
+
if (ref.kind === "text") {
|
|
1546
|
+
if (!run.config.sources.bibliographic) {
|
|
1547
|
+
return {
|
|
1548
|
+
reached: fail("none", "not_locatable", `"${clip(ref.text, 80)}" is not a URL, a doi: or a workspace path`),
|
|
1549
|
+
};
|
|
1550
|
+
}
|
|
1551
|
+
return {
|
|
1552
|
+
reached: fail("cited", "not_locatable", "a bibliographic reference names no body this run read — cite its URL, DOI or workspace path to ground the claim"),
|
|
1553
|
+
};
|
|
1554
|
+
}
|
|
1555
|
+
let origin;
|
|
1556
|
+
if (ref.kind === "path") {
|
|
1557
|
+
const located = citedPath(run, ref.path);
|
|
1558
|
+
if ("rung" in located)
|
|
1559
|
+
return { reached: located };
|
|
1560
|
+
origin = located.origin;
|
|
1561
|
+
const state = stateRootOf(run, located.origin, located.real);
|
|
1562
|
+
if (state !== undefined) {
|
|
1563
|
+
// Cited — the file is there, as the offline gate also finds — and
|
|
1564
|
+
// never read: what is under a state root is the model's own text.
|
|
1565
|
+
return {
|
|
1566
|
+
reached: fail("cited", "not_fetched", `${origin} is under ${state}, the harness's own state — session logs, proof bundles, spilled results and memory hold the model's own text, so nothing there is a source; cite the source itself`, { origin }),
|
|
1567
|
+
};
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
else {
|
|
1571
|
+
const written = ref.kind === "url" ? ref.url : ref.doi;
|
|
1572
|
+
const url = ref.kind === "url" ? normalizeUrl(ref.url) : doiUrl(ref.doi);
|
|
1573
|
+
if (url === undefined) {
|
|
1574
|
+
return {
|
|
1575
|
+
reached: fail("none", "not_locatable", `${clip(written, 120)} is not a well-formed URL`),
|
|
1576
|
+
};
|
|
1577
|
+
}
|
|
1578
|
+
if (ref.kind === "url" && hasUserinfo(ref.url)) {
|
|
1579
|
+
// The reader takes the text before `@` for the host; the page is on
|
|
1580
|
+
// the host after it. Named without the credentials themselves.
|
|
1581
|
+
const host = hostOf(url) ?? url;
|
|
1582
|
+
return {
|
|
1583
|
+
reached: fail("none", "host_denied", `this URL puts a user name or password before its host, so it reads as another site — the page is on ${host}; cite its URL without them`, { origin: url }),
|
|
1584
|
+
};
|
|
1585
|
+
}
|
|
1586
|
+
const refused = await citedUrl(run, url);
|
|
1587
|
+
if (refused !== undefined)
|
|
1588
|
+
return { reached: refused };
|
|
1589
|
+
origin = url;
|
|
1590
|
+
}
|
|
1591
|
+
return grounded(run, claim, origin);
|
|
1592
|
+
}
|
|
1593
|
+
// ---------------------------------------------------------------------------
|
|
1594
|
+
// execution bindings: checked
|
|
1595
|
+
// ---------------------------------------------------------------------------
|
|
1596
|
+
/**
|
|
1597
|
+
* Is a recorded tool name on `execution.allow`? The IR carries CANONICAL
|
|
1598
|
+
* entries (ir `IrProvability.execution.allow`): a builtin is its spec key
|
|
1599
|
+
* (`moneyAdd`), anything else its exact name. A recorded builtin name
|
|
1600
|
+
* (`MoneyAdd`) is mapped to its key case-insensitively, the way the compiler
|
|
1601
|
+
* maps `tools:`; a non-builtin name (`mcp__calc__add`) must match exactly,
|
|
1602
|
+
* because MCP tool names are case-sensitive.
|
|
1603
|
+
*/
|
|
1604
|
+
export function isToolAllowed(toolName, allow) {
|
|
1605
|
+
if (allow.includes(toolName))
|
|
1606
|
+
return true;
|
|
1607
|
+
const key = builtinKey(toolName);
|
|
1608
|
+
return key !== undefined && allow.includes(key);
|
|
1609
|
+
}
|
|
1610
|
+
async function replayOnce(run, record) {
|
|
1611
|
+
let pending = run.replays.get(record.toolUseId);
|
|
1612
|
+
if (pending === undefined) {
|
|
1613
|
+
if (run.replaysUsed >= run.config.execution.maxReplays)
|
|
1614
|
+
return "budget";
|
|
1615
|
+
run.replaysUsed += 1;
|
|
1616
|
+
const replay = run.opts.replay;
|
|
1617
|
+
// A replay that throws, or answers with something that is not a result,
|
|
1618
|
+
// could not run: the binding stays `checked` with `replayed: false`.
|
|
1619
|
+
pending = (async () => {
|
|
1620
|
+
try {
|
|
1621
|
+
const result = await replay(record.toolName, record.input);
|
|
1622
|
+
if (result === undefined || typeof result.content !== "string")
|
|
1623
|
+
return "not_run";
|
|
1624
|
+
if (result.isError === true)
|
|
1625
|
+
return "mismatch";
|
|
1626
|
+
return outputDigest(result.content) === outputDigest(record.content) ? "match" : "mismatch";
|
|
1627
|
+
}
|
|
1628
|
+
catch {
|
|
1629
|
+
return "not_run";
|
|
1630
|
+
}
|
|
1631
|
+
})();
|
|
1632
|
+
run.replays.set(record.toolUseId, pending);
|
|
1633
|
+
}
|
|
1634
|
+
return pending;
|
|
1635
|
+
}
|
|
1636
|
+
/** The caller's purity predicate; one that throws is read as "not pure". */
|
|
1637
|
+
function isPure(run, toolName) {
|
|
1638
|
+
try {
|
|
1639
|
+
return run.opts.isPureTool?.(toolName) === true;
|
|
1640
|
+
}
|
|
1641
|
+
catch {
|
|
1642
|
+
return false;
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1645
|
+
function callFacts(run, record) {
|
|
1646
|
+
let facts = run.calls.get(record);
|
|
1647
|
+
if (facts === undefined) {
|
|
1648
|
+
facts = { content: record.content };
|
|
1649
|
+
run.calls.set(record, facts);
|
|
1650
|
+
}
|
|
1651
|
+
return facts;
|
|
1652
|
+
}
|
|
1653
|
+
/**
|
|
1654
|
+
* A token as the figure polarity rule compares it: a numeral by the value
|
|
1655
|
+
* the number layer reads (`#1284.5` for `1,284.50` and `1284.5`), so a
|
|
1656
|
+
* claim's figure meets the result's however each is written; any other word
|
|
1657
|
+
* by its normalized form.
|
|
1658
|
+
*/
|
|
1659
|
+
function polarityKey(token) {
|
|
1660
|
+
if (/^\d/.test(token.raw)) {
|
|
1661
|
+
const value = normalizeNumber(token.raw);
|
|
1662
|
+
if (value !== null)
|
|
1663
|
+
return `#${value}`;
|
|
1664
|
+
}
|
|
1665
|
+
return token.norm;
|
|
1666
|
+
}
|
|
1667
|
+
/**
|
|
1668
|
+
* Is token `at` denied — governed by a negation (claim-ledger's
|
|
1669
|
+
* `governingNegation`) other than a partitive? "None of the 3 invoices is
|
|
1670
|
+
* overdue" denies the invoices' being overdue, not that there are 3.
|
|
1671
|
+
*/
|
|
1672
|
+
function deniedAt(index, at) {
|
|
1673
|
+
const negation = governingNegation(index, at);
|
|
1674
|
+
return negation !== -1 && index.tokens[negation + 1]?.norm !== "of";
|
|
1675
|
+
}
|
|
1676
|
+
/**
|
|
1677
|
+
* The negations that deny a predicate: "not", "never", "cannot" and the
|
|
1678
|
+
* `n't` forms. A determiner or a preposition ("no", "none", "without",
|
|
1679
|
+
* "neither … nor") qualifies the noun after it instead: "no tests failed
|
|
1680
|
+
* out of 120" says none of the tests failed, not that there were no tests,
|
|
1681
|
+
* and "the subtotal without VAT" names a subtotal, it does not deny VAT.
|
|
1682
|
+
* Read as denying the noun, those fail a correct claim that uses it ("all
|
|
1683
|
+
* 120 tests passed", "the subtotal excluding VAT"), so the figure polarity
|
|
1684
|
+
* rule counts only these for a word that is not a figure — on the claim's
|
|
1685
|
+
* side and the result's alike, so "without VAT" in both still agrees. A
|
|
1686
|
+
* figure keeps every negation: "no 2 meetings overlap" denies 2.
|
|
1687
|
+
*/
|
|
1688
|
+
const PREDICATE_NEGATIONS = new Set([...NEGATIONS].filter((word) => !["no", "none", "without", "neither", "nor"].includes(word)));
|
|
1689
|
+
/** Is token `at`, a word that is not a figure, denied by a predicate negation? */
|
|
1690
|
+
function wordDeniedAt(index, at) {
|
|
1691
|
+
const negation = governingNegation(index, at);
|
|
1692
|
+
return negation !== -1 && PREDICATE_NEGATIONS.has(index.tokens[negation].norm);
|
|
1693
|
+
}
|
|
1694
|
+
/**
|
|
1695
|
+
* What ends a statement: sentence punctuation, a comma or a semicolon, a
|
|
1696
|
+
* table bar, a line break. Every other clause break (claim-ledger
|
|
1697
|
+
* `breaksClause`) — a colon, a bracket, a quote, a dash — joins a figure to
|
|
1698
|
+
* what is said of it: "91: not prime", "91 (not prime)", "91 - not prime",
|
|
1699
|
+
* "not prime: 91 = 7 * 13", a JSON key and its value. A question mark ends
|
|
1700
|
+
* one too, so "Is 91 prime? No." is two statements — and a bare "No." denies
|
|
1701
|
+
* nothing `governingNegation` can see (a known limit, not closed here).
|
|
1702
|
+
*/
|
|
1703
|
+
const STATEMENT_END = /[.,;!?…|\n\r]/;
|
|
1704
|
+
/**
|
|
1705
|
+
* For each token of `index`, the number of the statement it sits in, and
|
|
1706
|
+
* where each statement starts: one pass over the text between tokens.
|
|
1707
|
+
*/
|
|
1708
|
+
function statementNumbers(index) {
|
|
1709
|
+
const tokens = index.tokens;
|
|
1710
|
+
const statement = new Uint32Array(tokens.length);
|
|
1711
|
+
const starts = tokens.length > 0 ? [0] : [];
|
|
1712
|
+
let n = 0;
|
|
1713
|
+
for (let i = 1; i < tokens.length; i++) {
|
|
1714
|
+
const left = tokens[i - 1];
|
|
1715
|
+
const gap = index.text.slice(left.at + left.raw.length, tokens[i].at);
|
|
1716
|
+
if (STATEMENT_END.test(gap)) {
|
|
1717
|
+
n += 1;
|
|
1718
|
+
starts.push(i);
|
|
1719
|
+
}
|
|
1720
|
+
statement[i] = n;
|
|
1721
|
+
}
|
|
1722
|
+
return { statement, starts };
|
|
1723
|
+
}
|
|
1724
|
+
/**
|
|
1725
|
+
* Result tokens the figure polarity rule has read into a statement's table
|
|
1726
|
+
* of words: for a test to count with, as claim-ledger's `scanMeter` is.
|
|
1727
|
+
* Only ever added to.
|
|
1728
|
+
*/
|
|
1729
|
+
export const polarityMeter = { tokens: 0 };
|
|
1730
|
+
function resultPolarity(content) {
|
|
1731
|
+
const index = indexSource(content);
|
|
1732
|
+
const { statement, starts } = statementNumbers(index);
|
|
1733
|
+
const figureAt = new Map();
|
|
1734
|
+
index.tokens.forEach((token, i) => {
|
|
1735
|
+
const key = polarityKey(token);
|
|
1736
|
+
if (!key.startsWith("#"))
|
|
1737
|
+
return;
|
|
1738
|
+
const at = figureAt.get(key);
|
|
1739
|
+
if (at === undefined)
|
|
1740
|
+
figureAt.set(key, [i]);
|
|
1741
|
+
else
|
|
1742
|
+
at.push(i);
|
|
1743
|
+
});
|
|
1744
|
+
return {
|
|
1745
|
+
index,
|
|
1746
|
+
statement,
|
|
1747
|
+
statementStarts: starts,
|
|
1748
|
+
figureAt,
|
|
1749
|
+
statements: new Map(),
|
|
1750
|
+
figures: new Map(),
|
|
1751
|
+
};
|
|
1752
|
+
}
|
|
1753
|
+
/** Every word of statement `n` that is not a figure or a negation, read once. */
|
|
1754
|
+
function statementWords(result, n) {
|
|
1755
|
+
let words = result.statements.get(n);
|
|
1756
|
+
if (words === undefined) {
|
|
1757
|
+
const table = new Map();
|
|
1758
|
+
const { index } = result;
|
|
1759
|
+
const end = result.statementStarts[n + 1] ?? index.tokens.length;
|
|
1760
|
+
for (let at = result.statementStarts[n]; at < end; at++) {
|
|
1761
|
+
polarityMeter.tokens += 1;
|
|
1762
|
+
const key = polarityKey(index.tokens[at]);
|
|
1763
|
+
if (key.startsWith("#") || NEGATIONS.has(key))
|
|
1764
|
+
continue;
|
|
1765
|
+
let facts = table.get(key);
|
|
1766
|
+
if (facts === undefined) {
|
|
1767
|
+
facts = { occurrences: 0, denials: 0 };
|
|
1768
|
+
table.set(key, facts);
|
|
1769
|
+
}
|
|
1770
|
+
facts.occurrences += 1;
|
|
1771
|
+
if (wordDeniedAt(index, at))
|
|
1772
|
+
facts.denials += 1;
|
|
1773
|
+
}
|
|
1774
|
+
result.statements.set(n, table);
|
|
1775
|
+
words = table;
|
|
1776
|
+
}
|
|
1777
|
+
return words;
|
|
1778
|
+
}
|
|
1779
|
+
/**
|
|
1780
|
+
* What the result says of one figure, read once per figure: its own
|
|
1781
|
+
* occurrences, and the words of every statement holding it, merged. A word
|
|
1782
|
+
* is compared only by whether it occurs there, whether any occurrence is
|
|
1783
|
+
* denied and whether every one is, and each of those reads the same over
|
|
1784
|
+
* the statements of several figures as over each figure's in turn — so a
|
|
1785
|
+
* claim with several figures needs no table of its own.
|
|
1786
|
+
*/
|
|
1787
|
+
function figureFacts(result, key) {
|
|
1788
|
+
let facts = result.figures.get(key);
|
|
1789
|
+
if (facts === undefined) {
|
|
1790
|
+
const at = result.figureAt.get(key) ?? [];
|
|
1791
|
+
const self = {
|
|
1792
|
+
occurrences: at.length,
|
|
1793
|
+
denials: at.filter((i) => deniedAt(result.index, i)).length,
|
|
1794
|
+
};
|
|
1795
|
+
const statements = [...new Set(at.map((i) => result.statement[i]))];
|
|
1796
|
+
let words;
|
|
1797
|
+
if (statements.length === 1)
|
|
1798
|
+
words = statementWords(result, statements[0]);
|
|
1799
|
+
else {
|
|
1800
|
+
const merged = new Map();
|
|
1801
|
+
for (const n of statements) {
|
|
1802
|
+
for (const [word, one] of statementWords(result, n)) {
|
|
1803
|
+
const into = merged.get(word);
|
|
1804
|
+
if (into === undefined)
|
|
1805
|
+
merged.set(word, { ...one });
|
|
1806
|
+
else {
|
|
1807
|
+
into.occurrences += one.occurrences;
|
|
1808
|
+
into.denials += one.denials;
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
words = merged;
|
|
1813
|
+
}
|
|
1814
|
+
facts = { self, words };
|
|
1815
|
+
result.figures.set(key, facts);
|
|
1816
|
+
}
|
|
1817
|
+
return facts;
|
|
1818
|
+
}
|
|
1819
|
+
/**
|
|
1820
|
+
* Where a claim that states a figure and the result it cites disagree on a
|
|
1821
|
+
* negation (design §6.3), or `undefined` when they do not.
|
|
1822
|
+
*
|
|
1823
|
+
* Value containment finds the claim's figures in the result; it cannot see
|
|
1824
|
+
* which way the sentence around them points. Reading the whole sentence for
|
|
1825
|
+
* a negation is too coarse in both directions: "no discount was applied, so
|
|
1826
|
+
* the total is 1,284.50" states the total, and a result saying "1284.50
|
|
1827
|
+
* (no rounding)" does not deny it. So polarity is read per word, as
|
|
1828
|
+
* claim-ledger's `governingNegation` scopes it, and only where the result
|
|
1829
|
+
* speaks to the claimed figures — the statement each sits in, which a
|
|
1830
|
+
* colon, a bracket or a dash does not end ({@link STATEMENT_END}):
|
|
1831
|
+
*
|
|
1832
|
+
* - A figure the claim denies ("the total is NOT 1,284.50") must be denied
|
|
1833
|
+
* in the result too. A result that states it, unnegated, everywhere it
|
|
1834
|
+
* appears says the reverse.
|
|
1835
|
+
* - Any other word of the claim that the result denies everywhere it
|
|
1836
|
+
* appears in a statement holding one of the claim's figures, while the
|
|
1837
|
+
* claim states it: "91 is NOT prime", "91: not prime", "91 (not prime)"
|
|
1838
|
+
* do not check "91 is prime". The same word stated by the result where
|
|
1839
|
+
* the claim denies it is the reverse case ("91 is not prime" citing "91:
|
|
1840
|
+
* prime"). For such a word only a predicate negation counts, on either
|
|
1841
|
+
* side ({@link PREDICATE_NEGATIONS}): "no tests failed out of 120" does
|
|
1842
|
+
* not deny the tests of "all 120 tests passed".
|
|
1843
|
+
*
|
|
1844
|
+
* A word the result never uses, or uses only in statements about other
|
|
1845
|
+
* figures (a JSON field of its own, a sentence about something else), tells
|
|
1846
|
+
* nothing either way. The detail names the claim's own word, never the
|
|
1847
|
+
* result's text: it reaches the retry nudge.
|
|
1848
|
+
*/
|
|
1849
|
+
function figurePolarity(claimText, result) {
|
|
1850
|
+
const claim = indexSource(claimText);
|
|
1851
|
+
const words = new Set(claimWords(claimText));
|
|
1852
|
+
const figures = [];
|
|
1853
|
+
const stated = new Map();
|
|
1854
|
+
claim.tokens.forEach((token, i) => {
|
|
1855
|
+
if (!words.has(token.norm) || NEGATIONS.has(token.norm))
|
|
1856
|
+
return;
|
|
1857
|
+
const key = polarityKey(token);
|
|
1858
|
+
const figure = key.startsWith("#");
|
|
1859
|
+
if (figure && !figures.includes(key))
|
|
1860
|
+
figures.push(key);
|
|
1861
|
+
const entry = stated.get(key) ?? { raw: token.raw, denied: 0, asserted: 0 };
|
|
1862
|
+
if (figure ? deniedAt(claim, i) : wordDeniedAt(claim, i))
|
|
1863
|
+
entry.denied += 1;
|
|
1864
|
+
else
|
|
1865
|
+
entry.asserted += 1;
|
|
1866
|
+
stated.set(key, entry);
|
|
1867
|
+
});
|
|
1868
|
+
const facts = figures.map((key) => figureFacts(result, key));
|
|
1869
|
+
for (const [key, word] of stated) {
|
|
1870
|
+
// A claim that both states and denies one word says too much to read.
|
|
1871
|
+
if (word.denied > 0 && word.asserted > 0)
|
|
1872
|
+
continue;
|
|
1873
|
+
const figure = key.startsWith("#");
|
|
1874
|
+
const there = figure
|
|
1875
|
+
? [figureFacts(result, key).self]
|
|
1876
|
+
: facts.map((one) => one.words.get(key)).filter((one) => one !== undefined);
|
|
1877
|
+
const seen = there.some((one) => one.occurrences > 0);
|
|
1878
|
+
const anyDenied = there.some((one) => one.denials > 0);
|
|
1879
|
+
const allDenied = there.every((one) => one.denials === one.occurrences);
|
|
1880
|
+
if (word.denied > 0) {
|
|
1881
|
+
if ((figure || seen) && !anyDenied) {
|
|
1882
|
+
return figure
|
|
1883
|
+
? `states no negation of ${word.raw}, and the claim denies it — state only what the result says`
|
|
1884
|
+
: `says the opposite: it states "${word.raw}" beside the figure, where the claim denies it`;
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
else if (seen && allDenied) {
|
|
1888
|
+
return `says the opposite: it denies ${figure ? word.raw : `"${word.raw}"`}, which the claim states`;
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
return undefined;
|
|
1892
|
+
}
|
|
1893
|
+
/**
|
|
1894
|
+
* Where a claim with no figure and the call it cites disagree, by the locate
|
|
1895
|
+
* kernel reading the call for the claim the way `grounded` reads a body —
|
|
1896
|
+
* figure rule on, so a number word and a negation are never excused — or
|
|
1897
|
+
* `undefined`. Coverage cannot see which way the words point ("not" is a
|
|
1898
|
+
* stopword to it), and a result that says "no two meetings overlap" must
|
|
1899
|
+
* not check "two meetings overlap".
|
|
1900
|
+
*/
|
|
1901
|
+
function wordPolarity(run, claimText, said) {
|
|
1902
|
+
const location = locateClaim(claimQuery(claimText), said, {
|
|
1903
|
+
coverage: run.config.grounded.coverage,
|
|
1904
|
+
windowTokens: run.config.grounded.windowTokens,
|
|
1905
|
+
strictFigures: true,
|
|
1906
|
+
});
|
|
1907
|
+
// Only the claim's own words and the negations themselves reach the
|
|
1908
|
+
// detail (and so the nudge): never the result's other text.
|
|
1909
|
+
const negations = (location.differing ?? []).filter((word) => NEGATIONS.has(word));
|
|
1910
|
+
const listed = negations.length > 0 ? ` (${negations.join(", ")})` : "";
|
|
1911
|
+
switch (location.cause) {
|
|
1912
|
+
case "polarity":
|
|
1913
|
+
return `says the opposite: it carries the claim's words with a negation${listed} the claim lacks`;
|
|
1914
|
+
case "negation":
|
|
1915
|
+
return `does not carry the claim's negation${listed} beside its words — state only what the result says`;
|
|
1916
|
+
case "value": {
|
|
1917
|
+
const missing = (location.missing ?? []).filter((word) => figureKey(word) !== null);
|
|
1918
|
+
return `does not state ${missing.length > 0 ? missing.slice(0, 6).join(", ") : "the claim's figure"} beside the claim's words`;
|
|
1919
|
+
}
|
|
1920
|
+
default:
|
|
1921
|
+
return undefined;
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
/** The call's input as text: what the coverage fallback reads beside the result. */
|
|
1925
|
+
function inputText(input) {
|
|
1926
|
+
try {
|
|
1927
|
+
return JSON.stringify(input) ?? "";
|
|
1928
|
+
}
|
|
1929
|
+
catch {
|
|
1930
|
+
return "";
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
/** The result alone, indexed once per call. */
|
|
1934
|
+
function resultOf(facts) {
|
|
1935
|
+
facts.result ??= { tokens: contentTokens(facts.content), index: indexSource(facts.content) };
|
|
1936
|
+
return facts.result;
|
|
1937
|
+
}
|
|
1938
|
+
/** The input alone: its words, and each string it carries, indexed once per call. */
|
|
1939
|
+
function inputOf(facts, record) {
|
|
1940
|
+
facts.input ??= {
|
|
1941
|
+
tokens: contentTokens(inputText(record.input)),
|
|
1942
|
+
strings: inputStrings(record.input)
|
|
1943
|
+
.filter((value) => value.trim() !== "")
|
|
1944
|
+
.map((value) => indexSource(value)),
|
|
1945
|
+
};
|
|
1946
|
+
return facts.input;
|
|
1947
|
+
}
|
|
1948
|
+
/** At most this many distinct runs of one claim are looked for in a result ({@link handedRuns}). */
|
|
1949
|
+
const MAX_HANDED_RUNS = 8;
|
|
1950
|
+
/**
|
|
1951
|
+
* The claim as the call's input HANDED IT OVER: in each input string, every
|
|
1952
|
+
* run of it that holds the claim's words at their tightest (claim-ledger's
|
|
1953
|
+
* `tightestWindows`) and reads as the claim sentence — the claim's own
|
|
1954
|
+
* words, stopwords included, at the grounded bar, at least
|
|
1955
|
+
* `MIN_CLAIM_TOKENS` long and at most twice the claim. A draft handed to a
|
|
1956
|
+
* lint, a sentence handed to NormalizeText or WrapText, a claim handed to a
|
|
1957
|
+
* fact-check are that; a document a value is selected from is not — `{
|
|
1958
|
+
* "account": { "id": 7, "status": "suspended" } }` holds "account" and
|
|
1959
|
+
* "suspended", not "the account is suspended", and JsonQuery's
|
|
1960
|
+
* `["suspended"]` is the tool's answer, not the model's sentence (design
|
|
1961
|
+
* §6.3: the result contains the claim's values). Distinct runs, at most
|
|
1962
|
+
* {@link MAX_HANDED_RUNS}, in input order.
|
|
1963
|
+
*/
|
|
1964
|
+
function handedRuns(facts, record, claimText, bar) {
|
|
1965
|
+
const claim = indexSource(claimText);
|
|
1966
|
+
const own = new Set(claim.tokens.map((token) => wordKey(token.norm)));
|
|
1967
|
+
const need = new Set(claimWords(claimText).map(wordKey));
|
|
1968
|
+
if (own.size === 0 || need.size === 0)
|
|
1969
|
+
return [];
|
|
1970
|
+
const longest = 2 * claim.tokens.length;
|
|
1971
|
+
const out = new Map();
|
|
1972
|
+
for (const input of inputOf(facts, record).strings) {
|
|
1973
|
+
const keys = input.tokens.map((token) => wordKey(token.norm));
|
|
1974
|
+
for (const [from, to] of tightestWindows(keys, need)) {
|
|
1975
|
+
const length = to - from + 1;
|
|
1976
|
+
if (length < MIN_CLAIM_TOKENS || length > longest)
|
|
1977
|
+
continue;
|
|
1978
|
+
const window = new Set(keys.slice(from, to + 1));
|
|
1979
|
+
const share = (of) => [...of].filter((key) => window.has(key)).length / of.size;
|
|
1980
|
+
if (share(need) < bar || share(own) < bar)
|
|
1981
|
+
continue;
|
|
1982
|
+
const words = input.tokens.slice(from, to + 1).map((token) => token.norm);
|
|
1983
|
+
out.set(words.join(" "), words);
|
|
1984
|
+
if (out.size >= MAX_HANDED_RUNS)
|
|
1985
|
+
return [...out.values()];
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
return [...out.values()];
|
|
1989
|
+
}
|
|
1990
|
+
/** Where `words` occurs in `index`, every start, word for word (KMP: linear in the index). */
|
|
1991
|
+
function occurrences(index, words) {
|
|
1992
|
+
const m = words.length;
|
|
1993
|
+
const out = [];
|
|
1994
|
+
if (m === 0)
|
|
1995
|
+
return out;
|
|
1996
|
+
const border = new Int32Array(m);
|
|
1997
|
+
for (let j = 1, k = 0; j < m; j++) {
|
|
1998
|
+
while (k > 0 && words[j] !== words[k])
|
|
1999
|
+
k = border[k - 1];
|
|
2000
|
+
if (words[j] === words[k])
|
|
2001
|
+
k += 1;
|
|
2002
|
+
border[j] = k;
|
|
2003
|
+
}
|
|
2004
|
+
let k = 0;
|
|
2005
|
+
index.tokens.forEach((token, at) => {
|
|
2006
|
+
while (k > 0 && token.norm !== words[k])
|
|
2007
|
+
k = border[k - 1];
|
|
2008
|
+
if (token.norm === words[k])
|
|
2009
|
+
k += 1;
|
|
2010
|
+
if (k === m) {
|
|
2011
|
+
out.push(at - m + 1);
|
|
2012
|
+
k = border[k - 1];
|
|
2013
|
+
}
|
|
2014
|
+
});
|
|
2015
|
+
return out;
|
|
2016
|
+
}
|
|
2017
|
+
/**
|
|
2018
|
+
* The result with the claim the input handed over ({@link handedRuns})
|
|
2019
|
+
* blanked out where the tool repeats it — what the tool said of its own
|
|
2020
|
+
* (design §10: a real-but-irrelevant tool result fails `checked`). A tool
|
|
2021
|
+
* the gate allows because it is pure can still be one that returns what it
|
|
2022
|
+
* is given (NormalizeText, WrapText, a check that repeats the claim it was
|
|
2023
|
+
* asked about), and a figure or a word in the repeated sentence is the
|
|
2024
|
+
* model's, not a computation's. Each run is blanked as many times as the
|
|
2025
|
+
* input holds it, first occurrence first: a tool that repeats the claim and
|
|
2026
|
+
* then quotes a source saying the same keeps the source's copy. The facts
|
|
2027
|
+
* themselves when nothing is blanked; otherwise the blanked result's own,
|
|
2028
|
+
* kept per call and runs.
|
|
2029
|
+
*
|
|
2030
|
+
* What stays open is text the tool reassembles from several input strings
|
|
2031
|
+
* (a template filled from parts), and a draft padded past twice the
|
|
2032
|
+
* claim's length: no one input string hands that claim over.
|
|
2033
|
+
*/
|
|
2034
|
+
function ownResult(facts, record, runs) {
|
|
2035
|
+
if (runs.length === 0)
|
|
2036
|
+
return facts;
|
|
2037
|
+
const key = runs.map((words) => words.join(" ")).join("\u0000");
|
|
2038
|
+
facts.owns ??= new Map();
|
|
2039
|
+
let own = facts.owns.get(key);
|
|
2040
|
+
if (own === undefined) {
|
|
2041
|
+
const result = resultOf(facts).index;
|
|
2042
|
+
const { strings } = inputOf(facts, record);
|
|
2043
|
+
const blank = new Uint8Array(result.tokens.length);
|
|
2044
|
+
let blanked = false;
|
|
2045
|
+
for (const words of runs) {
|
|
2046
|
+
const handed = strings.reduce((n, input) => n + occurrences(input, words).length, 0);
|
|
2047
|
+
for (const at of occurrences(result, words).slice(0, handed)) {
|
|
2048
|
+
blank.fill(1, at, at + words.length);
|
|
2049
|
+
blanked = true;
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
if (!blanked)
|
|
2053
|
+
own = facts;
|
|
2054
|
+
else {
|
|
2055
|
+
let text = "";
|
|
2056
|
+
let from = 0;
|
|
2057
|
+
result.tokens.forEach((token, i) => {
|
|
2058
|
+
if (blank[i] !== 1)
|
|
2059
|
+
return;
|
|
2060
|
+
text += result.text.slice(from, token.at) + " ".repeat(token.raw.length);
|
|
2061
|
+
from = token.at + token.raw.length;
|
|
2062
|
+
});
|
|
2063
|
+
own = { content: text + result.text.slice(from) };
|
|
2064
|
+
}
|
|
2065
|
+
facts.owns.set(key, own);
|
|
2066
|
+
}
|
|
2067
|
+
return own;
|
|
2068
|
+
}
|
|
2069
|
+
/** The report a `FactCrossCheck` result is, read once per call; `null` when it is not one. */
|
|
2070
|
+
function factCheckReport(facts, record) {
|
|
2071
|
+
if (facts.factCheck !== undefined)
|
|
2072
|
+
return facts.factCheck;
|
|
2073
|
+
let report = null;
|
|
2074
|
+
try {
|
|
2075
|
+
const parsed = JSON.parse(record.body ?? facts.content);
|
|
2076
|
+
const claims = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
|
|
2077
|
+
? parsed["claims"]
|
|
2078
|
+
: undefined;
|
|
2079
|
+
if (Array.isArray(claims)) {
|
|
2080
|
+
const text = (value) => typeof value === "string" && value.trim() !== "" ? value : undefined;
|
|
2081
|
+
const field = (value, name) => typeof value === "object" && value !== null
|
|
2082
|
+
? value[name]
|
|
2083
|
+
: undefined;
|
|
2084
|
+
report = claims.flatMap((entry) => {
|
|
2085
|
+
const claim = text(field(entry, "claim"));
|
|
2086
|
+
const verdict = text(field(entry, "verdict"));
|
|
2087
|
+
if (claim === undefined || verdict === undefined)
|
|
2088
|
+
return [];
|
|
2089
|
+
const sources = field(entry, "sources");
|
|
2090
|
+
const supports = [];
|
|
2091
|
+
const seen = new Set();
|
|
2092
|
+
const add = (found) => {
|
|
2093
|
+
const excerpt = text(field(found, "excerpt"));
|
|
2094
|
+
if (excerpt === undefined)
|
|
2095
|
+
return;
|
|
2096
|
+
const source = text(field(found, "source"));
|
|
2097
|
+
const key = `${source ?? ""}\u0000${excerpt}`;
|
|
2098
|
+
if (seen.has(key))
|
|
2099
|
+
return;
|
|
2100
|
+
seen.add(key);
|
|
2101
|
+
supports.push(source === undefined ? { excerpt } : { source, excerpt });
|
|
2102
|
+
};
|
|
2103
|
+
add(field(entry, "foundIn"));
|
|
2104
|
+
if (Array.isArray(sources)) {
|
|
2105
|
+
for (const source of sources) {
|
|
2106
|
+
if (field(source, "status") === "supported")
|
|
2107
|
+
add(source);
|
|
2108
|
+
}
|
|
2109
|
+
}
|
|
2110
|
+
return [{ claim, verdict, supports }];
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
catch {
|
|
2115
|
+
report = null;
|
|
2116
|
+
}
|
|
2117
|
+
facts.factCheck = report;
|
|
2118
|
+
return report;
|
|
2119
|
+
}
|
|
2120
|
+
/**
|
|
2121
|
+
* What a `FactCrossCheck` result says of this claim (design §6.3, §10): the
|
|
2122
|
+
* source text of an entry that checked THIS claim — the sentence states
|
|
2123
|
+
* the entry's claim and the entry's claim states the sentence — with the
|
|
2124
|
+
* verdict `supported`, as the facts the checks read; or why it is none. The
|
|
2125
|
+
* report repeats every claim it was asked about, whatever it found, so read
|
|
2126
|
+
* whole it "contains" a refuted claim's figure: `notFound`, `polarityConflict`
|
|
2127
|
+
* and `misquoted` are the tool saying the source does NOT carry it. Read
|
|
2128
|
+
* only off the tool that prints it, as WebFetch's header is.
|
|
2129
|
+
*/
|
|
2130
|
+
function factCheckFacts(run, facts, record, claimText) {
|
|
2131
|
+
const report = factCheckReport(facts, record);
|
|
2132
|
+
if (report === null) {
|
|
2133
|
+
return "is not a FactCrossCheck report this gate can read — cite the call whose result it is";
|
|
2134
|
+
}
|
|
2135
|
+
const bar = run.config.grounded.coverage;
|
|
2136
|
+
const words = contentTokens(claimText);
|
|
2137
|
+
const stating = report
|
|
2138
|
+
.map((entry, i) => ({ entry, i }))
|
|
2139
|
+
.filter(({ entry }) => statementMismatch(entry.claim, claimText, bar) === undefined &&
|
|
2140
|
+
coverage(words, contentTokens(entry.claim)) >= bar);
|
|
2141
|
+
if (stating.length === 0) {
|
|
2142
|
+
return "reports no check of this claim — FactCrossCheck the sentence as the answer states it, then cite that call";
|
|
2143
|
+
}
|
|
2144
|
+
const supported = stating.find(({ entry }) => entry.verdict === "supported" && entry.supports.length > 0);
|
|
2145
|
+
if (supported === undefined) {
|
|
2146
|
+
return `reports this claim ${stating[0].entry.verdict}, not supported: the check found the cited source does not carry it — state only what the source says`;
|
|
2147
|
+
}
|
|
2148
|
+
facts.factChecks ??= new Map();
|
|
2149
|
+
let entry = facts.factChecks.get(supported.i);
|
|
2150
|
+
if (entry === undefined) {
|
|
2151
|
+
const read = factCheckSupport(run, record, supported.entry.supports, claimText);
|
|
2152
|
+
if (typeof read === "string")
|
|
2153
|
+
return read;
|
|
2154
|
+
entry = read;
|
|
2155
|
+
facts.factChecks.set(supported.i, entry);
|
|
2156
|
+
}
|
|
2157
|
+
return entry;
|
|
2158
|
+
}
|
|
2159
|
+
/** The most of a supporting source re-read when FactCrossCheck clipped its excerpt. */
|
|
2160
|
+
const FACT_SOURCE_BYTES = 1024 * 1024;
|
|
2161
|
+
/**
|
|
2162
|
+
* The source text a supported `FactCrossCheck` entry stands on, as the
|
|
2163
|
+
* checks read it (design §1 principle 1, §6.3).
|
|
2164
|
+
*
|
|
2165
|
+
* - A source under a state root is no source (`stateRootOf`): FactCrossCheck
|
|
2166
|
+
* reads any workspace path, the run's own session log among them, and a
|
|
2167
|
+
* claim the model wrote once was found there, verbatim, and served
|
|
2168
|
+
* `checked ✓`. Such a source is dropped; with none left the claim fails,
|
|
2169
|
+
* naming the root.
|
|
2170
|
+
* So is a COPY of one: a file the run copied, moved, concatenated or
|
|
2171
|
+
* unpacked out of a state root before the check ({@link provenanceOf}) —
|
|
2172
|
+
* the session log copied into the workspace held the answer the gate had
|
|
2173
|
+
* just refused, and FactCrossCheck over the copy served it `checked ✓`.
|
|
2174
|
+
* - An excerpt the tool clipped (it ends in `…`, at 240 characters) may stop
|
|
2175
|
+
* short of the claim's figure: the claim is located again in the file the
|
|
2176
|
+
* tool read, with the gate's bar, and checked against that span. Where it
|
|
2177
|
+
* is not located there, the clipped excerpt is what there is.
|
|
2178
|
+
* - A file this run wrote the claim into before the check (design §10,
|
|
2179
|
+
* {@link selfWriter}) is the run's own text: checking it against itself
|
|
2180
|
+
* is no check. Where every file the check found the claim in is one, the
|
|
2181
|
+
* result carries the call that wrote it (`writtenBy`), and the binding
|
|
2182
|
+
* reaches a marked `grounded` at most — as the file itself, cited, does.
|
|
2183
|
+
*
|
|
2184
|
+
* A label that names no workspace file (a written reference, a URL the tool
|
|
2185
|
+
* did not read) keeps its excerpt as the tool reported it.
|
|
2186
|
+
*/
|
|
2187
|
+
function factCheckSupport(run, record, supports, claimText) {
|
|
2188
|
+
const texts = [];
|
|
2189
|
+
const refused = [];
|
|
2190
|
+
let writer;
|
|
2191
|
+
let outside = false;
|
|
2192
|
+
// The tool names a file it read by its workspace-relative path, whatever
|
|
2193
|
+
// directory the document cited it from.
|
|
2194
|
+
const fromRoot = { ...run, baseDir: run.workspaceRoot };
|
|
2195
|
+
for (const { source, excerpt } of supports) {
|
|
2196
|
+
let file;
|
|
2197
|
+
let state;
|
|
2198
|
+
if (source !== undefined) {
|
|
2199
|
+
const located = citedPath(fromRoot, source);
|
|
2200
|
+
if (!("rung" in located)) {
|
|
2201
|
+
file = located;
|
|
2202
|
+
state = stateRootOf(run, located.origin, located.real);
|
|
2203
|
+
}
|
|
2204
|
+
else {
|
|
2205
|
+
// Gone since, or never a file: a state root is still named by its path.
|
|
2206
|
+
const rel = normalizePath(source, {
|
|
2207
|
+
workspaceRoot: run.workspaceRoot,
|
|
2208
|
+
baseDir: run.workspaceRoot,
|
|
2209
|
+
});
|
|
2210
|
+
if (rel !== undefined)
|
|
2211
|
+
state = stateRootOf(run, rel, path.resolve(run.workspaceRoot, rel));
|
|
2212
|
+
}
|
|
2213
|
+
}
|
|
2214
|
+
if (state !== undefined) {
|
|
2215
|
+
refused.push(`${file?.origin ?? source} (under ${state})`);
|
|
2216
|
+
continue;
|
|
2217
|
+
}
|
|
2218
|
+
const provenance = file !== undefined ? provenanceOf(run, record, [file.origin]) : { into: [] };
|
|
2219
|
+
if (provenance.state !== undefined) {
|
|
2220
|
+
refused.push(`${file?.origin ?? source} (a copy of ${provenance.state.place}, under ${provenance.state.shown})`);
|
|
2221
|
+
continue;
|
|
2222
|
+
}
|
|
2223
|
+
const whole = excerpt.endsWith("\u2026") && file !== undefined
|
|
2224
|
+
? relocated(run, file.real, claimText)
|
|
2225
|
+
: undefined;
|
|
2226
|
+
const text = whole ?? excerpt;
|
|
2227
|
+
texts.push(text);
|
|
2228
|
+
let own;
|
|
2229
|
+
if (provenance.into.length > 0 &&
|
|
2230
|
+
file !== undefined &&
|
|
2231
|
+
!readBeforeWriting(run, file.origin, provenance.into, claimText)) {
|
|
2232
|
+
const claim = claimRunsIn(run, claimText, text);
|
|
2233
|
+
own = selfWriter(run, provenance.into, claim.runs, claim.index);
|
|
2234
|
+
}
|
|
2235
|
+
if (own === undefined)
|
|
2236
|
+
outside = true;
|
|
2237
|
+
else
|
|
2238
|
+
writer ??= own;
|
|
2239
|
+
}
|
|
2240
|
+
if (texts.length === 0) {
|
|
2241
|
+
return `found this claim only in ${refused.join(", ")}, the harness's own state — session logs, proof bundles, spilled results and memory hold the model's own text, so nothing there, or copied from there, is a source; check the claim against the source itself`;
|
|
2242
|
+
}
|
|
2243
|
+
return {
|
|
2244
|
+
content: [...new Set(texts)].join("\n"),
|
|
2245
|
+
...(!outside && writer !== undefined ? { writtenBy: writer } : {}),
|
|
2246
|
+
};
|
|
2247
|
+
}
|
|
2248
|
+
/**
|
|
2249
|
+
* The claim at its tightest in `text` ({@link spanRuns}), for the
|
|
2250
|
+
* self-written test of a source FactCrossCheck read, with the index of
|
|
2251
|
+
* `text` the runs stand in; the claim's own words, as one run, where the
|
|
2252
|
+
* gate's bar does not locate it there.
|
|
2253
|
+
*/
|
|
2254
|
+
function claimRunsIn(run, claimText, text) {
|
|
2255
|
+
const query = claimQuery(claimText);
|
|
2256
|
+
const index = indexSource(text);
|
|
2257
|
+
const location = locateClaim(query, index, locateOptions(run));
|
|
2258
|
+
if (location.status === "supported" || isWaivableMiss(location)) {
|
|
2259
|
+
const runs = spanRuns(run, query, index, location);
|
|
2260
|
+
if (runs.length > 0)
|
|
2261
|
+
return { runs, index };
|
|
2262
|
+
}
|
|
2263
|
+
return { runs: [wordsOf(claimText)], index };
|
|
2264
|
+
}
|
|
2265
|
+
/**
|
|
2266
|
+
* Did the run read `origin` before the first of `writers` wrote into it,
|
|
2267
|
+
* and find the claim there (design §10)? Then the file said it before the
|
|
2268
|
+
* run wrote anything, and a FactCrossCheck of it is no check of the run's
|
|
2269
|
+
* own text — as on the path route, where a read made before any writer
|
|
2270
|
+
* grounds unmarked.
|
|
2271
|
+
*/
|
|
2272
|
+
function readBeforeWriting(run, origin, writers, claimText) {
|
|
2273
|
+
const sent = sentText(run);
|
|
2274
|
+
const first = Math.min(...writers);
|
|
2275
|
+
const options = locateOptions(run);
|
|
2276
|
+
for (const record of run.evidence.bySource(origin)) {
|
|
2277
|
+
if (record.isError || !servesPage(record.status))
|
|
2278
|
+
continue;
|
|
2279
|
+
const at = sent.position.get(callOf(record));
|
|
2280
|
+
if (at === undefined || at >= first)
|
|
2281
|
+
continue;
|
|
2282
|
+
if (run.groundingTokens >= groundingBudget)
|
|
2283
|
+
return false;
|
|
2284
|
+
const body = bodyOf(run, record);
|
|
2285
|
+
run.groundingTokens += body.index.tokens.length;
|
|
2286
|
+
if (locateClaim(claimQuery(claimText), body.index, options).status === "supported")
|
|
2287
|
+
return true;
|
|
2288
|
+
}
|
|
2289
|
+
return false;
|
|
2290
|
+
}
|
|
2291
|
+
/** The claim located in a workspace file at the gate's bar, its span unclipped; `undefined` when it is not there. */
|
|
2292
|
+
function relocated(run, real, claimText) {
|
|
2293
|
+
let body;
|
|
2294
|
+
try {
|
|
2295
|
+
const stat = statSync(real);
|
|
2296
|
+
if (!stat.isFile() || stat.size > FACT_SOURCE_BYTES)
|
|
2297
|
+
return undefined;
|
|
2298
|
+
body = readFileSync(real, "utf-8");
|
|
2299
|
+
}
|
|
2300
|
+
catch {
|
|
2301
|
+
return undefined;
|
|
2302
|
+
}
|
|
2303
|
+
const found = locateClaim(claimText, body, {
|
|
2304
|
+
...locateOptions(run),
|
|
2305
|
+
maxExcerptChars: MAX_ECHO_SPAN_CHARS,
|
|
2306
|
+
});
|
|
2307
|
+
return found.status === "supported" ? found.excerpt : undefined;
|
|
2308
|
+
}
|
|
2309
|
+
/**
|
|
2310
|
+
* The excerpt an execution binding freezes (design §5.3): the whole result
|
|
2311
|
+
* when it fits in {@link MAX_EXCERPT_CHARS}; otherwise the part around the
|
|
2312
|
+
* first of the claim's figures the result states (or its quotation), so a
|
|
2313
|
+
* bundle read after the session log is gone shows the value that met the
|
|
2314
|
+
* floor — the first 240 characters of a forty-line result did not. Kept
|
|
2315
|
+
* per call and position, so many claims citing one result read it once.
|
|
2316
|
+
*/
|
|
2317
|
+
function executionExcerpt(facts, claimText, figured, quoted) {
|
|
2318
|
+
facts.excerpt ??= clip(facts.content, MAX_EXCERPT_CHARS);
|
|
2319
|
+
if (!facts.excerpt.endsWith("…"))
|
|
2320
|
+
return facts.excerpt;
|
|
2321
|
+
let index;
|
|
2322
|
+
let at = -1;
|
|
2323
|
+
let length = 1;
|
|
2324
|
+
if (figured) {
|
|
2325
|
+
facts.polarity ??= resultPolarity(facts.content);
|
|
2326
|
+
index = facts.polarity.index;
|
|
2327
|
+
for (const token of indexSource(claimText).tokens) {
|
|
2328
|
+
if (!/^\d/.test(token.raw))
|
|
2329
|
+
continue;
|
|
2330
|
+
const first = facts.polarity.figureAt.get(polarityKey(token))?.[0];
|
|
2331
|
+
if (first !== undefined) {
|
|
2332
|
+
at = first;
|
|
2333
|
+
break;
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
}
|
|
2337
|
+
else if (quoted) {
|
|
2338
|
+
const quote = claimQuery(claimText).quotes[0];
|
|
2339
|
+
if (quote !== undefined) {
|
|
2340
|
+
index = resultOf(facts).index;
|
|
2341
|
+
at = findSequence(index, quote);
|
|
2342
|
+
length = quote.length;
|
|
2343
|
+
}
|
|
2344
|
+
}
|
|
2345
|
+
if (index === undefined || at < 0)
|
|
2346
|
+
return facts.excerpt;
|
|
2347
|
+
const key = at * 1_000 + Math.min(length, 999);
|
|
2348
|
+
facts.excerpts ??= new Map();
|
|
2349
|
+
let excerpt = facts.excerpts.get(key);
|
|
2350
|
+
if (excerpt === undefined) {
|
|
2351
|
+
const first = index.tokens[at];
|
|
2352
|
+
const last = index.tokens[at + length - 1];
|
|
2353
|
+
// The result as written, not the folded text its tokens index (design §5.3).
|
|
2354
|
+
const cut = writtenSpan(index, first.at, last.at + last.raw.length);
|
|
2355
|
+
excerpt = clipAround(cut.text, cut.from, cut.to, MAX_EXCERPT_CHARS);
|
|
2356
|
+
facts.excerpts.set(key, excerpt);
|
|
2357
|
+
}
|
|
2358
|
+
return excerpt;
|
|
2359
|
+
}
|
|
2360
|
+
async function executionBinding(run, claim, toolUseId) {
|
|
2361
|
+
const record = run.evidence.byToolUseId(toolUseId);
|
|
2362
|
+
const calls = record === undefined ? (run.evidence.callsById?.(toolUseId) ?? []) : [];
|
|
2363
|
+
if (calls.length > 1) {
|
|
2364
|
+
// Several sub-agents made a call by this id (a provider that numbers
|
|
2365
|
+
// ids per conversation) and the session itself did not: grading one of
|
|
2366
|
+
// them for the claim would be a guess (`buildEvidenceIndex`).
|
|
2367
|
+
return fail("none", "tool_missing", `${toolUseId} names ${calls.length} calls in this run's sub-agent sessions and none in the session itself — the gate cannot tell which one is cited; cite a result of this session's own calls`, { origin: toolUseId });
|
|
2368
|
+
}
|
|
2369
|
+
if (record === undefined) {
|
|
2370
|
+
return fail("none", "tool_missing", `no recorded call ${toolUseId} in this run's logs${run.opts.turnNumbers !== undefined ? " (this turn)" : ""}`, { origin: toolUseId });
|
|
2371
|
+
}
|
|
2372
|
+
const facts = callFacts(run, record);
|
|
2373
|
+
const content = facts.content;
|
|
2374
|
+
facts.sha256 ??= sha256Of(content);
|
|
2375
|
+
const base = { origin: toolUseId, sha256: facts.sha256 };
|
|
2376
|
+
if (record.isError) {
|
|
2377
|
+
return fail("none", "tool_errored", `${toolUseId} (${record.toolName}) returned an error`, base);
|
|
2378
|
+
}
|
|
2379
|
+
if (!isToolAllowed(record.toolName, run.config.execution.allow)) {
|
|
2380
|
+
return fail("none", "tool_not_allowed", `${record.toolName} is not on provability.execution.allow`, base);
|
|
2381
|
+
}
|
|
2382
|
+
const valueAbsent = (detail) => fail("none", "value_absent", `the result of ${toolUseId} (${record.toolName}) ${detail}`, base);
|
|
2383
|
+
const percent = (share) => `${Math.round(share * 100)}%`;
|
|
2384
|
+
const bar = run.config.grounded.coverage;
|
|
2385
|
+
// The result as the checks read it. A FactCrossCheck result is read by
|
|
2386
|
+
// its verdict: the claim is checked by the source text of an entry that
|
|
2387
|
+
// found it supported, never by the report's repetition of the claim it
|
|
2388
|
+
// was asked about.
|
|
2389
|
+
let result = facts;
|
|
2390
|
+
if (builtinKey(record.toolName) === builtinKey(FACT_CROSS_CHECK)) {
|
|
2391
|
+
const entry = factCheckFacts(run, facts, record, claim.text);
|
|
2392
|
+
if (typeof entry === "string")
|
|
2393
|
+
return valueAbsent(entry);
|
|
2394
|
+
result = entry;
|
|
2395
|
+
}
|
|
2396
|
+
// Anything else is read with the claim the input handed over blanked out
|
|
2397
|
+
// where the tool repeats it (`ownResult`): what is left is the tool's.
|
|
2398
|
+
const handed = handedRuns(facts, record, claim.text, bar);
|
|
2399
|
+
const own = result === facts ? ownResult(facts, record, handed) : result;
|
|
2400
|
+
const handsBack = () => valueAbsent("hands the call's own input back: the claim's words in it are text the call was given — cite a result that computes or reads what the claim states");
|
|
2401
|
+
// Value containment (design §6.3): a figure or a quotation the claim
|
|
2402
|
+
// states must be in what the tool returned. A claim with neither still has
|
|
2403
|
+
// to be ABOUT the result — an allowed tool's unrelated output must not
|
|
2404
|
+
// check any sentence that cites it — so its words are measured against the
|
|
2405
|
+
// call (input and result) at the grounded bar.
|
|
2406
|
+
const numbers = extractNumbers(claim.text);
|
|
2407
|
+
const quotes = quotedSpans(claim.text);
|
|
2408
|
+
if (numbers.length > 0 || quotes.length > 0) {
|
|
2409
|
+
const absentFrom = (from) => {
|
|
2410
|
+
from.numbers ??= new Set(extractNumbers(from.content));
|
|
2411
|
+
const stated = from.numbers;
|
|
2412
|
+
return [
|
|
2413
|
+
...numbers.filter((value) => !containsValue(stated, value)),
|
|
2414
|
+
...quotes.filter((quote) => !containsQuote(from.content, quote)).map((q) => `"${q.text}"`),
|
|
2415
|
+
];
|
|
2416
|
+
};
|
|
2417
|
+
const absent = absentFrom(own);
|
|
2418
|
+
if (absent.length > 0) {
|
|
2419
|
+
// In the result only where it repeats what the call was handed.
|
|
2420
|
+
if (own !== result && absentFrom(result).length === 0)
|
|
2421
|
+
return handsBack();
|
|
2422
|
+
return valueAbsent(`does not contain ${absent.slice(0, 6).join(", ")}`);
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
if (numbers.length > 0) {
|
|
2426
|
+
// The figures are there; which way the sentence around them points is
|
|
2427
|
+
// read word by word, where the result speaks to those figures.
|
|
2428
|
+
own.polarity ??= resultPolarity(own.content);
|
|
2429
|
+
const opposite = figurePolarity(claim.text, own.polarity);
|
|
2430
|
+
if (opposite !== undefined)
|
|
2431
|
+
return valueAbsent(opposite);
|
|
2432
|
+
}
|
|
2433
|
+
else {
|
|
2434
|
+
if (own.said === undefined) {
|
|
2435
|
+
const text = `${own.content}\n${inputText(record.input)}`;
|
|
2436
|
+
own.said = { tokens: contentTokens(text), index: indexSource(text) };
|
|
2437
|
+
}
|
|
2438
|
+
if (quotes.length === 0) {
|
|
2439
|
+
const words = contentTokens(claim.text);
|
|
2440
|
+
const share = coverage(words, own.said.tokens);
|
|
2441
|
+
if (share < bar) {
|
|
2442
|
+
return valueAbsent(`does not carry this claim: the claim states no figure or quotation, and ${percent(share)} of its words are in the call's input and result (needs ${percent(bar)})`);
|
|
2443
|
+
}
|
|
2444
|
+
const left = coverage(words, resultOf(own).tokens);
|
|
2445
|
+
// The words were in the result only where it repeats the claim the
|
|
2446
|
+
// call was handed (NormalizeText, WrapText).
|
|
2447
|
+
if (own !== result && left < bar && coverage(words, resultOf(result).tokens) >= bar) {
|
|
2448
|
+
return handsBack();
|
|
2449
|
+
}
|
|
2450
|
+
// The input is read beside the result so a claim may name what the
|
|
2451
|
+
// call was asked ("the currency rounding check") and state what came
|
|
2452
|
+
// back ("found no drift"). But when the input HANDS THE CLAIM OVER —
|
|
2453
|
+
// a lint run over the draft answer, given the very sentence that
|
|
2454
|
+
// cites it — the input is all the call says of it, and would otherwise
|
|
2455
|
+
// check any sentence at all, so the result has to carry it too. A
|
|
2456
|
+
// document a value was selected from does not hand the claim over
|
|
2457
|
+
// (`handedRuns`): JsonQuery's `["suspended"]` checks "the account is
|
|
2458
|
+
// suspended" without repeating the account.
|
|
2459
|
+
if (left < bar && handed.length > 0) {
|
|
2460
|
+
return valueAbsent(`does not carry this claim: its words are the call's own input, and ${percent(left)} of them are in what the tool returned (needs ${percent(bar)})`);
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
// The words are there; the coverage measure cannot see which way they
|
|
2464
|
+
// point. The locate kernel, which keeps negations and number words,
|
|
2465
|
+
// reads the call for the claim (or for the framing of its quotation).
|
|
2466
|
+
const opposite = wordPolarity(run, claim.text, own.said.index);
|
|
2467
|
+
if (opposite !== undefined)
|
|
2468
|
+
return valueAbsent(opposite);
|
|
2469
|
+
}
|
|
2470
|
+
if (result.writtenBy !== undefined) {
|
|
2471
|
+
// FactCrossCheck found the claim only in files this run wrote it into
|
|
2472
|
+
// (`factCheckSupport`): checking the run's own text against itself is
|
|
2473
|
+
// no check (design §0 B, §10). It grounds as the file itself, cited,
|
|
2474
|
+
// does — and says whose text it is.
|
|
2475
|
+
const by = result.writtenBy;
|
|
2476
|
+
return {
|
|
2477
|
+
rung: "grounded",
|
|
2478
|
+
...base,
|
|
2479
|
+
excerpt: executionExcerpt(result, claim.text, numbers.length > 0, quotes.length > 0),
|
|
2480
|
+
replayed: false,
|
|
2481
|
+
writtenBy: { toolUseId: by.toolUseId, toolName: by.toolName },
|
|
2482
|
+
failure: {
|
|
2483
|
+
reason: "below_floor",
|
|
2484
|
+
detail: `${toolUseId} (${record.toolName}) found this claim only in text this run wrote there itself — ${by.toolUseId} (${by.toolName}) wrote it before the check — so it reaches grounded at most, marked as written by this run; check the claim against a source the run did not write`,
|
|
2485
|
+
},
|
|
2486
|
+
};
|
|
2487
|
+
}
|
|
2488
|
+
let replayed = false;
|
|
2489
|
+
// Replay only where it decides the floor: at `checked`. Below it the
|
|
2490
|
+
// binding already meets the floor; at `proven` it cannot.
|
|
2491
|
+
if (run.floor === "checked" &&
|
|
2492
|
+
run.config.execution.replay &&
|
|
2493
|
+
run.opts.replay !== undefined &&
|
|
2494
|
+
isPure(run, record.toolName)) {
|
|
2495
|
+
const outcome = await replayOnce(run, record);
|
|
2496
|
+
if (outcome === "mismatch") {
|
|
2497
|
+
return fail("none", "replay_mismatch", `re-running ${record.toolName} with the logged input produced a different output`, { ...base, replayed: true });
|
|
2498
|
+
}
|
|
2499
|
+
replayed = outcome === "match";
|
|
2500
|
+
}
|
|
2501
|
+
return {
|
|
2502
|
+
rung: "checked",
|
|
2503
|
+
...base,
|
|
2504
|
+
excerpt: executionExcerpt(result, claim.text, numbers.length > 0, quotes.length > 0),
|
|
2505
|
+
replayed,
|
|
2506
|
+
};
|
|
2507
|
+
}
|
|
2508
|
+
// ---------------------------------------------------------------------------
|
|
2509
|
+
// formal bindings: proven
|
|
2510
|
+
// ---------------------------------------------------------------------------
|
|
2511
|
+
async function artifactFor(run, id) {
|
|
2512
|
+
let pending = run.artifacts.get(id);
|
|
2513
|
+
if (pending === undefined) {
|
|
2514
|
+
const store = run.opts.artifacts;
|
|
2515
|
+
// Wrapped so a store that throws synchronously rejects like one that fails.
|
|
2516
|
+
pending = (async () => store.get(id))();
|
|
2517
|
+
run.artifacts.set(id, pending);
|
|
2518
|
+
}
|
|
2519
|
+
return pending;
|
|
2520
|
+
}
|
|
2521
|
+
/** Every word of a text that is not a negation, by {@link wordKey}, with its polarity. */
|
|
2522
|
+
function polarities(index) {
|
|
2523
|
+
const out = new Map();
|
|
2524
|
+
index.tokens.forEach((token, i) => {
|
|
2525
|
+
if (NEGATIONS.has(token.norm))
|
|
2526
|
+
return;
|
|
2527
|
+
const key = wordKey(token.norm);
|
|
2528
|
+
let polarity = out.get(key);
|
|
2529
|
+
if (polarity === undefined) {
|
|
2530
|
+
polarity = { stated: 0, denied: 0 };
|
|
2531
|
+
out.set(key, polarity);
|
|
2532
|
+
}
|
|
2533
|
+
if (deniedAt(index, i))
|
|
2534
|
+
polarity.denied += 1;
|
|
2535
|
+
else
|
|
2536
|
+
polarity.stated += 1;
|
|
2537
|
+
});
|
|
2538
|
+
return out;
|
|
2539
|
+
}
|
|
2540
|
+
/** "denied" or "stated" when every occurrence agrees; `undefined` when they do not, or there are none. */
|
|
2541
|
+
function polarityOf(polarity) {
|
|
2542
|
+
if (polarity === undefined)
|
|
2543
|
+
return undefined;
|
|
2544
|
+
if (polarity.denied > 0 && polarity.stated === 0)
|
|
2545
|
+
return "denied";
|
|
2546
|
+
if (polarity.stated > 0 && polarity.denied === 0)
|
|
2547
|
+
return "stated";
|
|
2548
|
+
return undefined;
|
|
2549
|
+
}
|
|
2550
|
+
/**
|
|
2551
|
+
* Does the text deny, through a partitive, a statement whose words are
|
|
2552
|
+
* `words`? "None of the 3 invoices is overdue" denies that the 3 invoices
|
|
2553
|
+
* are overdue, yet no single word of it is denied ({@link deniedAt} leaves
|
|
2554
|
+
* a partitive out — it denies the predicate, not the word after it — and
|
|
2555
|
+
* the predicate sits beyond the negation's reach), so the word-by-word
|
|
2556
|
+
* comparison cannot see it. A
|
|
2557
|
+
* partitive counts only when every content word it governs is one of
|
|
2558
|
+
* `words`: "none of the discounts applied, so the total is 1,284.50" says
|
|
2559
|
+
* nothing of the total.
|
|
2560
|
+
*/
|
|
2561
|
+
function partitiveDenial(index, words) {
|
|
2562
|
+
const governed = new Map();
|
|
2563
|
+
for (let i = 0; i < index.tokens.length; i++) {
|
|
2564
|
+
const negation = governingNegation(index, i);
|
|
2565
|
+
if (negation === -1 || index.tokens[negation + 1]?.norm !== "of")
|
|
2566
|
+
continue;
|
|
2567
|
+
const list = governed.get(negation);
|
|
2568
|
+
if (list === undefined)
|
|
2569
|
+
governed.set(negation, [i]);
|
|
2570
|
+
else
|
|
2571
|
+
list.push(i);
|
|
2572
|
+
}
|
|
2573
|
+
for (const targets of governed.values()) {
|
|
2574
|
+
const about = targets.every((i) => {
|
|
2575
|
+
const token = index.tokens[i];
|
|
2576
|
+
const content = claimWords(token.raw).length > 0 && !NEGATIONS.has(token.norm);
|
|
2577
|
+
return !content || words.has(wordKey(token.norm));
|
|
2578
|
+
});
|
|
2579
|
+
if (about)
|
|
2580
|
+
return true;
|
|
2581
|
+
}
|
|
2582
|
+
return false;
|
|
2583
|
+
}
|
|
2584
|
+
/** What may open a statement-level negation frame ({@link negationFrames}). */
|
|
2585
|
+
const FRAME_SUBJECTS = new Set(["it", "this", "that"]);
|
|
2586
|
+
/** What a frame says of a clause, plainly, to deny it: "it is false that …". */
|
|
2587
|
+
const FALSITY = new Set([
|
|
2588
|
+
"false",
|
|
2589
|
+
"untrue",
|
|
2590
|
+
"incorrect",
|
|
2591
|
+
"wrong",
|
|
2592
|
+
"inaccurate",
|
|
2593
|
+
]);
|
|
2594
|
+
/** What a frame says of a clause, negated, to deny it: "it is not true that …". */
|
|
2595
|
+
const TRUTH = new Set(["true", "correct", "accurate", "right"]);
|
|
2596
|
+
/**
|
|
2597
|
+
* Where the predicate of a negation frame that starts at `at` ends — "is
|
|
2598
|
+
* not true", "isn't the case", "is false", "is incorrect" — or -1.
|
|
2599
|
+
*/
|
|
2600
|
+
function framePredicate(words, at, subject) {
|
|
2601
|
+
let i = at;
|
|
2602
|
+
let negated = false;
|
|
2603
|
+
if (words[i] === "is" || words[i] === "was")
|
|
2604
|
+
i += 1;
|
|
2605
|
+
else if (words[i] === "isn't" || words[i] === "wasn't") {
|
|
2606
|
+
i += 1;
|
|
2607
|
+
negated = true;
|
|
2608
|
+
}
|
|
2609
|
+
else if (!subject)
|
|
2610
|
+
return -1;
|
|
2611
|
+
// After a subject the copula may be gone from the words: the tokenizer
|
|
2612
|
+
// reads "it's" as "it".
|
|
2613
|
+
if (!negated && (words[i] === "not" || words[i] === "never")) {
|
|
2614
|
+
i += 1;
|
|
2615
|
+
negated = true;
|
|
2616
|
+
}
|
|
2617
|
+
if (negated && TRUTH.has(words[i]))
|
|
2618
|
+
return i + 1;
|
|
2619
|
+
if (negated && words[i] === "the" && words[i + 1] === "case")
|
|
2620
|
+
return i + 2;
|
|
2621
|
+
if (!negated && FALSITY.has(words[i]))
|
|
2622
|
+
return i + 1;
|
|
2623
|
+
return -1;
|
|
2624
|
+
}
|
|
2625
|
+
/**
|
|
2626
|
+
* How many statement-level negation frames a text opens: "it is not true
|
|
2627
|
+
* that …", "it isn't the case that …", "it's not true that …", "it is false
|
|
2628
|
+
* that …", "it is incorrect that …", "it is wrong that …", "it is not
|
|
2629
|
+
* correct that …"; a "that"-clause put first and denied after it, "that no
|
|
2630
|
+
* two of the three meetings overlap is false"; and a denial heading the
|
|
2631
|
+
* text, "not true: …". Such a frame negates the whole clause it frames, yet
|
|
2632
|
+
* its negation governs "true", "case" or "false" — words no proved claim
|
|
2633
|
+
* states — so the word-by-word comparison sees no flip: "it is not true that
|
|
2634
|
+
* no two of the three meetings overlap" covers "no two of the three meetings
|
|
2635
|
+
* overlap" word for word, figures and every negation included, and states
|
|
2636
|
+
* its opposite. Counted, so that two cancel. Read off the tokens, one
|
|
2637
|
+
* bounded look ahead per word.
|
|
2638
|
+
*/
|
|
2639
|
+
function negationFrames(index) {
|
|
2640
|
+
const words = index.tokens.map((token) => token.norm);
|
|
2641
|
+
let frames = 0;
|
|
2642
|
+
// "Not true: …", "Not the case: …", "Not correct — …".
|
|
2643
|
+
if (words[0] === "not" &&
|
|
2644
|
+
(TRUTH.has(words[1]) || (words[1] === "the" && words[2] === "case"))) {
|
|
2645
|
+
frames += 1;
|
|
2646
|
+
}
|
|
2647
|
+
for (let i = 0; i < words.length; i++) {
|
|
2648
|
+
const word = words[i];
|
|
2649
|
+
if (FRAME_SUBJECTS.has(word)) {
|
|
2650
|
+
const end = framePredicate(words, i + 1, true);
|
|
2651
|
+
if (end !== -1 && words[end] === "that") {
|
|
2652
|
+
frames += 1;
|
|
2653
|
+
continue;
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
// The clause first, its denial last: "that … is false", "that … is not
|
|
2657
|
+
// the case". Only where the text opens with the complementizer, and the
|
|
2658
|
+
// predicate ends it.
|
|
2659
|
+
if (i > 1 && words[0] === "that") {
|
|
2660
|
+
const end = framePredicate(words, i, false);
|
|
2661
|
+
if (end === words.length)
|
|
2662
|
+
frames += 1;
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
return frames;
|
|
2666
|
+
}
|
|
2667
|
+
/**
|
|
2668
|
+
* Why the answer's sentence does not state what a proof's recorded claim
|
|
2669
|
+
* says, or `undefined` when it does (design §6.4): "the grounded coverage
|
|
2670
|
+
* test", read the way `grounded` reads it — the words, AND the figures and
|
|
2671
|
+
* the negations that coverage alone cannot see.
|
|
2672
|
+
*
|
|
2673
|
+
* - Words: the artifact claim's content tokens covered by the sentence's
|
|
2674
|
+
* at the grounded bar, so an accepted proof of something else cannot be
|
|
2675
|
+
* pinned to this sentence.
|
|
2676
|
+
* - Figures: the same figures on both sides (`two` and `2` are one). A
|
|
2677
|
+
* proof about meetings 1 and 2 does not prove a sentence about all six,
|
|
2678
|
+
* and `contentTokens` drops one- and two-digit numbers.
|
|
2679
|
+
* - Polarity, word by word: each word of the proof's claim (a figure by its
|
|
2680
|
+
* value) is denied on both sides or on neither, as claim-ledger's
|
|
2681
|
+
* `governingNegation` scopes a negation, wherever the sentence uses it.
|
|
2682
|
+
* `contentTokens` drops "no" (two letters) and "not" (a stopword), so
|
|
2683
|
+
* without this an accepted proof that two of the six meetings overlap
|
|
2684
|
+
* would prove "no two of the six meetings overlap" — acceptance scenario
|
|
2685
|
+
* C served false. Read per word, not per sentence, because the sentence
|
|
2686
|
+
* may say more than the proof (design §6.4): "no discount was applied, so
|
|
2687
|
+
* the invoice total is 1,284.50 EUR" states the proved total, and its
|
|
2688
|
+
* "no" denies a discount the proof never mentions. A word one side both
|
|
2689
|
+
* states and denies is not compared, and a negation that moves from one
|
|
2690
|
+
* word to another ("no two of the meetings overlap", "two of the
|
|
2691
|
+
* meetings didn't overlap") is a different statement. A partitive denial
|
|
2692
|
+
* of the whole statement ("none of the 3 invoices is overdue") is read
|
|
2693
|
+
* too ({@link partitiveDenial}), and so is a frame that negates it ("it
|
|
2694
|
+
* is not true that …", {@link negationFrames}): one side framed and the
|
|
2695
|
+
* other not states the opposite (acceptance C).
|
|
2696
|
+
*
|
|
2697
|
+
* What it cannot see is a sentence that adds a clause of its own to the
|
|
2698
|
+
* proved statement ("…, and the CEO approved it"): the coverage runs from
|
|
2699
|
+
* the proof's claim into the sentence, as the design specifies, so the
|
|
2700
|
+
* sentence may carry more words. That residual, like the formalization gap
|
|
2701
|
+
* itself, is shown beside the claim (the statement), not closed here.
|
|
2702
|
+
*/
|
|
2703
|
+
export function statementMismatch(proved, sentence, bar) {
|
|
2704
|
+
const share = coverage(contentTokens(proved), contentTokens(sentence));
|
|
2705
|
+
if (share < bar) {
|
|
2706
|
+
return `this sentence does not state it (${Math.round(share * 100)}% of its words, needs ${Math.round(bar * 100)}%)`;
|
|
2707
|
+
}
|
|
2708
|
+
const want = figureKeys(proved);
|
|
2709
|
+
const have = figureKeys(sentence);
|
|
2710
|
+
const missing = [...want].filter((key) => !have.has(key));
|
|
2711
|
+
const extra = [...have].filter((key) => !want.has(key));
|
|
2712
|
+
if (missing.length > 0 || extra.length > 0) {
|
|
2713
|
+
return `the figures differ (${[
|
|
2714
|
+
...(missing.length > 0 ? [`the proof's claim states ${missing.slice(0, 6).join(", ")}`] : []),
|
|
2715
|
+
...(extra.length > 0 ? [`this sentence states ${extra.slice(0, 6).join(", ")}`] : []),
|
|
2716
|
+
].join("; ")})`;
|
|
2717
|
+
}
|
|
2718
|
+
const provedIndex = indexSource(proved);
|
|
2719
|
+
const sentenceIndex = indexSource(sentence);
|
|
2720
|
+
const provedFramed = negationFrames(provedIndex) % 2 === 1;
|
|
2721
|
+
if (provedFramed !== (negationFrames(sentenceIndex) % 2 === 1)) {
|
|
2722
|
+
return provedFramed
|
|
2723
|
+
? "the proof's claim is negated and this sentence is not — it states the opposite"
|
|
2724
|
+
: "this sentence is negated and the proof's claim is not — it states the opposite";
|
|
2725
|
+
}
|
|
2726
|
+
const ours = polarities(provedIndex);
|
|
2727
|
+
const theirs = polarities(sentenceIndex);
|
|
2728
|
+
const words = new Set();
|
|
2729
|
+
for (const word of claimWords(proved)) {
|
|
2730
|
+
if (NEGATIONS.has(word))
|
|
2731
|
+
continue;
|
|
2732
|
+
const key = wordKey(word);
|
|
2733
|
+
words.add(key);
|
|
2734
|
+
const there = polarityOf(theirs.get(key));
|
|
2735
|
+
const here = polarityOf(ours.get(key));
|
|
2736
|
+
if (there === undefined || here === undefined || there === here)
|
|
2737
|
+
continue;
|
|
2738
|
+
return here === "denied"
|
|
2739
|
+
? `the proof's claim denies "${word}" and this sentence states it — it states the opposite`
|
|
2740
|
+
: `this sentence denies "${word}" and the proof's claim states it — it states the opposite`;
|
|
2741
|
+
}
|
|
2742
|
+
const provedDenied = partitiveDenial(provedIndex, words);
|
|
2743
|
+
if (provedDenied !== partitiveDenial(sentenceIndex, words)) {
|
|
2744
|
+
return provedDenied
|
|
2745
|
+
? "the proof's claim is negated and this sentence is not — it states the opposite"
|
|
2746
|
+
: "this sentence is negated and the proof's claim is not — it states the opposite";
|
|
2747
|
+
}
|
|
2748
|
+
return undefined;
|
|
2749
|
+
}
|
|
2750
|
+
function proofCheckInput(input) {
|
|
2751
|
+
if (typeof input !== "object" || input === null || Array.isArray(input))
|
|
2752
|
+
return undefined;
|
|
2753
|
+
const i = input;
|
|
2754
|
+
const { checker, claim, statement, proof } = i;
|
|
2755
|
+
return typeof checker === "string" &&
|
|
2756
|
+
typeof claim === "string" &&
|
|
2757
|
+
typeof statement === "string" &&
|
|
2758
|
+
typeof proof === "string"
|
|
2759
|
+
? { checker, claim, statement, proof }
|
|
2760
|
+
: undefined;
|
|
2761
|
+
}
|
|
2762
|
+
/** Was the artifact written from exactly this call's input? */
|
|
2763
|
+
const sameInput = (input, artifact) => input !== undefined &&
|
|
2764
|
+
input.checker === artifact.checker &&
|
|
2765
|
+
input.claim === artifact.claim &&
|
|
2766
|
+
input.statement === artifact.statement &&
|
|
2767
|
+
input.proof === artifact.proof;
|
|
2768
|
+
const stringList = (value) => Array.isArray(value) && value.every((item) => typeof item === "string")
|
|
2769
|
+
? value
|
|
2770
|
+
: undefined;
|
|
2771
|
+
/**
|
|
2772
|
+
* Every `ProofCheck` report in the run's evidence (in scope: under `scope:
|
|
2773
|
+
* turn`, this turn's), by the artifact id it names — read once per
|
|
2774
|
+
* verification. A report is the tool's result (tool-proof `checkReport`:
|
|
2775
|
+
* `{ ok, id, checker, …, assumptions, axioms, output }`), not an error, as
|
|
2776
|
+
* the run's log recorded it.
|
|
2777
|
+
*/
|
|
2778
|
+
function proofReports(run) {
|
|
2779
|
+
if (run.reports !== undefined)
|
|
2780
|
+
return run.reports;
|
|
2781
|
+
const out = new Map();
|
|
2782
|
+
const key = builtinKey(PROOF_CHECK);
|
|
2783
|
+
for (const record of run.evidence.all()) {
|
|
2784
|
+
if (record.isError || builtinKey(record.toolName) !== key)
|
|
2785
|
+
continue;
|
|
2786
|
+
let report;
|
|
2787
|
+
try {
|
|
2788
|
+
report = JSON.parse(record.body ?? record.content);
|
|
2789
|
+
}
|
|
2790
|
+
catch {
|
|
2791
|
+
continue;
|
|
2792
|
+
}
|
|
2793
|
+
if (typeof report !== "object" || report === null || Array.isArray(report))
|
|
2794
|
+
continue;
|
|
2795
|
+
const r = report;
|
|
2796
|
+
if (!isProofId(r["id"]) || typeof r["ok"] !== "boolean")
|
|
2797
|
+
continue;
|
|
2798
|
+
const assumptions = stringList(r["assumptions"]);
|
|
2799
|
+
const axioms = stringList(r["axioms"]);
|
|
2800
|
+
const entry = {
|
|
2801
|
+
ok: r["ok"],
|
|
2802
|
+
...(typeof r["checker"] === "string" ? { checker: r["checker"] } : {}),
|
|
2803
|
+
...(assumptions !== undefined ? { assumptions } : {}),
|
|
2804
|
+
...(axioms !== undefined ? { axioms } : {}),
|
|
2805
|
+
input: proofCheckInput(record.input),
|
|
2806
|
+
};
|
|
2807
|
+
const list = out.get(r["id"]);
|
|
2808
|
+
if (list === undefined)
|
|
2809
|
+
out.set(r["id"], [entry]);
|
|
2810
|
+
else
|
|
2811
|
+
list.push(entry);
|
|
2812
|
+
}
|
|
2813
|
+
run.reports = out;
|
|
2814
|
+
return out;
|
|
2815
|
+
}
|
|
2816
|
+
/**
|
|
2817
|
+
* Does a list a `ProofCheck` report shows agree with the artifact's? The
|
|
2818
|
+
* report lists each item clipped (a trailing `…`) and, when they do not all
|
|
2819
|
+
* fit, ends with a `… and N more` line (tool-proof `listed`); the artifact
|
|
2820
|
+
* keeps every item whole. So each shown item is the stored item at its
|
|
2821
|
+
* place, or a clipped start of it, and the stored list is exactly as long —
|
|
2822
|
+
* or longer, where the report says more were left out.
|
|
2823
|
+
*/
|
|
2824
|
+
function listAgrees(shown, stored) {
|
|
2825
|
+
if (shown === undefined)
|
|
2826
|
+
return true;
|
|
2827
|
+
const more = shown.length > 0 && shown[shown.length - 1].startsWith("… and ");
|
|
2828
|
+
const items = more ? shown.slice(0, -1) : shown;
|
|
2829
|
+
if (more ? stored.length <= items.length : stored.length !== items.length)
|
|
2830
|
+
return false;
|
|
2831
|
+
return items.every((item, i) => {
|
|
2832
|
+
const whole = stored[i];
|
|
2833
|
+
return item === whole || (item.endsWith("…") && whole.startsWith(item.slice(0, -1)));
|
|
2834
|
+
});
|
|
2835
|
+
}
|
|
2836
|
+
/** The artifact re-checked once per verification (`VerifyOptions.recheckProof`); a throw reads as not re-checked. */
|
|
2837
|
+
function recheckFor(run, artifact) {
|
|
2838
|
+
let pending = run.rechecks.get(artifact.id);
|
|
2839
|
+
if (pending === undefined) {
|
|
2840
|
+
const recheck = run.opts.recheckProof;
|
|
2841
|
+
pending = (async () => {
|
|
2842
|
+
try {
|
|
2843
|
+
const again = await recheck(artifact);
|
|
2844
|
+
return again !== undefined && typeof again.ok === "boolean" ? again : undefined;
|
|
2845
|
+
}
|
|
2846
|
+
catch {
|
|
2847
|
+
return undefined;
|
|
2848
|
+
}
|
|
2849
|
+
})();
|
|
2850
|
+
run.rechecks.set(artifact.id, pending);
|
|
2851
|
+
}
|
|
2852
|
+
return pending;
|
|
2853
|
+
}
|
|
2854
|
+
const hypothesisWords = (n) => `${n} ${n === 1 ? "hypothesis" : "hypotheses"}`;
|
|
2855
|
+
/**
|
|
2856
|
+
* A `proof:prf_…` binding (design §6.4): the artifact a registered checker
|
|
2857
|
+
* accepted, whose recorded claim this sentence states.
|
|
2858
|
+
*
|
|
2859
|
+
* "A registered checker accepted it" is read off the RUN, not off the
|
|
2860
|
+
* artifact file alone. The file sits in the workspace, where the model's
|
|
2861
|
+
* own file tools can write one by hand or flip a rejected one to `ok:
|
|
2862
|
+
* true` — so the artifact is accepted only when
|
|
2863
|
+
*
|
|
2864
|
+
* - its checker is on the resolved `formal.checkers`;
|
|
2865
|
+
* - the run's evidence holds the `ProofCheck` call that wrote it (in
|
|
2866
|
+
* scope: a proof checked in an earlier turn does not back a claim under
|
|
2867
|
+
* `scope: turn`), not an error, reporting this id as accepted by that
|
|
2868
|
+
* checker, with the assumptions and axioms the artifact lists
|
|
2869
|
+
* (`listAgrees`), and given exactly the checker, claim, statement and
|
|
2870
|
+
* proof the artifact holds (`sameInput`: a file rewritten after the check
|
|
2871
|
+
* keeps its id and verdict, never its input) — `trustStoredProofs` alone
|
|
2872
|
+
* skips this, for an offline check with no log;
|
|
2873
|
+
* - where a re-check is wired (`recheckProof`: the builtin checker, pure
|
|
2874
|
+
* and bounded), the checker accepts the stored statement and proof
|
|
2875
|
+
* again.
|
|
2876
|
+
*
|
|
2877
|
+
* The hypotheses the proof rests on (`assumptions`: builtin `given`
|
|
2878
|
+
* steps, SMT asserts and definitions) are listed on the binding beside the
|
|
2879
|
+
* statement and, design §6.4, are assumptions: under `assumptions: forbid`
|
|
2880
|
+
* a proof that rests on any is refused (`assumption_forbidden`), and
|
|
2881
|
+
* `max_assumptions` counts them (`verifyDocument`). A proof that assumes
|
|
2882
|
+
* its own conclusion is otherwise `proven` with "0 assumptions".
|
|
2883
|
+
*/
|
|
2884
|
+
async function formalBinding(run, claim, proofId) {
|
|
2885
|
+
const base = { origin: proofId };
|
|
2886
|
+
if (!isProofId(proofId)) {
|
|
2887
|
+
return fail("none", "proof_missing", `${clip(proofId, 40)} is not a proof id (prf_<16 hex>)`, base);
|
|
2888
|
+
}
|
|
2889
|
+
if (run.opts.artifacts === undefined) {
|
|
2890
|
+
return fail("none", "proof_missing", "no proof-artifact store is wired here", base);
|
|
2891
|
+
}
|
|
2892
|
+
let artifact;
|
|
2893
|
+
try {
|
|
2894
|
+
artifact = await artifactFor(run, proofId);
|
|
2895
|
+
}
|
|
2896
|
+
catch (err) {
|
|
2897
|
+
return fail("none", "proof_missing", `${proofId} could not be read: ${clip(err instanceof Error ? err.message : String(err), 160)}`, base);
|
|
2898
|
+
}
|
|
2899
|
+
if (artifact === undefined) {
|
|
2900
|
+
return fail("none", "proof_missing", `no proof artifact ${proofId} — run ProofCheck first`, base);
|
|
2901
|
+
}
|
|
2902
|
+
if (!isProofArtifact(artifact)) {
|
|
2903
|
+
return fail("none", "proof_missing", `the artifact stored as ${proofId} is malformed`, base);
|
|
2904
|
+
}
|
|
2905
|
+
if (artifact.id !== proofId) {
|
|
2906
|
+
return fail("none", "proof_missing", `the artifact stored as ${proofId} names another id`, base);
|
|
2907
|
+
}
|
|
2908
|
+
const listed = (texts) => [...new Set(texts)].map((text) => clip(text, MAX_STATEMENT_CHARS));
|
|
2909
|
+
const shownWith = (held) => ({
|
|
2910
|
+
...base,
|
|
2911
|
+
sha256: artifact.outputDigest,
|
|
2912
|
+
checker: artifact.checker,
|
|
2913
|
+
statement: clip(artifact.statement, MAX_STATEMENT_CHARS),
|
|
2914
|
+
...(held.length > 0 ? { assumptions: held } : {}),
|
|
2915
|
+
});
|
|
2916
|
+
const hypotheses = listed(artifact.assumptions);
|
|
2917
|
+
const shown = shownWith(hypotheses);
|
|
2918
|
+
const checkers = run.config.formal.checkers;
|
|
2919
|
+
if (!checkers.includes(artifact.checker)) {
|
|
2920
|
+
return fail("none", "proof_missing", `${proofId} was checked by ${clip(artifact.checker, 40)}, which is not on provability.formal.checkers (${checkers.join(", ") || "none"})`, shown);
|
|
2921
|
+
}
|
|
2922
|
+
if (!artifact.ok) {
|
|
2923
|
+
const why = [
|
|
2924
|
+
artifact.reason !== undefined ? clip(artifact.reason, 200) : undefined,
|
|
2925
|
+
artifact.counterexample !== undefined
|
|
2926
|
+
? `counterexample: ${clip(artifact.counterexample, 300)}`
|
|
2927
|
+
: undefined,
|
|
2928
|
+
].filter((part) => part !== undefined);
|
|
2929
|
+
return fail("none", "proof_rejected", `the ${artifact.checker} checker rejected ${proofId}${why.length > 0 ? ` — ${why.join("; ")}` : ""}`, shown);
|
|
2930
|
+
}
|
|
2931
|
+
if (run.opts.trustStoredProofs !== true) {
|
|
2932
|
+
const reports = proofReports(run).get(proofId) ?? [];
|
|
2933
|
+
if (reports.length === 0) {
|
|
2934
|
+
return fail("none", "proof_missing", `no ProofCheck call in this run's evidence${run.opts.turnNumbers !== undefined ? " (this turn)" : ""} wrote ${proofId} — run ProofCheck and cite the id it returns`, shown);
|
|
2935
|
+
}
|
|
2936
|
+
if (!reports.some((report) => report.ok)) {
|
|
2937
|
+
return fail("none", "proof_rejected", `the ProofCheck call that wrote ${proofId} reported it rejected; the artifact stored under that id says otherwise`, shown);
|
|
2938
|
+
}
|
|
2939
|
+
const accepted = reports.filter((report) => report.ok &&
|
|
2940
|
+
(report.checker === undefined || report.checker === artifact.checker) &&
|
|
2941
|
+
listAgrees(report.assumptions, artifact.assumptions) &&
|
|
2942
|
+
listAgrees(report.axioms, artifact.axioms));
|
|
2943
|
+
if (accepted.length === 0) {
|
|
2944
|
+
return fail("none", "proof_missing", `the artifact stored as ${proofId} is not the one ProofCheck reported in this run (its checker, assumptions or axioms differ)`, shown);
|
|
2945
|
+
}
|
|
2946
|
+
// The report names the id; the call's input is what was checked. An
|
|
2947
|
+
// artifact whose claim, statement or proof was rewritten after the check
|
|
2948
|
+
// keeps the id and the verdict, so it is matched against the input too.
|
|
2949
|
+
if (!accepted.some((report) => sameInput(report.input, artifact))) {
|
|
2950
|
+
return fail("none", "proof_missing", `the artifact stored as ${proofId} is not the one ProofCheck checked in this run (its checker, claim, statement or proof differ from that call's input)`, shown);
|
|
2951
|
+
}
|
|
2952
|
+
}
|
|
2953
|
+
let held = hypotheses;
|
|
2954
|
+
let axioms = artifact.axioms;
|
|
2955
|
+
if (run.opts.recheckProof !== undefined) {
|
|
2956
|
+
const again = await recheckFor(run, artifact);
|
|
2957
|
+
if (again !== undefined && !again.ok) {
|
|
2958
|
+
return fail("none", "proof_rejected", `the ${artifact.checker} checker, run again on the statement and proof stored as ${proofId}, does not accept them`, shown);
|
|
2959
|
+
}
|
|
2960
|
+
if (again?.assumptions !== undefined)
|
|
2961
|
+
held = listed(again.assumptions);
|
|
2962
|
+
if (again?.axioms !== undefined)
|
|
2963
|
+
axioms = again.axioms;
|
|
2964
|
+
}
|
|
2965
|
+
const proved = shownWith(held);
|
|
2966
|
+
const mismatch = statementMismatch(artifact.claim, claim.text, run.config.grounded.coverage);
|
|
2967
|
+
if (mismatch !== undefined) {
|
|
2968
|
+
return fail("none", "claim_mismatch", `${proofId} proves "${clip(artifact.claim, 160)}"; ${mismatch}`, proved);
|
|
2969
|
+
}
|
|
2970
|
+
if (held.length > 0 && run.config.assumptions === "forbid") {
|
|
2971
|
+
return fail("none", "assumption_forbidden", `${proofId} rests on ${hypothesisWords(held.length)} (${clip(held.join("; "), 200)}) and provability.assumptions is forbid — prove it from none, or drop the claim`, proved);
|
|
2972
|
+
}
|
|
2973
|
+
const key = `${artifact.checker}\u0000${proofId}`;
|
|
2974
|
+
if (!run.checkers.has(key)) {
|
|
2975
|
+
run.checkers.set(key, {
|
|
2976
|
+
id: artifact.checker,
|
|
2977
|
+
artifact: proofId,
|
|
2978
|
+
axioms: [...axioms],
|
|
2979
|
+
...(held.length > 0 ? { assumptions: [...held] } : {}),
|
|
2980
|
+
});
|
|
2981
|
+
}
|
|
2982
|
+
return { rung: "proven", ...proved };
|
|
2983
|
+
}
|
|
2984
|
+
// ---------------------------------------------------------------------------
|
|
2985
|
+
// claims
|
|
2986
|
+
// ---------------------------------------------------------------------------
|
|
2987
|
+
/** What may open an evidence id the model dressed, and what may close one. */
|
|
2988
|
+
const ID_OPENERS = new Set(["`", "'", '"', "<", "("]);
|
|
2989
|
+
const ID_CLOSERS = new Set(["`", "'", '"', ">", ")", ".", ",", ";", ":", "!", "?"]);
|
|
2990
|
+
/**
|
|
2991
|
+
* An evidence id as the model may have dressed it — `` tool:`toolu_01` ``,
|
|
2992
|
+
* `proof:prf_….` at the end of a sentence — down to the id. Ids are
|
|
2993
|
+
* `[A-Za-z0-9_-]` runs, so wrapping quotes, backticks and angle brackets and
|
|
2994
|
+
* trailing sentence punctuation are never part of one. Trimmed by walking
|
|
2995
|
+
* in from each end: an end-anchored pattern (`/[…]+$/`) restarts at every
|
|
2996
|
+
* character of a long run of them that does not end the id — quadratic in
|
|
2997
|
+
* text the model writes, six seconds for 80 000 dots.
|
|
2998
|
+
*/
|
|
2999
|
+
export function bareId(id) {
|
|
3000
|
+
let from = 0;
|
|
3001
|
+
let to = id.length;
|
|
3002
|
+
while (from < to && ID_OPENERS.has(id[from]))
|
|
3003
|
+
from += 1;
|
|
3004
|
+
while (to > from && ID_CLOSERS.has(id[to - 1]))
|
|
3005
|
+
to -= 1;
|
|
3006
|
+
return id.slice(from, to);
|
|
3007
|
+
}
|
|
3008
|
+
/** The marker as written: `[^a]` when the claim spelled it as a footnote. */
|
|
3009
|
+
function refOf(id, raw) {
|
|
3010
|
+
return raw.includes(`[^${id}]`) ? `[^${id}]` : `[${id}]`;
|
|
3011
|
+
}
|
|
3012
|
+
async function bindingFor(run, claim, id, definitions) {
|
|
3013
|
+
const ref = refOf(id, claim.raw);
|
|
3014
|
+
const source = definitions.get(id);
|
|
3015
|
+
if (source === undefined) {
|
|
3016
|
+
return {
|
|
3017
|
+
binding: finish(ref, "source", fail("none", "no_definition", `${ref} has no "${ref}: …" definition line`), run.floor),
|
|
3018
|
+
};
|
|
3019
|
+
}
|
|
3020
|
+
const evidence = classifyDefinition(source);
|
|
3021
|
+
switch (evidence.kind) {
|
|
3022
|
+
case "assume":
|
|
3023
|
+
if (run.config.assumptions === "forbid") {
|
|
3024
|
+
return {
|
|
3025
|
+
binding: finish(ref, "assumption", fail("none", "assumption_forbidden", "provability.assumptions is forbid", {
|
|
3026
|
+
origin: clip(evidence.text, MAX_CLAIM_CHARS),
|
|
3027
|
+
}), run.floor),
|
|
3028
|
+
};
|
|
3029
|
+
}
|
|
3030
|
+
// Not evidence and not a failure: the claim's status says what it is.
|
|
3031
|
+
return {
|
|
3032
|
+
binding: {
|
|
3033
|
+
ref,
|
|
3034
|
+
kind: "assumption",
|
|
3035
|
+
rung: "none",
|
|
3036
|
+
ok: false,
|
|
3037
|
+
origin: clip(evidence.text, MAX_CLAIM_CHARS),
|
|
3038
|
+
},
|
|
3039
|
+
};
|
|
3040
|
+
case "tool":
|
|
3041
|
+
return {
|
|
3042
|
+
binding: finish(ref, "execution", await executionBinding(run, claim, bareId(evidence.toolUseId)), run.floor),
|
|
3043
|
+
};
|
|
3044
|
+
case "proof":
|
|
3045
|
+
return {
|
|
3046
|
+
binding: finish(ref, "formal", await formalBinding(run, claim, bareId(evidence.proofId)), run.floor),
|
|
3047
|
+
};
|
|
3048
|
+
default: {
|
|
3049
|
+
const outcome = await sourceBinding(run, claim, evidence);
|
|
3050
|
+
return {
|
|
3051
|
+
binding: finish(ref, "source", outcome.reached, run.floor),
|
|
3052
|
+
...(outcome.waivable !== undefined ? { waivable: outcome.waivable } : {}),
|
|
3053
|
+
};
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
3056
|
+
}
|
|
3057
|
+
/**
|
|
3058
|
+
* What `grounded.independent` counts as one origin: the registrable domain
|
|
3059
|
+
* of a URL (`docs.acme.example` and `www.acme.example` are one publisher), a
|
|
3060
|
+
* DOI's registrant prefix (`10.1000` — one publisher's papers are one
|
|
3061
|
+
* origin, like one site's pages), and a workspace file's own path.
|
|
3062
|
+
*/
|
|
3063
|
+
export function independenceKey(origin) {
|
|
3064
|
+
const host = /^https?:\/\//i.test(origin) ? hostOf(origin) : undefined;
|
|
3065
|
+
if (host === undefined)
|
|
3066
|
+
return `path:${origin}`;
|
|
3067
|
+
if (host === "doi.org") {
|
|
3068
|
+
const prefix = /^https:\/\/doi\.org\/([^/]+)/.exec(origin)?.[1];
|
|
3069
|
+
return `doi:${prefix ?? origin}`;
|
|
3070
|
+
}
|
|
3071
|
+
return `host:${registrableDomain(host)}`;
|
|
3072
|
+
}
|
|
3073
|
+
/**
|
|
3074
|
+
* Distinct independence keys among a claim's source bindings that reached
|
|
3075
|
+
* `grounded`: each by the origin that served the body it grounded in
|
|
3076
|
+
* ({@link bodyKey}), and by its cited origin only where no body is known.
|
|
3077
|
+
*/
|
|
3078
|
+
function groundedOrigins(run, bindings) {
|
|
3079
|
+
const keys = new Set();
|
|
3080
|
+
for (const b of bindings) {
|
|
3081
|
+
if (b.kind === "source" &&
|
|
3082
|
+
levelRank(b.rung) >= levelRank("grounded") &&
|
|
3083
|
+
b.origin !== undefined) {
|
|
3084
|
+
keys.add((b.sha256 !== undefined ? run.bodyKeys.get(groundedAs(b.sha256, b.origin)) : undefined) ??
|
|
3085
|
+
independenceKey(b.origin));
|
|
3086
|
+
}
|
|
3087
|
+
}
|
|
3088
|
+
return keys;
|
|
3089
|
+
}
|
|
3090
|
+
/**
|
|
3091
|
+
* `grounded.independent: n` at a `grounded` floor: a claim met only by
|
|
3092
|
+
* source bindings needs n distinct origins among them. One met by an
|
|
3093
|
+
* execution or formal binding (a stronger rung) is met regardless. When it
|
|
3094
|
+
* falls short its bindings keep their rungs, and every source binding that
|
|
3095
|
+
* was `ok` becomes not ok with `not_independent` — the best one first in
|
|
3096
|
+
* document order — so `statusOf`'s "met when some binding is ok" still
|
|
3097
|
+
* decides it.
|
|
3098
|
+
*/
|
|
3099
|
+
function applyIndependence(run, bindings) {
|
|
3100
|
+
const n = run.config.grounded.independent;
|
|
3101
|
+
if (run.floor !== "grounded" || n <= 1)
|
|
3102
|
+
return bindings;
|
|
3103
|
+
if (bindings.some((b) => b.ok && b.kind !== "source"))
|
|
3104
|
+
return bindings;
|
|
3105
|
+
const okSources = bindings.filter((b) => b.ok && b.kind === "source");
|
|
3106
|
+
if (okSources.length === 0)
|
|
3107
|
+
return bindings;
|
|
3108
|
+
const keys = groundedOrigins(run, bindings);
|
|
3109
|
+
if (keys.size >= n)
|
|
3110
|
+
return bindings;
|
|
3111
|
+
const shown = [...keys].map((key) => key.replace(/^[a-z]+:/, "")).join(", ");
|
|
3112
|
+
const detail = `needs ${n} independent origins; this claim grounds in ${keys.size} (${shown})`;
|
|
3113
|
+
return bindings.map((b) => b.ok && b.kind === "source"
|
|
3114
|
+
? { ...b, ok: false, reason: "not_independent", detail }
|
|
3115
|
+
: b);
|
|
3116
|
+
}
|
|
3117
|
+
/**
|
|
3118
|
+
* The waiver judge (design §6.2), asked once a claim's deterministic
|
|
3119
|
+
* bindings are settled and only while the claim still needs it: at
|
|
3120
|
+
* `independent: 1`, only when no binding meets the floor — a claim another
|
|
3121
|
+
* binding already meets cannot be changed by a waiver, so asking would
|
|
3122
|
+
* spend `judge_share` (§18) and print "1 judge-waived" beside an answer
|
|
3123
|
+
* that passed deterministically, and draw down the share a claim that does
|
|
3124
|
+
* need the judge would use. At `independent: n > 1`, only for an origin the
|
|
3125
|
+
* claim does not already count, and only while it is short of n with
|
|
3126
|
+
* enough waivable origins to make up the count. In marker order, until the
|
|
3127
|
+
* claim is met.
|
|
3128
|
+
*/
|
|
3129
|
+
async function waive(run, bound) {
|
|
3130
|
+
const bindings = bound.map((b) => b.binding);
|
|
3131
|
+
if (!bound.some((b) => b.waivable !== undefined))
|
|
3132
|
+
return bindings;
|
|
3133
|
+
// A dangling marker unmeets the claim whatever is waived (`statusOf`).
|
|
3134
|
+
if (bindings.some((b) => b.reason === "no_definition"))
|
|
3135
|
+
return bindings;
|
|
3136
|
+
const n = run.config.grounded.independent;
|
|
3137
|
+
const needed = () => {
|
|
3138
|
+
if (n <= 1)
|
|
3139
|
+
return !bindings.some((b) => b.ok);
|
|
3140
|
+
if (bindings.some((b) => b.ok && b.kind !== "source"))
|
|
3141
|
+
return false;
|
|
3142
|
+
return groundedOrigins(run, bindings).size < n;
|
|
3143
|
+
};
|
|
3144
|
+
if (n > 1) {
|
|
3145
|
+
const could = groundedOrigins(run, bindings);
|
|
3146
|
+
for (const b of bound) {
|
|
3147
|
+
if (b.waivable !== undefined)
|
|
3148
|
+
could.add(b.waivable.key);
|
|
3149
|
+
}
|
|
3150
|
+
if (could.size < n)
|
|
3151
|
+
return bindings;
|
|
3152
|
+
}
|
|
3153
|
+
for (const [i, b] of bound.entries()) {
|
|
3154
|
+
if (b.waivable === undefined)
|
|
3155
|
+
continue;
|
|
3156
|
+
if (!needed())
|
|
3157
|
+
break;
|
|
3158
|
+
if (n > 1 && groundedOrigins(run, bindings).has(b.waivable.key))
|
|
3159
|
+
continue;
|
|
3160
|
+
const waived = await askJudge(run, b.waivable);
|
|
3161
|
+
if (waived !== undefined)
|
|
3162
|
+
bindings[i] = finish(b.binding.ref, "source", waived, run.floor);
|
|
3163
|
+
}
|
|
3164
|
+
return bindings;
|
|
3165
|
+
}
|
|
3166
|
+
/**
|
|
3167
|
+
* A claim's status. `met` iff some binding is `ok` — and no marker in it is
|
|
3168
|
+
* dangling: design §2/§6.1's `cited` rung is that EVERY marker in a claim
|
|
3169
|
+
* sentence has a definition, so "14 days [1] [2]" with no `[2]:` line puts
|
|
3170
|
+
* a citation of nothing in front of the reader, however good `[1]` is. (A
|
|
3171
|
+
* defined marker that fails — a second source never fetched — does not
|
|
3172
|
+
* unmeet a claim another binding meets; the bundle and the report's table
|
|
3173
|
+
* still show it.) `assumption` when every binding is an allowed assumption.
|
|
3174
|
+
*/
|
|
3175
|
+
function statusOf(bindings) {
|
|
3176
|
+
if (bindings.some((b) => b.reason === "no_definition"))
|
|
3177
|
+
return "unmet";
|
|
3178
|
+
if (bindings.some((b) => b.ok))
|
|
3179
|
+
return "met";
|
|
3180
|
+
if (bindings.length > 0 &&
|
|
3181
|
+
bindings.every((b) => b.kind === "assumption" && b.reason === undefined)) {
|
|
3182
|
+
return "assumption";
|
|
3183
|
+
}
|
|
3184
|
+
return "unmet";
|
|
3185
|
+
}
|
|
3186
|
+
/**
|
|
3187
|
+
* Is the claim unmet for its assumptions alone — every marker an `assume:`
|
|
3188
|
+
* definition (allowed or refused), or a proof refused for the hypotheses
|
|
3189
|
+
* it rests on (design §6.4: those are assumptions too)?
|
|
3190
|
+
*/
|
|
3191
|
+
function refusedForAssumptions(claim) {
|
|
3192
|
+
return (claim.status === "unmet" &&
|
|
3193
|
+
claim.bindings.length > 0 &&
|
|
3194
|
+
claim.bindings.every((b) => b.kind === "assumption" || (b.kind === "formal" && b.reason === "assumption_forbidden")));
|
|
3195
|
+
}
|
|
3196
|
+
/**
|
|
3197
|
+
* The highest floor one claim would meet: its best evidence rung, with
|
|
3198
|
+
* `grounded.independent` applied at `grounded` (a claim grounded in too few
|
|
3199
|
+
* origins would meet `cited`, not `grounded`), and none at all with a
|
|
3200
|
+
* dangling marker in it (see {@link statusOf}).
|
|
3201
|
+
*/
|
|
3202
|
+
function claimCeiling(run, claim) {
|
|
3203
|
+
if (claim.bindings.some((b) => b.reason === "no_definition"))
|
|
3204
|
+
return "none";
|
|
3205
|
+
const evidence = claim.bindings.filter((b) => b.kind !== "assumption");
|
|
3206
|
+
const best = maxRung(...evidence.map((b) => b.rung));
|
|
3207
|
+
const n = run.config.grounded.independent;
|
|
3208
|
+
if (n > 1 && best === "grounded" && groundedOrigins(run, evidence).size < n)
|
|
3209
|
+
return "cited";
|
|
3210
|
+
return best;
|
|
3211
|
+
}
|
|
3212
|
+
/** One claim bound and settled: every marker's binding, the waiver, `grounded.independent`. */
|
|
3213
|
+
async function verifyClaim(run, { claim, definition }, definitions) {
|
|
3214
|
+
// Long answers over large bodies are real work: let timers, aborts and
|
|
3215
|
+
// the process's other sessions run between claims.
|
|
3216
|
+
if (performance.now() - run.lastYield >= YIELD_EVERY_MS) {
|
|
3217
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
3218
|
+
run.lastYield = performance.now();
|
|
3219
|
+
}
|
|
3220
|
+
const bound = [];
|
|
3221
|
+
for (const id of claim.markers)
|
|
3222
|
+
bound.push(await bindingFor(run, claim, id, definitions));
|
|
3223
|
+
const settled = applyIndependence(run, await waive(run, bound));
|
|
3224
|
+
return {
|
|
3225
|
+
index: claim.index,
|
|
3226
|
+
text: clip(claim.text, MAX_CLAIM_CHARS),
|
|
3227
|
+
...(claim.line > 0 ? { line: claim.line } : {}),
|
|
3228
|
+
status: statusOf(settled),
|
|
3229
|
+
best: maxRung(...settled.map((b) => b.rung)),
|
|
3230
|
+
bindings: settled,
|
|
3231
|
+
...(definition !== undefined ? { definition } : {}),
|
|
3232
|
+
};
|
|
3233
|
+
}
|
|
3234
|
+
/**
|
|
3235
|
+
* Claims stated INSIDE definition lines. claim-ledger's extractor leaves
|
|
3236
|
+
* out the definition `citationDefinitions` keeps for each id — but `[8]:
|
|
3237
|
+
* https://acme.example/x Payments clear within 5 days` is not a valid link
|
|
3238
|
+
* reference definition, so it renders as that sentence, and the gate would
|
|
3239
|
+
* otherwise put an assertion in front of the reader that it never read. So
|
|
3240
|
+
* the text such a definition shows beyond what it names (claim-ledger's
|
|
3241
|
+
* `definitionGloss`) is read the way the answer is — by `extractClaims`,
|
|
3242
|
+
* which decides what a claim is — and each claim in it cites that
|
|
3243
|
+
* definition: checked against what it names, like any other. Every other
|
|
3244
|
+
* definition-shaped line that shows text (a second `[8]:` line, a bullet
|
|
3245
|
+
* `- [8]: …`) the extractor reads as prose itself, uncited.
|
|
3246
|
+
*
|
|
3247
|
+
* One rule for every definition, cited or not, whatever its kind and
|
|
3248
|
+
* whether or not the text ends like a sentence; a cited definition's
|
|
3249
|
+
* trailing sentence is as visible as an uncited one's. A short gloss
|
|
3250
|
+
* (`— Acme refunds`: under three content words, no figure) is not a claim,
|
|
3251
|
+
* and a title in quotes (`[1]: https://… "Acme refund policy"`) is not
|
|
3252
|
+
* shown at all. A reference entry with bibliographic text after its URL or
|
|
3253
|
+
* DOI (`… Smith, J. (2020). Refunds in retail. 12(3), 45-67.`) IS read: it
|
|
3254
|
+
* describes its source, and at `grounded` the run must have read that
|
|
3255
|
+
* source for the description to stand — the retry nudge says how to keep a
|
|
3256
|
+
* title out. `sources.bibliographic` spares only a free-text source
|
|
3257
|
+
* itself, which is then the reference (see `definitionGloss`).
|
|
3258
|
+
*/
|
|
3259
|
+
function definitionClaims(text, definitions, bibliographic, firstIndex) {
|
|
3260
|
+
const out = [];
|
|
3261
|
+
for (const [id, source] of definitions) {
|
|
3262
|
+
const gloss = definitionGloss(source, classifyDefinition(source), bibliographic);
|
|
3263
|
+
if (gloss === "")
|
|
3264
|
+
continue;
|
|
3265
|
+
// The marker as the definition spelled it, so a footnote keeps its caret.
|
|
3266
|
+
const definition = text.includes(`[^${id}]:`) ? `[^${id}]` : `[${id}]`;
|
|
3267
|
+
for (const claim of extractClaims(gloss)) {
|
|
3268
|
+
out.push({
|
|
3269
|
+
claim: {
|
|
3270
|
+
index: firstIndex + out.length,
|
|
3271
|
+
text: claim.text,
|
|
3272
|
+
// Read as if the sentence carried the marker it sits under.
|
|
3273
|
+
raw: `${claim.raw} ${definition}`,
|
|
3274
|
+
line: 0,
|
|
3275
|
+
markers: [id],
|
|
3276
|
+
},
|
|
3277
|
+
definition,
|
|
3278
|
+
});
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
return out;
|
|
3282
|
+
}
|
|
3283
|
+
// ---------------------------------------------------------------------------
|
|
3284
|
+
// the bundle
|
|
3285
|
+
// ---------------------------------------------------------------------------
|
|
3286
|
+
/** Keys sorted at every depth; `undefined` dropped, as JSON drops it. */
|
|
3287
|
+
function canonical(value) {
|
|
3288
|
+
if (Array.isArray(value))
|
|
3289
|
+
return value.map(canonical);
|
|
3290
|
+
if (value !== null && typeof value === "object") {
|
|
3291
|
+
const out = {};
|
|
3292
|
+
for (const key of Object.keys(value).sort()) {
|
|
3293
|
+
const v = value[key];
|
|
3294
|
+
if (v !== undefined)
|
|
3295
|
+
out[key] = canonical(v);
|
|
3296
|
+
}
|
|
3297
|
+
return out;
|
|
3298
|
+
}
|
|
3299
|
+
return value;
|
|
3300
|
+
}
|
|
3301
|
+
/**
|
|
3302
|
+
* `sha256:<hex>` over the canonical JSON (keys sorted recursively) of every
|
|
3303
|
+
* field of the bundle except `bundleHash` itself — what the audit record
|
|
3304
|
+
* and the trace line carry, so a stored bundle can be shown to be the one
|
|
3305
|
+
* that was graded.
|
|
3306
|
+
*/
|
|
3307
|
+
export function bundleHash(bundle) {
|
|
3308
|
+
const { bundleHash: _ignored, ...rest } = bundle;
|
|
3309
|
+
return sha256Of(JSON.stringify(canonical(rest)));
|
|
3310
|
+
}
|
|
3311
|
+
function nowOf(now) {
|
|
3312
|
+
if (now === undefined)
|
|
3313
|
+
return new Date();
|
|
3314
|
+
return typeof now === "function" ? now() : now;
|
|
3315
|
+
}
|
|
3316
|
+
/**
|
|
3317
|
+
* The bundle for a verifier that THREW (design §5.1): grading
|
|
3318
|
+
* infrastructure failed, the gate fails open, and the bundle says so rather
|
|
3319
|
+
* than printing a ✓ — `verdict: "unverified"`, no claims, the error clipped.
|
|
3320
|
+
* Only the floor and the threshold are read, so the runtime gate — which
|
|
3321
|
+
* holds `RunProvability`, not the lowered IR block — can build one too.
|
|
3322
|
+
*/
|
|
3323
|
+
export function unverifiedBundle(config, error, retryIndex = 0, now) {
|
|
3324
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3325
|
+
const core = {
|
|
3326
|
+
version: 1,
|
|
3327
|
+
level: config.level,
|
|
3328
|
+
verdict: "unverified",
|
|
3329
|
+
score: 0,
|
|
3330
|
+
threshold: config.threshold,
|
|
3331
|
+
achieved: "none",
|
|
3332
|
+
claims: [],
|
|
3333
|
+
claimCount: 0,
|
|
3334
|
+
met: 0,
|
|
3335
|
+
unmet: 0,
|
|
3336
|
+
assumptions: 0,
|
|
3337
|
+
sources: 0,
|
|
3338
|
+
judgeWaived: 0,
|
|
3339
|
+
retryIndex,
|
|
3340
|
+
error: clip(message === "" ? "the verifier failed" : message, 500),
|
|
3341
|
+
createdAt: nowOf(now).toISOString(),
|
|
3342
|
+
};
|
|
3343
|
+
return { ...core, bundleHash: bundleHash(core) };
|
|
3344
|
+
}
|
|
3345
|
+
/**
|
|
3346
|
+
* Citations the up-front HEAD collection in {@link verifyDocument} has
|
|
3347
|
+
* examined: for a test to count with, as claim-ledger's `scanMeter` is.
|
|
3348
|
+
* Only ever added to.
|
|
3349
|
+
*/
|
|
3350
|
+
export const headCollectMeter = { markers: 0 };
|
|
3351
|
+
/**
|
|
3352
|
+
* Verify every claim of `text` against `opts.evidence` at `opts.config`'s
|
|
3353
|
+
* floor, and return the proof bundle (design §5.3).
|
|
3354
|
+
*
|
|
3355
|
+
* - Claims are claim-ledger's `extractClaims` — plus the text any
|
|
3356
|
+
* definition line shows beyond what it names, read as claims citing that
|
|
3357
|
+
* definition (see `definitionClaims`); a claim with no marker is unmet
|
|
3358
|
+
* with zero bindings (`no_citation`), one with a dangling marker is unmet
|
|
3359
|
+
* (`no_definition`).
|
|
3360
|
+
* - Each marker binds to its definition, classified by claim-ledger's
|
|
3361
|
+
* evidence-ref grammar: a source (cited → grounded), `tool:` (checked),
|
|
3362
|
+
* `proof:` (proven) or `assume:` (an assumption).
|
|
3363
|
+
* - A binding records the highest rung it reached and, when that is below
|
|
3364
|
+
* the floor, the FIRST check it failed. Rungs above the floor are still
|
|
3365
|
+
* computed — `achieved` is "the highest floor this answer would have
|
|
3366
|
+
* passed" — but the three outside calls are made only where the floor
|
|
3367
|
+
* needs them: `resolveHead` at `cited`/`grounded` (no source can meet a
|
|
3368
|
+
* higher floor), `judgeWaiver` at `grounded` (for a coverage-only
|
|
3369
|
+
* paraphrase miss), `replay` at `checked`. So `achieved` above the floor
|
|
3370
|
+
* is computed without them.
|
|
3371
|
+
* - `scope: turn` (`turnNumbers`) restricts the evidence to those turns.
|
|
3372
|
+
*
|
|
3373
|
+
* Never throws for anything a document or its evidence can contain — an
|
|
3374
|
+
* unreadable artifact, a failing judge or replay, a refused HEAD are each a
|
|
3375
|
+
* binding's reason. Throws only for a malformed config (a programmer error).
|
|
3376
|
+
*/
|
|
3377
|
+
export async function verifyDocument(text, opts) {
|
|
3378
|
+
const { config } = opts;
|
|
3379
|
+
if (!isProvabilityLevel(config?.level)) {
|
|
3380
|
+
throw new ProvabilityError("config", `provability: config.level must be one of ${PROVABILITY_LEVELS.join(", ")}`);
|
|
3381
|
+
}
|
|
3382
|
+
const workspaceRoot = path.resolve(opts.workspaceRoot ?? process.cwd());
|
|
3383
|
+
const run = {
|
|
3384
|
+
config,
|
|
3385
|
+
floor: config.level,
|
|
3386
|
+
opts,
|
|
3387
|
+
evidence: opts.turnNumbers !== undefined
|
|
3388
|
+
? restrictToTurns(opts.evidence, opts.turnNumbers)
|
|
3389
|
+
: opts.evidence,
|
|
3390
|
+
workspaceRoot,
|
|
3391
|
+
baseDir: path.resolve(workspaceRoot, opts.baseDir ?? "."),
|
|
3392
|
+
heads: new Map(),
|
|
3393
|
+
bodies: new Map(),
|
|
3394
|
+
calls: new Map(),
|
|
3395
|
+
inputs: new Map(),
|
|
3396
|
+
masked: new Map(),
|
|
3397
|
+
served: new Map(),
|
|
3398
|
+
bodyKeys: new Map(),
|
|
3399
|
+
stateRoots: stateRootsOf(workspaceRoot, opts.stateRoots),
|
|
3400
|
+
replays: new Map(),
|
|
3401
|
+
artifacts: new Map(),
|
|
3402
|
+
checkers: new Map(),
|
|
3403
|
+
rechecks: new Map(),
|
|
3404
|
+
judge: { costUsdMicros: 0, calls: 0 },
|
|
3405
|
+
replaysUsed: 0,
|
|
3406
|
+
headsUsed: 0,
|
|
3407
|
+
groundingTokens: 0,
|
|
3408
|
+
lastYield: performance.now(),
|
|
3409
|
+
};
|
|
3410
|
+
const definitions = citationDefinitions(text);
|
|
3411
|
+
const extracted = extractClaims(text, { bibliographic: config.sources.bibliographic });
|
|
3412
|
+
const all = [
|
|
3413
|
+
...extracted.map((claim) => ({ claim })),
|
|
3414
|
+
...definitionClaims(text, definitions, config.sources.bibliographic, extracted.length),
|
|
3415
|
+
];
|
|
3416
|
+
// Every HEAD this verification will send, sent up front: a few at once,
|
|
3417
|
+
// under one deadline (`sendHeads`), not one at a time as each claim is
|
|
3418
|
+
// reached — in the order the claims ask, so the cap keeps the same URLs.
|
|
3419
|
+
// Collected into a set, and no further than the cap: the answer is
|
|
3420
|
+
// model-written, and a list searched once per URL it cites made this
|
|
3421
|
+
// quadratic in them (Pillar 3); a URL past the cap is `budget` anyway
|
|
3422
|
+
// (`headStatus`).
|
|
3423
|
+
if (sendsHeads(run)) {
|
|
3424
|
+
const urls = new Set();
|
|
3425
|
+
collect: for (const { claim } of all) {
|
|
3426
|
+
for (const id of claim.markers) {
|
|
3427
|
+
if (urls.size >= MAX_HEAD_REQUESTS)
|
|
3428
|
+
break collect;
|
|
3429
|
+
headCollectMeter.markers += 1;
|
|
3430
|
+
const source = definitions.get(id);
|
|
3431
|
+
const url = source !== undefined ? definitionUrl(source) : undefined;
|
|
3432
|
+
if (url === undefined || urls.has(url) || hostRefusal(run, url) !== undefined)
|
|
3433
|
+
continue;
|
|
3434
|
+
urls.add(url);
|
|
3435
|
+
}
|
|
3436
|
+
}
|
|
3437
|
+
sendHeads(run, [...urls]);
|
|
3438
|
+
}
|
|
3439
|
+
const claims = [];
|
|
3440
|
+
for (const pending of all)
|
|
3441
|
+
claims.push(await verifyClaim(run, pending, definitions));
|
|
3442
|
+
// The vacuous-pass backstop (design §2): nothing counted is a pass only
|
|
3443
|
+
// when nothing a reader sees cites or states a figure outside a question
|
|
3444
|
+
// (`backstopClaims`). The extractor's misses are read here, bluntly,
|
|
3445
|
+
// rather than one shape at a time.
|
|
3446
|
+
if (claims.every((claim) => claim.status === "assumption")) {
|
|
3447
|
+
const seen = all.map((pending) => pending.claim);
|
|
3448
|
+
for (const claim of backstopClaims(text, definitions, seen, all.length)) {
|
|
3449
|
+
claims.push(await verifyClaim(run, { claim }, definitions));
|
|
3450
|
+
}
|
|
3451
|
+
}
|
|
3452
|
+
// `max_assumptions` (design §2): assumptions are counted as DECLARED, and
|
|
3453
|
+
// the turn passes only when that count is within the cap. The hypotheses
|
|
3454
|
+
// an accepted proof rests on are assumptions too (design §6.4), counted
|
|
3455
|
+
// where the proof is first cited, each distinct one once. Past the cap,
|
|
3456
|
+
// in document order, an assumption claim is unmet with
|
|
3457
|
+
// `assumption_forbidden`, and so is a proof binding resting on a
|
|
3458
|
+
// hypothesis past it, so the nudge names them; under `assumptions:
|
|
3459
|
+
// forbid` every one already is. Either way each one is a refusal, and a
|
|
3460
|
+
// refusal fails the verdict whatever the score — otherwise a threshold
|
|
3461
|
+
// below 1 would pass an answer over the cap, the dodge the cap exists
|
|
3462
|
+
// to close.
|
|
3463
|
+
if (config.maxAssumptions !== undefined) {
|
|
3464
|
+
const cap = config.maxAssumptions;
|
|
3465
|
+
const detail = `past provability.max_assumptions (${cap})`;
|
|
3466
|
+
const numbered = new Map();
|
|
3467
|
+
let declared = 0;
|
|
3468
|
+
for (let i = 0; i < claims.length; i++) {
|
|
3469
|
+
const claim = claims[i];
|
|
3470
|
+
if (claim.status === "assumption") {
|
|
3471
|
+
declared += 1;
|
|
3472
|
+
if (declared <= cap)
|
|
3473
|
+
continue;
|
|
3474
|
+
claims[i] = {
|
|
3475
|
+
...claim,
|
|
3476
|
+
status: "unmet",
|
|
3477
|
+
bindings: claim.bindings.map((b) => ({
|
|
3478
|
+
...b,
|
|
3479
|
+
reason: "assumption_forbidden",
|
|
3480
|
+
detail,
|
|
3481
|
+
})),
|
|
3482
|
+
};
|
|
3483
|
+
continue;
|
|
3484
|
+
}
|
|
3485
|
+
let refused = false;
|
|
3486
|
+
const bindings = claim.bindings.map((b) => {
|
|
3487
|
+
if (b.kind !== "formal" || !b.ok || b.assumptions === undefined)
|
|
3488
|
+
return b;
|
|
3489
|
+
let past = false;
|
|
3490
|
+
for (const hypothesis of b.assumptions) {
|
|
3491
|
+
let n = numbered.get(hypothesis);
|
|
3492
|
+
if (n === undefined) {
|
|
3493
|
+
declared += 1;
|
|
3494
|
+
n = declared;
|
|
3495
|
+
numbered.set(hypothesis, n);
|
|
3496
|
+
}
|
|
3497
|
+
if (n > cap)
|
|
3498
|
+
past = true;
|
|
3499
|
+
}
|
|
3500
|
+
if (!past)
|
|
3501
|
+
return b;
|
|
3502
|
+
refused = true;
|
|
3503
|
+
return {
|
|
3504
|
+
...b,
|
|
3505
|
+
rung: "none",
|
|
3506
|
+
ok: false,
|
|
3507
|
+
reason: "assumption_forbidden",
|
|
3508
|
+
detail: `the proof rests on hypotheses ${detail}`,
|
|
3509
|
+
};
|
|
3510
|
+
});
|
|
3511
|
+
if (refused) {
|
|
3512
|
+
claims[i] = {
|
|
3513
|
+
...claim,
|
|
3514
|
+
status: statusOf(bindings),
|
|
3515
|
+
best: maxRung(...bindings.map((b) => b.rung)),
|
|
3516
|
+
bindings,
|
|
3517
|
+
};
|
|
3518
|
+
}
|
|
3519
|
+
}
|
|
3520
|
+
}
|
|
3521
|
+
const assumptions = claims.filter((c) => c.status === "assumption").length;
|
|
3522
|
+
const assumptionsRefused = claims.filter(refusedForAssumptions).length;
|
|
3523
|
+
const met = claims.filter((c) => c.status === "met").length;
|
|
3524
|
+
const unmet = claims.filter((c) => c.status === "unmet").length;
|
|
3525
|
+
const counted = claims.length - assumptions;
|
|
3526
|
+
const vacuous = claims.length === 0
|
|
3527
|
+
? "no claims to prove"
|
|
3528
|
+
: counted === 0
|
|
3529
|
+
? "every claim is an assumption"
|
|
3530
|
+
: undefined;
|
|
3531
|
+
const score = counted === 0 ? 1 : met / counted;
|
|
3532
|
+
const assumptionsOk = assumptionsRefused === 0;
|
|
3533
|
+
const verdict = (counted === 0 || met >= needed(config.threshold, counted)) && assumptionsOk ? "pass" : "fail";
|
|
3534
|
+
// A refused assumption fails the turn at every floor, so the answer would
|
|
3535
|
+
// have passed none.
|
|
3536
|
+
let achieved = "none";
|
|
3537
|
+
if (assumptionsOk && counted === 0)
|
|
3538
|
+
achieved = config.level;
|
|
3539
|
+
else if (assumptionsOk) {
|
|
3540
|
+
const ceilings = claims
|
|
3541
|
+
.filter((c) => c.status !== "assumption")
|
|
3542
|
+
.map((c) => levelRank(claimCeiling(run, c)));
|
|
3543
|
+
for (const level of [...PROVABILITY_LEVELS].reverse()) {
|
|
3544
|
+
const reaching = ceilings.filter((rank) => rank >= levelRank(level)).length;
|
|
3545
|
+
if (reaching >= needed(config.threshold, counted) && reaching > 0) {
|
|
3546
|
+
achieved = level;
|
|
3547
|
+
break;
|
|
3548
|
+
}
|
|
3549
|
+
}
|
|
3550
|
+
}
|
|
3551
|
+
// "N sources" beside the floor: at `cited` the origins cited, which is
|
|
3552
|
+
// all that rung asks of a source; at `grounded` and above the origins
|
|
3553
|
+
// that grounded a claim. Counting only grounded origins at `cited` put
|
|
3554
|
+
// "0 sources" beside a passing answer that cites a URL in every claim.
|
|
3555
|
+
const sourceRung = levelRank(config.level === "cited" ? "cited" : "grounded");
|
|
3556
|
+
const sourceOrigins = new Set();
|
|
3557
|
+
// Of those, the ones the run wrote itself (`ProofBinding.writtenBy`). A
|
|
3558
|
+
// FactCrossCheck of a file the run wrote stands where that file would
|
|
3559
|
+
// (`factCheckSupport`): it counts as a source, the run's own.
|
|
3560
|
+
const selfWritten = new Set();
|
|
3561
|
+
const hypotheses = new Set();
|
|
3562
|
+
const backing = new Set();
|
|
3563
|
+
let judgeWaived = 0;
|
|
3564
|
+
for (const claim of claims) {
|
|
3565
|
+
for (const b of claim.bindings) {
|
|
3566
|
+
const asSource = b.kind === "source" || (b.kind === "execution" && b.writtenBy !== undefined);
|
|
3567
|
+
if (asSource && levelRank(b.rung) >= sourceRung && b.origin) {
|
|
3568
|
+
sourceOrigins.add(b.origin);
|
|
3569
|
+
if (b.writtenBy !== undefined)
|
|
3570
|
+
selfWritten.add(b.origin);
|
|
3571
|
+
}
|
|
3572
|
+
if (b.kind === "formal" && b.ok && b.origin !== undefined) {
|
|
3573
|
+
backing.add(b.origin);
|
|
3574
|
+
for (const hypothesis of b.assumptions ?? [])
|
|
3575
|
+
hypotheses.add(hypothesis);
|
|
3576
|
+
}
|
|
3577
|
+
if (b.judgeWaived === true)
|
|
3578
|
+
judgeWaived += 1;
|
|
3579
|
+
}
|
|
3580
|
+
}
|
|
3581
|
+
// The checkers whose artifacts back a proven binding that still stands.
|
|
3582
|
+
const checkers = [...run.checkers.values()].filter((entry) => backing.has(entry.artifact));
|
|
3583
|
+
const core = {
|
|
3584
|
+
version: 1,
|
|
3585
|
+
level: config.level,
|
|
3586
|
+
verdict,
|
|
3587
|
+
score,
|
|
3588
|
+
threshold: config.threshold,
|
|
3589
|
+
achieved,
|
|
3590
|
+
claims,
|
|
3591
|
+
claimCount: claims.length,
|
|
3592
|
+
met,
|
|
3593
|
+
unmet,
|
|
3594
|
+
assumptions,
|
|
3595
|
+
...(assumptionsRefused > 0 ? { assumptionsRefused } : {}),
|
|
3596
|
+
...(hypotheses.size > 0 ? { hypotheses: hypotheses.size } : {}),
|
|
3597
|
+
sources: sourceOrigins.size,
|
|
3598
|
+
...(selfWritten.size > 0 ? { selfWritten: selfWritten.size } : {}),
|
|
3599
|
+
judgeWaived,
|
|
3600
|
+
retryIndex: opts.retryIndex ?? 0,
|
|
3601
|
+
...(checkers.length > 0 ? { checkers } : {}),
|
|
3602
|
+
...(vacuous !== undefined ? { vacuous } : {}),
|
|
3603
|
+
...(run.judge.model !== undefined ? { judgeModel: run.judge.model } : {}),
|
|
3604
|
+
...(run.judge.calls > 0 ? { spentUsdMicros: run.judge.costUsdMicros } : {}),
|
|
3605
|
+
createdAt: nowOf(opts.now).toISOString(),
|
|
3606
|
+
};
|
|
3607
|
+
return { ...core, bundleHash: bundleHash(core) };
|
|
3608
|
+
}
|