@crewhaus/provability 0.0.0-stage → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +300 -2
- package/dist/artifact.d.ts +9 -0
- package/dist/artifact.js +38 -0
- package/dist/builtins.d.ts +32 -0
- package/dist/builtins.js +46 -0
- package/dist/evidence.d.ts +326 -0
- package/dist/evidence.js +1391 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +22 -0
- package/dist/levels.d.ts +19 -0
- package/dist/levels.js +42 -0
- package/dist/origin.d.ts +83 -0
- package/dist/origin.js +296 -0
- package/dist/private-address.d.ts +34 -0
- package/dist/private-address.js +209 -0
- package/dist/purity.d.ts +18 -0
- package/dist/purity.js +15 -0
- package/dist/render.d.ts +47 -0
- package/dist/render.js +315 -0
- package/dist/store.d.ts +59 -0
- package/dist/store.js +319 -0
- package/dist/summary.d.ts +8 -0
- package/dist/summary.js +22 -0
- package/dist/types.d.ts +459 -0
- package/dist/types.js +2 -0
- package/dist/vacuous.d.ts +50 -0
- package/dist/vacuous.js +423 -0
- package/dist/verify.d.ts +176 -0
- package/dist/verify.js +3608 -0
- package/package.json +47 -4
package/dist/evidence.js
ADDED
|
@@ -0,0 +1,1391 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The evidence index: every body the run read and every result it got,
|
|
3
|
+
* keyed by where it came from (design §5.2).
|
|
4
|
+
*
|
|
5
|
+
* Grounding needs the bodies, and they are already persisted: the session
|
|
6
|
+
* event log records every `tool_use` (full input) and `tool_result` (full
|
|
7
|
+
* output, `isError`) verbatim — the fact the 0.3.0 proof ladder was built on.
|
|
8
|
+
* The index is built from the log, children included, because compaction may
|
|
9
|
+
* have evicted a fetch from the conversation but cannot evict it from the
|
|
10
|
+
* log; `evidenceFromMessages` reads the in-memory transcript for the turn
|
|
11
|
+
* still in flight (and for callers with no log), and `buildEvidenceIndex`
|
|
12
|
+
* merges the two into one record per call.
|
|
13
|
+
*
|
|
14
|
+
* Nothing here classifies, fetches or trusts anything. A record is what the
|
|
15
|
+
* tool returned, verbatim; an origin is read off the call's input and the
|
|
16
|
+
* result's own header; the verifiers decide what a record proves.
|
|
17
|
+
*/
|
|
18
|
+
import { createHash } from "node:crypto";
|
|
19
|
+
import { existsSync } from "node:fs";
|
|
20
|
+
import * as path from "node:path";
|
|
21
|
+
import { classifyDefinition, contentTokens, coverage } from "@crewhaus/claim-ledger";
|
|
22
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
23
|
+
import { DEFAULT_ROOT_DIR, openEventLog } from "@crewhaus/event-log";
|
|
24
|
+
import { defaultCatalog } from "@crewhaus/tool-catalog";
|
|
25
|
+
import { openForReadSync } from "@crewhaus/tool-safety/fs";
|
|
26
|
+
import { FETCH, PROVABILITY_PREFLIGHT, READ, RETRIEVE, WEB_FETCH, builtinKey, isMcpTool, } from "./builtins";
|
|
27
|
+
import { doiUrl, normalizePath, normalizeUrl } from "./origin";
|
|
28
|
+
const SESSION_ID_REGEX = /^sess_[0-9a-f]{16}$/;
|
|
29
|
+
/** How deep the sub-agent bracket walk descends (continuity-store's bound). */
|
|
30
|
+
const DEFAULT_MAX_DEPTH = 8;
|
|
31
|
+
/** Input fields that name a URL the call fetched. */
|
|
32
|
+
const URL_FIELDS = ["url", "uri", "href"];
|
|
33
|
+
/** Input fields that name a workspace file the call read. */
|
|
34
|
+
const PATH_FIELDS = ["file_path", "path", "file", "filePath"];
|
|
35
|
+
/** Fields of a JSON hit that name where the hit came from, in preference order. */
|
|
36
|
+
const HIT_ORIGIN_FIELDS = ["url", "uri", "href", "link", "source"];
|
|
37
|
+
/** Fields of a JSON hit that carry its text, in the order they are joined. */
|
|
38
|
+
const HIT_TEXT_FIELDS = [
|
|
39
|
+
"title",
|
|
40
|
+
"text",
|
|
41
|
+
"content",
|
|
42
|
+
"body",
|
|
43
|
+
"snippet",
|
|
44
|
+
"excerpt",
|
|
45
|
+
"chunk",
|
|
46
|
+
"pageContent",
|
|
47
|
+
"page_content",
|
|
48
|
+
"summary",
|
|
49
|
+
"abstract",
|
|
50
|
+
"description",
|
|
51
|
+
];
|
|
52
|
+
/** A JSON result larger than this is not parsed for hits (it is still one body). */
|
|
53
|
+
const MAX_JSON_HIT_CHARS = 8 * 1024 * 1024;
|
|
54
|
+
/** Hits indexed per result at most, and how deep a JSON result is walked for them. */
|
|
55
|
+
const MAX_HITS = 1_000;
|
|
56
|
+
const MAX_HIT_DEPTH = 6;
|
|
57
|
+
/** A WebFetch / Fetch response header block longer than this is not a header. */
|
|
58
|
+
const MAX_HEADER_CHARS = 64 * 1024;
|
|
59
|
+
/** Retrieve's schema cap on `k`: no Retrieve result carries more hits. */
|
|
60
|
+
const MAX_RETRIEVE_HITS = 50;
|
|
61
|
+
/**
|
|
62
|
+
* A result whose text is this much the call's own input is an echo of it
|
|
63
|
+
* (see `echoesInput`) — the grounded coverage default, so "an echo" and "a
|
|
64
|
+
* body that says the claim" are measured on one scale.
|
|
65
|
+
*/
|
|
66
|
+
const ECHO_COVERAGE = 0.8;
|
|
67
|
+
/**
|
|
68
|
+
* Where the runtime spills a result over its threshold, relative to the
|
|
69
|
+
* workspace (`@crewhaus/tool-result-store`'s `DEFAULT_ROOT_DIR`: the
|
|
70
|
+
* runtime resolves it against its cwd, the workspace root here).
|
|
71
|
+
*/
|
|
72
|
+
const DEFAULT_TOOL_RESULTS_DIR = ".crewhaus/tool-results";
|
|
73
|
+
/** The largest spilled output read back as a body (tool-web caps a page at 5 MB). */
|
|
74
|
+
const MAX_SPILL_BYTES = 8 * 1024 * 1024;
|
|
75
|
+
export class ProvabilityError extends CrewhausError {
|
|
76
|
+
name = "ProvabilityError";
|
|
77
|
+
}
|
|
78
|
+
/** `sha256:<hex>` of a string's UTF-8 bytes. */
|
|
79
|
+
export function sha256Of(text) {
|
|
80
|
+
return `sha256:${createHash("sha256").update(text, "utf8").digest("hex")}`;
|
|
81
|
+
}
|
|
82
|
+
function isRecordObject(value) {
|
|
83
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The origin a value names, or `undefined`: an `http(s)` URL (taken whole —
|
|
87
|
+
* it is what the tool was given, not prose to be read), a DOI, or — where
|
|
88
|
+
* `paths` is allowed — a workspace path. Whether a non-URL value is a DOI or
|
|
89
|
+
* a path is claim-ledger's `classifyDefinition`'s call, the one reader of
|
|
90
|
+
* "what does this reference name", so a citation and a tool input agree.
|
|
91
|
+
*/
|
|
92
|
+
function originOfValue(value, roots, paths) {
|
|
93
|
+
const trimmed = value.trim();
|
|
94
|
+
if (trimmed === "")
|
|
95
|
+
return undefined;
|
|
96
|
+
if (/^https?:\/\//i.test(trimmed))
|
|
97
|
+
return normalizeUrl(trimmed);
|
|
98
|
+
if (/^file:\/\//i.test(trimmed))
|
|
99
|
+
return paths ? normalizePath(trimmed, roots) : undefined;
|
|
100
|
+
const ref = classifyDefinition(trimmed);
|
|
101
|
+
if (ref.kind === "doi")
|
|
102
|
+
return doiUrl(ref.doi);
|
|
103
|
+
if (paths && ref.kind === "path")
|
|
104
|
+
return normalizePath(ref.path, roots);
|
|
105
|
+
return undefined;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* The header a WebFetch result opens with, read line by line rather than by
|
|
109
|
+
* a pattern that could backtrack over a large body. Returns the URL it
|
|
110
|
+
* names, the HTTP status, and where the page itself starts, or `undefined`
|
|
111
|
+
* when the result does not open so.
|
|
112
|
+
*
|
|
113
|
+
* tool-web prints `URL: <final>\nRequested: <asked>\nStatus: <n>\nContent-Type:
|
|
114
|
+
* <ct>\n\n`, the `Requested:` line only when a redirect moved the fetch, and
|
|
115
|
+
* no `Requested:` line at all when the final URL is the one asked for. Design
|
|
116
|
+
* §5.2 indexes both: the URL on the first line here, beside the requested one
|
|
117
|
+
* the call's own `url` names, so a citation of either grounds. A header
|
|
118
|
+
* without the `Requested:` line (an older tool-web, or no redirect) reads the
|
|
119
|
+
* same way.
|
|
120
|
+
*/
|
|
121
|
+
function webFetchHeader(content) {
|
|
122
|
+
if (!content.startsWith("URL: "))
|
|
123
|
+
return undefined;
|
|
124
|
+
const lines = [];
|
|
125
|
+
let at = 0;
|
|
126
|
+
while (at < content.length && at < MAX_HEADER_CHARS) {
|
|
127
|
+
const nl = content.indexOf("\n", at);
|
|
128
|
+
if (nl === -1)
|
|
129
|
+
return undefined;
|
|
130
|
+
const line = content.slice(at, nl).replace(/\r$/, "");
|
|
131
|
+
at = nl + 1;
|
|
132
|
+
if (line === "")
|
|
133
|
+
break;
|
|
134
|
+
lines.push(line);
|
|
135
|
+
if (lines.length > 4)
|
|
136
|
+
return undefined;
|
|
137
|
+
}
|
|
138
|
+
const url = /^URL: (\S+)$/.exec(lines[0] ?? "")?.[1];
|
|
139
|
+
if (url === undefined)
|
|
140
|
+
return undefined;
|
|
141
|
+
const rest = /^Requested: \S+$/.test(lines[1] ?? "") ? lines.slice(2) : lines.slice(1);
|
|
142
|
+
const status = /^Status: (\d{3})$/.exec(rest[0] ?? "")?.[1];
|
|
143
|
+
if (status === undefined || rest.length > 2)
|
|
144
|
+
return undefined;
|
|
145
|
+
if (rest.length === 2 && !/^Content-Type:/i.test(rest[1] ?? ""))
|
|
146
|
+
return undefined;
|
|
147
|
+
return { url, status: Number(status), end: at };
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* A Fetch result's response head (`tool-fetch` `formatResponse`: `HTTP
|
|
151
|
+
* <status> <reason>\n<name>: <value>…\n\n<body>`): the status, and where
|
|
152
|
+
* the body starts — or `undefined`. Strict on purpose — a status line with
|
|
153
|
+
* a short reason phrase and at least one header line — so a file that
|
|
154
|
+
* happens to open with "HTTP 200 is…" keeps its first paragraph.
|
|
155
|
+
*/
|
|
156
|
+
function httpHead(content) {
|
|
157
|
+
if (!content.startsWith("HTTP"))
|
|
158
|
+
return undefined;
|
|
159
|
+
const blank = content.slice(0, MAX_HEADER_CHARS).indexOf("\n\n");
|
|
160
|
+
if (blank === -1)
|
|
161
|
+
return undefined;
|
|
162
|
+
const [line = "", ...headers] = content.slice(0, blank).split("\n");
|
|
163
|
+
const status = /^HTTP(?:\/[\d.]+)? (\d{3})(?: [A-Za-z][A-Za-z '-]{0,40})?$/.exec(line)?.[1];
|
|
164
|
+
if (status === undefined || headers.length === 0)
|
|
165
|
+
return undefined;
|
|
166
|
+
return headers.every((h) => /^[!#$%&'*+.^_`|~0-9A-Za-z-]+: ?\S?/.test(h))
|
|
167
|
+
? { status: Number(status), end: blank + 2 }
|
|
168
|
+
: undefined;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* The HTTP status a fetcher's own header reports for its result (WebFetch's
|
|
172
|
+
* `Status:` line, Fetch's status line), or `undefined` for any other tool or
|
|
173
|
+
* a result with no such header. Read only off the tool that prints it.
|
|
174
|
+
*/
|
|
175
|
+
function responseStatus(toolName, content) {
|
|
176
|
+
if (toolName === WEB_FETCH)
|
|
177
|
+
return webFetchHeader(content)?.status;
|
|
178
|
+
if (toolName === FETCH)
|
|
179
|
+
return httpHead(content)?.status;
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Is `status` a page's own answer — 2xx? A 404 or a 503 body is the
|
|
184
|
+
* server's error page, not the page a citation names (design §2's
|
|
185
|
+
* `grounded` row: the cited source's BODY was fetched): one that prints
|
|
186
|
+
* the path it was asked for would otherwise ground any claim the model
|
|
187
|
+
* spelled into that path. A 3xx a fetcher hands back (it follows
|
|
188
|
+
* redirects) carries no page either.
|
|
189
|
+
*/
|
|
190
|
+
export function servesPage(status) {
|
|
191
|
+
return status === undefined || (status >= 200 && status < 300);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* The text a record is grounded against: the result — the whole output
|
|
195
|
+
* where the log holds tool-result-store's preview of it (`body`) — with its
|
|
196
|
+
* transport header taken off: WebFetch's `URL:`/`Status:` block and the `[user prompt:
|
|
197
|
+
* …]` line it prepends when the call carried a `prompt` (that is the model's
|
|
198
|
+
* OWN text: left in, a model could write the claim into `prompt` and find it
|
|
199
|
+
* "in the source"), and Fetch's status line and response headers. Read only
|
|
200
|
+
* off the tool that prints each; anything else is the result, verbatim.
|
|
201
|
+
*/
|
|
202
|
+
export function groundingBody(record) {
|
|
203
|
+
const content = record.body ?? record.content;
|
|
204
|
+
if (record.toolName === FETCH) {
|
|
205
|
+
const head = httpHead(content);
|
|
206
|
+
return head !== undefined ? content.slice(head.end) : content;
|
|
207
|
+
}
|
|
208
|
+
const header = record.toolName === WEB_FETCH ? webFetchHeader(content) : undefined;
|
|
209
|
+
if (header !== undefined) {
|
|
210
|
+
let body = content.slice(header.end);
|
|
211
|
+
const prompt = isRecordObject(record.input) ? record.input["prompt"] : undefined;
|
|
212
|
+
if (typeof prompt === "string" && prompt !== "") {
|
|
213
|
+
const echo = `[user prompt: ${prompt}]\n\n`;
|
|
214
|
+
if (body.startsWith(echo))
|
|
215
|
+
body = body.slice(echo.length);
|
|
216
|
+
}
|
|
217
|
+
return body;
|
|
218
|
+
}
|
|
219
|
+
return content;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* The builtins whose result IS the content of the origin their input names
|
|
223
|
+
* (design §5.2's list: `WebFetch.url`, `Fetch.url`, `Read`'s path), and the
|
|
224
|
+
* one field that names it. Every other builtin's `url`/`path` argument names
|
|
225
|
+
* what the tool acted ON, not a body it returned: SeoLint's `url` is context
|
|
226
|
+
* for its canonical-link check while its result echoes the markup the model
|
|
227
|
+
* passed in, UrlParse's result is the model's own URL taken apart, a
|
|
228
|
+
* GitBlame is an annotation of the file. Indexing those under the origin
|
|
229
|
+
* would let model-written text pose as a page the run never read.
|
|
230
|
+
*/
|
|
231
|
+
const READERS = {
|
|
232
|
+
[WEB_FETCH]: { field: "url", kind: "url" },
|
|
233
|
+
[FETCH]: { field: "url", kind: "url" },
|
|
234
|
+
[READ]: { field: "path", kind: "path" },
|
|
235
|
+
};
|
|
236
|
+
/**
|
|
237
|
+
* Fetch reads its URL only when it sends nothing: a POST's response is the
|
|
238
|
+
* server's answer to a body the model wrote (an echo endpoint returns it
|
|
239
|
+
* verbatim), not the page at that URL. tool-fetch drops a body on GET and
|
|
240
|
+
* HEAD, so those are reads whatever else the input carries.
|
|
241
|
+
*/
|
|
242
|
+
function fetchReads(input) {
|
|
243
|
+
const method = input["method"];
|
|
244
|
+
return method === undefined || (typeof method === "string" && /^(?:GET|HEAD)$/i.test(method));
|
|
245
|
+
}
|
|
246
|
+
/** Origins a builtin reader's input names: its one declared field, and nothing else. */
|
|
247
|
+
function readerOrigins(reader, toolName, input, roots) {
|
|
248
|
+
if (toolName === FETCH && !fetchReads(input))
|
|
249
|
+
return [];
|
|
250
|
+
const value = input[reader.field];
|
|
251
|
+
if (typeof value !== "string")
|
|
252
|
+
return [];
|
|
253
|
+
const origin = reader.kind === "url"
|
|
254
|
+
? /^https?:\/\//i.test(value.trim())
|
|
255
|
+
? normalizeUrl(value)
|
|
256
|
+
: undefined
|
|
257
|
+
: normalizePath(value, roots);
|
|
258
|
+
return origin !== undefined ? [origin] : [];
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Origins a call's input names (design §5.2: the bodies the run READ), and
|
|
262
|
+
* whether it was pointed at an address at all (`pointed`: such a call's
|
|
263
|
+
* result is that address's body, never hits — `resultHits`). A builtin
|
|
264
|
+
* names one only when it is a reader (`READERS`). An MCP server's tool
|
|
265
|
+
* (`isMcpTool`) — a fetcher or a file server whose flags the gate cannot
|
|
266
|
+
* see — by its URL fields, then its path fields, as far as its declared
|
|
267
|
+
* schema lets them (`mcpOrigins`); whether its result is then an echo of
|
|
268
|
+
* that input rather than a body is `echoesInput`'s question. Any other tool
|
|
269
|
+
* names none: the runtime's own non-builtins return MODEL output — `Task` a
|
|
270
|
+
* sub-agent's final message, `Consult` and `Escalate` a sibling model's
|
|
271
|
+
* reply, `Skill` a skill body, the memory, plan and goal tools what a model
|
|
272
|
+
* stored (design §8: child returns are inputs) — and their schemas strip a
|
|
273
|
+
* key they do not declare without refusing it, so a `url` the model adds to
|
|
274
|
+
* a `Recall` reaches the log whole. Read as an origin, it put a remembered
|
|
275
|
+
* sentence, or a child's wrong summary, under a page nobody fetched. A
|
|
276
|
+
* plugin's tool falls here too: fail closed, it grounds nothing by what its
|
|
277
|
+
* input names.
|
|
278
|
+
*/
|
|
279
|
+
function inputOrigins(toolName, input, roots, schemaOf) {
|
|
280
|
+
if (!isRecordObject(input))
|
|
281
|
+
return { origins: [], pointed: false };
|
|
282
|
+
const reader = READERS[toolName];
|
|
283
|
+
if (reader !== undefined) {
|
|
284
|
+
const origins = readerOrigins(reader, toolName, input, roots);
|
|
285
|
+
return { origins, pointed: origins.length > 0 };
|
|
286
|
+
}
|
|
287
|
+
if (builtinKey(toolName) !== undefined || !isMcpTool(toolName)) {
|
|
288
|
+
return { origins: [], pointed: false };
|
|
289
|
+
}
|
|
290
|
+
const named = namedOrigins(input, roots);
|
|
291
|
+
return {
|
|
292
|
+
origins: mcpOrigins(toolName, input, named, schemaOf).map((one) => one.origin),
|
|
293
|
+
pointed: named.length > 0,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
/** A non-builtin's URL fields, then its path fields: each value that names an origin, with it. */
|
|
297
|
+
function namedOrigins(input, roots) {
|
|
298
|
+
const out = [];
|
|
299
|
+
for (const field of URL_FIELDS) {
|
|
300
|
+
const value = input[field];
|
|
301
|
+
if (typeof value !== "string")
|
|
302
|
+
continue;
|
|
303
|
+
const origin = originOfValue(value, roots, false);
|
|
304
|
+
if (origin !== undefined)
|
|
305
|
+
out.push({ field, value, origin });
|
|
306
|
+
}
|
|
307
|
+
for (const field of PATH_FIELDS) {
|
|
308
|
+
const value = input[field];
|
|
309
|
+
if (typeof value !== "string")
|
|
310
|
+
continue;
|
|
311
|
+
const origin = /^https?:\/\//i.test(value.trim())
|
|
312
|
+
? normalizeUrl(value)
|
|
313
|
+
: normalizePath(value, roots);
|
|
314
|
+
if (origin !== undefined)
|
|
315
|
+
out.push({ field, value, origin });
|
|
316
|
+
}
|
|
317
|
+
return out;
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* The properties a tool's declared input schema lists at its top level, or
|
|
321
|
+
* `undefined` when no schema is known. A schema that lists none there (a
|
|
322
|
+
* `$ref`, an `anyOf`, a bare `{ type: "object" }`) declares no field this
|
|
323
|
+
* can read an address from: the empty set, so nothing its input carries
|
|
324
|
+
* names an origin (fail closed).
|
|
325
|
+
*/
|
|
326
|
+
export function declaredFields(schema) {
|
|
327
|
+
if (schema === undefined || schema === null)
|
|
328
|
+
return undefined;
|
|
329
|
+
if (!isRecordObject(schema))
|
|
330
|
+
return new Set();
|
|
331
|
+
const properties = schema["properties"];
|
|
332
|
+
return isRecordObject(properties) ? new Set(Object.keys(properties)) : new Set();
|
|
333
|
+
}
|
|
334
|
+
/** The schema the process's `defaultCatalog` holds for a tool (a compiled harness registers its MCP tools there). */
|
|
335
|
+
function catalogSchema(toolName) {
|
|
336
|
+
return defaultCatalog.get(toolName)?.jsonSchema;
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* The origins an MCP tool's input names that it can have been POINTED at
|
|
340
|
+
* (design §0 A, §5.2: a body is evidence because the run fetched it).
|
|
341
|
+
* tool-mcp registers every MCP tool with a permissive local validator, so a
|
|
342
|
+
* key the server does not declare reaches the log and the server whole, and
|
|
343
|
+
* the server ignores it: `mcp__filesystem__read_file {path: "notes.md", url:
|
|
344
|
+
* "https://acme.example/policy"}` reads `notes.md` and succeeds. Read as an
|
|
345
|
+
* origin, that `url` put a file the model wrote — or the run's own session
|
|
346
|
+
* log — under a page nobody fetched, served `grounded ✓`. So:
|
|
347
|
+
*
|
|
348
|
+
* - where the tool's declared schema is known (`InputSchemaLookup`), a field
|
|
349
|
+
* names an origin only when the schema declares it — a fetcher's `url`, a
|
|
350
|
+
* file server's `path` — and a key it does not declare names none;
|
|
351
|
+
* - where it is not known (a log read offline, a tool no catalog holds), the
|
|
352
|
+
* input names an origin only when it carries nothing else a call could
|
|
353
|
+
* have been pointed at: every address field names that one origin, and no
|
|
354
|
+
* other field carries a string at all. A `path` beside a `url` that names
|
|
355
|
+
* something else, a `paths` list beside a `url`
|
|
356
|
+
* (`read_multiple_files {paths: ["notes.md"], url}`), or a `command`
|
|
357
|
+
* beside one (`run_command {command: "cat notes.md", url}`) is a call that
|
|
358
|
+
* read one of them, and nothing says which, so it names neither: fail
|
|
359
|
+
* closed.
|
|
360
|
+
*
|
|
361
|
+
* A lookup that throws reads as an unknown schema.
|
|
362
|
+
*/
|
|
363
|
+
function mcpOrigins(toolName, input, named, schemaOf) {
|
|
364
|
+
if (named.length === 0)
|
|
365
|
+
return named;
|
|
366
|
+
let schema;
|
|
367
|
+
try {
|
|
368
|
+
schema = schemaOf(toolName);
|
|
369
|
+
}
|
|
370
|
+
catch {
|
|
371
|
+
schema = undefined;
|
|
372
|
+
}
|
|
373
|
+
const declared = declaredFields(schema);
|
|
374
|
+
if (declared !== undefined)
|
|
375
|
+
return named.filter((one) => declared.has(one.field));
|
|
376
|
+
if (carriesOtherStrings(input, new Set(named.map((one) => one.field))))
|
|
377
|
+
return [];
|
|
378
|
+
return new Set(named.map((one) => one.origin)).size === 1 ? named : [];
|
|
379
|
+
}
|
|
380
|
+
/** How deep {@link carriesOtherStrings} looks into an input. */
|
|
381
|
+
const MAX_INPUT_DEPTH = 6;
|
|
382
|
+
/**
|
|
383
|
+
* A fetcher's rendering options: how to return the page it was pointed at,
|
|
384
|
+
* never another place (`mcp__firecrawl__scrape {url, formats: ["markdown"]}`,
|
|
385
|
+
* `{url, method: "GET"}`).
|
|
386
|
+
*/
|
|
387
|
+
const RENDER_OPTIONS = new Set([
|
|
388
|
+
"format",
|
|
389
|
+
"formats",
|
|
390
|
+
"method",
|
|
391
|
+
"mode",
|
|
392
|
+
"outputFormat",
|
|
393
|
+
"output_format",
|
|
394
|
+
"encoding",
|
|
395
|
+
"lang",
|
|
396
|
+
"language",
|
|
397
|
+
"locale",
|
|
398
|
+
"country",
|
|
399
|
+
"waitFor",
|
|
400
|
+
"wait_for",
|
|
401
|
+
]);
|
|
402
|
+
/**
|
|
403
|
+
* Does `input` carry a non-blank string anywhere but in the address fields
|
|
404
|
+
* `named` read and a fetcher's rendering options ({@link RENDER_OPTIONS}) —
|
|
405
|
+
* a list entry, a nested field, a command line, an id? Each is something
|
|
406
|
+
* an undeclared tool may have been pointed at.
|
|
407
|
+
*/
|
|
408
|
+
function carriesOtherStrings(input, named, depth = 0) {
|
|
409
|
+
if (depth > MAX_INPUT_DEPTH)
|
|
410
|
+
return true;
|
|
411
|
+
if (typeof input === "string")
|
|
412
|
+
return input.trim() !== "";
|
|
413
|
+
if (Array.isArray(input))
|
|
414
|
+
return input.some((item) => carriesOtherStrings(item, named, depth + 1));
|
|
415
|
+
if (!isRecordObject(input))
|
|
416
|
+
return false;
|
|
417
|
+
return Object.entries(input).some(([field, value]) => !(depth === 0 && (named.has(field) || RENDER_OPTIONS.has(field))) &&
|
|
418
|
+
carriesOtherStrings(value, named, depth + 1));
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* Every place a call's input points at, whatever its schema declares — a
|
|
422
|
+
* builtin reader's one field, an MCP tool's URL and path fields and the
|
|
423
|
+
* entries of its `paths` / `files` lists — for the state-root test
|
|
424
|
+
* (verify.ts `provenanceOf`): a call whose input names the session log read
|
|
425
|
+
* it, whichever origin the body was then indexed under
|
|
426
|
+
* (`read_multiple_files {paths: [".crewhaus/sessions/…"], url}`).
|
|
427
|
+
*/
|
|
428
|
+
export function inputPlaces(toolName, input, roots = {}) {
|
|
429
|
+
if (!isRecordObject(input))
|
|
430
|
+
return [];
|
|
431
|
+
const reader = READERS[toolName];
|
|
432
|
+
if (reader !== undefined)
|
|
433
|
+
return readerOrigins(reader, toolName, input, roots);
|
|
434
|
+
if (builtinKey(toolName) !== undefined || !isMcpTool(toolName))
|
|
435
|
+
return [];
|
|
436
|
+
const places = namedOrigins(input, roots).map((one) => one.origin);
|
|
437
|
+
for (const field of LIST_FIELDS) {
|
|
438
|
+
const list = input[field];
|
|
439
|
+
if (!Array.isArray(list))
|
|
440
|
+
continue;
|
|
441
|
+
for (const item of list.slice(0, MAX_LISTED_PLACES)) {
|
|
442
|
+
if (typeof item !== "string" || item.trim() === "")
|
|
443
|
+
continue;
|
|
444
|
+
const place = /^https?:\/\//i.test(item.trim())
|
|
445
|
+
? normalizeUrl(item)
|
|
446
|
+
: normalizePath(item, roots);
|
|
447
|
+
if (place !== undefined)
|
|
448
|
+
places.push(place);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
return [...new Set(places)];
|
|
452
|
+
}
|
|
453
|
+
/** How many entries of one list field {@link inputPlaces} reads. */
|
|
454
|
+
const MAX_LISTED_PLACES = 256;
|
|
455
|
+
/** Is `toolName` one of the builtin readers (`WebFetch`, `Fetch`, `Read`: `READERS`)? */
|
|
456
|
+
export function isReader(toolName) {
|
|
457
|
+
return READERS[toolName] !== undefined;
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Input fields a call that writes names the place it writes by: `Write`'s
|
|
461
|
+
* and `Edit`'s `path`, `CopyPath`'s `destination`, a PUT's `url`, an
|
|
462
|
+
* export's `out`, an archive's `output`, a split's `outputDir`.
|
|
463
|
+
*/
|
|
464
|
+
const TARGET_FIELDS = [
|
|
465
|
+
...URL_FIELDS,
|
|
466
|
+
...PATH_FIELDS,
|
|
467
|
+
"destination",
|
|
468
|
+
"dest",
|
|
469
|
+
"target",
|
|
470
|
+
"to",
|
|
471
|
+
"output",
|
|
472
|
+
"outputPath",
|
|
473
|
+
"output_path",
|
|
474
|
+
"out",
|
|
475
|
+
"outFile",
|
|
476
|
+
"outputFile",
|
|
477
|
+
"output_file",
|
|
478
|
+
"outDir",
|
|
479
|
+
"outputDir",
|
|
480
|
+
"output_dir",
|
|
481
|
+
"golden",
|
|
482
|
+
"manifest",
|
|
483
|
+
"specFile",
|
|
484
|
+
];
|
|
485
|
+
/** Input fields a call names the place it copies, moves or unpacks FROM by. */
|
|
486
|
+
const SOURCE_FIELDS = ["source", "from", "src", "archive"];
|
|
487
|
+
/**
|
|
488
|
+
* List fields that name several places: what a call that also names a
|
|
489
|
+
* target reads into it (`ConcatFiles`' `paths`), else what it writes. Not
|
|
490
|
+
* `sources`: FactCrossCheck's `claims[].sources` are what it read.
|
|
491
|
+
*/
|
|
492
|
+
const LIST_FIELDS = ["paths", "files"];
|
|
493
|
+
/** Fields that carry a unified diff: the files it names are written, its `+` lines are the new text. */
|
|
494
|
+
const PATCH_FIELDS = ["patch", "diff"];
|
|
495
|
+
/** Fields that carry the text a call puts where it writes (`Write`'s `content`, `Edit`'s `newString`, a PUT's `body`). */
|
|
496
|
+
const CONTENT_FIELDS = new Set([
|
|
497
|
+
"content",
|
|
498
|
+
"contents",
|
|
499
|
+
"text",
|
|
500
|
+
"body",
|
|
501
|
+
"data",
|
|
502
|
+
"value",
|
|
503
|
+
"newString",
|
|
504
|
+
"new_string",
|
|
505
|
+
"newText",
|
|
506
|
+
"new_text",
|
|
507
|
+
"new_str",
|
|
508
|
+
"replacement",
|
|
509
|
+
"replace",
|
|
510
|
+
"append",
|
|
511
|
+
"insert",
|
|
512
|
+
"lines",
|
|
513
|
+
]);
|
|
514
|
+
/** Fields that carry the text an edit REPLACES (`Edit`'s `oldString`, an MCP file server's `oldText`). */
|
|
515
|
+
const OLD_TEXT_FIELDS = new Set([
|
|
516
|
+
"oldString",
|
|
517
|
+
"old_string",
|
|
518
|
+
"oldText",
|
|
519
|
+
"old_text",
|
|
520
|
+
"old_str",
|
|
521
|
+
"search",
|
|
522
|
+
"find",
|
|
523
|
+
]);
|
|
524
|
+
/** Places read off one list field at most. */
|
|
525
|
+
const MAX_LIST_PLACES = 64;
|
|
526
|
+
/** Files read off one diff at most. */
|
|
527
|
+
const MAX_PATCH_FILES = 64;
|
|
528
|
+
/**
|
|
529
|
+
* The files a unified diff names as written (`+++ b/<path>`), with `strip`
|
|
530
|
+
* leading components dropped (git's `-p`, default 1).
|
|
531
|
+
*/
|
|
532
|
+
function patchFiles(patch, strip) {
|
|
533
|
+
const out = [];
|
|
534
|
+
for (const m of patch.matchAll(/^\+\+\+ ([^\t\n\r]+)/gm)) {
|
|
535
|
+
const named = m[1].trim();
|
|
536
|
+
if (named === "/dev/null" || named === "")
|
|
537
|
+
continue;
|
|
538
|
+
const parts = named.split("/");
|
|
539
|
+
const file = parts.slice(Math.min(strip, parts.length - 1)).join("/");
|
|
540
|
+
if (!out.includes(file))
|
|
541
|
+
out.push(file);
|
|
542
|
+
if (out.length >= MAX_PATCH_FILES)
|
|
543
|
+
break;
|
|
544
|
+
}
|
|
545
|
+
return out;
|
|
546
|
+
}
|
|
547
|
+
/**
|
|
548
|
+
* The places a call may have WRITTEN, and the ones it copied, moved or
|
|
549
|
+
* unpacked them from, as its input names them, normalized like a
|
|
550
|
+
* citation's (verify.ts `provenanceOf` reads them: a source the run wrote
|
|
551
|
+
* itself and then read back, or a copy of the harness's own state). Any
|
|
552
|
+
* tool's input, builtin or not: the place it writes by
|
|
553
|
+
* {@link TARGET_FIELDS}; the place it copies from by {@link SOURCE_FIELDS};
|
|
554
|
+
* a list ({@link LIST_FIELDS}) as sources beside a target (`ConcatFiles`)
|
|
555
|
+
* and as written places otherwise; and each file a diff
|
|
556
|
+
* ({@link PATCH_FIELDS}) names, under the call's `cwd` (`GitApplyPatch`).
|
|
557
|
+
* None for a call that only reads — a builtin reader (`WebFetch`, `Read`, a
|
|
558
|
+
* GET or HEAD `Fetch`) and `ProvabilityPreflight`, which writes nowhere.
|
|
559
|
+
* Whether the tool can write at all (a pure tool cannot) is the caller's
|
|
560
|
+
* question.
|
|
561
|
+
*/
|
|
562
|
+
export function writtenPlaces(toolName, input, roots = {}) {
|
|
563
|
+
if (!isRecordObject(input) || toolName === PROVABILITY_PREFLIGHT)
|
|
564
|
+
return [];
|
|
565
|
+
if (toolName === FETCH ? fetchReads(input) : isReader(toolName))
|
|
566
|
+
return [];
|
|
567
|
+
const out = [];
|
|
568
|
+
const add = (value, source) => {
|
|
569
|
+
const trimmed = value.trim();
|
|
570
|
+
if (trimmed === "")
|
|
571
|
+
return;
|
|
572
|
+
const place = /^https?:\/\//i.test(trimmed)
|
|
573
|
+
? normalizeUrl(trimmed)
|
|
574
|
+
: normalizePath(trimmed, roots);
|
|
575
|
+
if (place === undefined)
|
|
576
|
+
return;
|
|
577
|
+
if (!out.some((one) => one.place === place && one.source === source)) {
|
|
578
|
+
out.push({ place, source });
|
|
579
|
+
}
|
|
580
|
+
};
|
|
581
|
+
let target = false;
|
|
582
|
+
for (const field of TARGET_FIELDS) {
|
|
583
|
+
const value = input[field];
|
|
584
|
+
if (typeof value !== "string")
|
|
585
|
+
continue;
|
|
586
|
+
add(value, false);
|
|
587
|
+
target = true;
|
|
588
|
+
}
|
|
589
|
+
for (const field of SOURCE_FIELDS) {
|
|
590
|
+
const value = input[field];
|
|
591
|
+
if (typeof value === "string")
|
|
592
|
+
add(value, true);
|
|
593
|
+
}
|
|
594
|
+
for (const field of LIST_FIELDS) {
|
|
595
|
+
const list = input[field];
|
|
596
|
+
if (!Array.isArray(list))
|
|
597
|
+
continue;
|
|
598
|
+
for (const item of list.slice(0, MAX_LIST_PLACES)) {
|
|
599
|
+
if (typeof item === "string")
|
|
600
|
+
add(item, target);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
const cwd = typeof input["cwd"] === "string" ? input["cwd"] : ".";
|
|
604
|
+
const strip = input["strip"];
|
|
605
|
+
for (const field of PATCH_FIELDS) {
|
|
606
|
+
const patch = input[field];
|
|
607
|
+
if (typeof patch !== "string")
|
|
608
|
+
continue;
|
|
609
|
+
const drop = typeof strip === "number" && Number.isInteger(strip) && strip >= 0 ? strip : 1;
|
|
610
|
+
for (const file of patchFiles(patch, drop))
|
|
611
|
+
add(path.posix.join(cwd, file), false);
|
|
612
|
+
}
|
|
613
|
+
return out;
|
|
614
|
+
}
|
|
615
|
+
/**
|
|
616
|
+
* The text a call put where it wrote ({@link writtenPlaces}), read off its
|
|
617
|
+
* input by field (verify.ts `selfWriter`): what a content field
|
|
618
|
+
* ({@link CONTENT_FIELDS}) carries, at any depth (`MultiEdit`'s `edits`, an
|
|
619
|
+
* MCP file server's `edits[].newText`), and a diff's `+` lines, as added;
|
|
620
|
+
* what an edit replaces ({@link OLD_TEXT_FIELDS}) and a diff's `-` lines, as
|
|
621
|
+
* removed — so an `Edit` of `14` to `30` added `30`, not `14`. A search's
|
|
622
|
+
* `pattern`, a commit `message`, an address: none of those is text put
|
|
623
|
+
* anywhere.
|
|
624
|
+
*/
|
|
625
|
+
export function writtenText(input) {
|
|
626
|
+
const added = [];
|
|
627
|
+
const removed = [];
|
|
628
|
+
const diff = (patch) => {
|
|
629
|
+
const plus = [];
|
|
630
|
+
const minus = [];
|
|
631
|
+
for (const line of patch.split(/\r?\n/)) {
|
|
632
|
+
if (line.startsWith("+++") || line.startsWith("---"))
|
|
633
|
+
continue;
|
|
634
|
+
if (line.startsWith("+"))
|
|
635
|
+
plus.push(line.slice(1));
|
|
636
|
+
else if (line.startsWith("-"))
|
|
637
|
+
minus.push(line.slice(1));
|
|
638
|
+
}
|
|
639
|
+
if (plus.length > 0)
|
|
640
|
+
added.push(plus.join("\n"));
|
|
641
|
+
if (minus.length > 0)
|
|
642
|
+
removed.push(minus.join("\n"));
|
|
643
|
+
};
|
|
644
|
+
const walk = (value, key, depth) => {
|
|
645
|
+
if (depth > MAX_HIT_DEPTH + 2)
|
|
646
|
+
return;
|
|
647
|
+
if (typeof value === "string") {
|
|
648
|
+
if (key === undefined || value.trim() === "")
|
|
649
|
+
return;
|
|
650
|
+
if (OLD_TEXT_FIELDS.has(key))
|
|
651
|
+
removed.push(value);
|
|
652
|
+
else if (PATCH_FIELDS.includes(key))
|
|
653
|
+
diff(value);
|
|
654
|
+
else if (CONTENT_FIELDS.has(key))
|
|
655
|
+
added.push(value);
|
|
656
|
+
return;
|
|
657
|
+
}
|
|
658
|
+
if (Array.isArray(value)) {
|
|
659
|
+
for (const item of value)
|
|
660
|
+
walk(item, key, depth + 1);
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
if (isRecordObject(value)) {
|
|
664
|
+
for (const [field, item] of Object.entries(value))
|
|
665
|
+
walk(item, field, depth + 1);
|
|
666
|
+
}
|
|
667
|
+
};
|
|
668
|
+
walk(input, undefined, 0);
|
|
669
|
+
return { added, removed };
|
|
670
|
+
}
|
|
671
|
+
/**
|
|
672
|
+
* The input strings a call names its origins by, as written: the URL or
|
|
673
|
+
* path it was pointed at, not text it was handed — a builtin reader's one
|
|
674
|
+
* declared field, a non-builtin's URL and path fields whose origin the
|
|
675
|
+
* record carries (what `inputOrigins` read off an MCP tool, and the address
|
|
676
|
+
* a record built outside the log was fetched under — the research
|
|
677
|
+
* crawler's carry their own origins). A key the tool does not declare names
|
|
678
|
+
* no origin (`mcpOrigins`), so its value is text the call was handed like
|
|
679
|
+
* any other. `grounded`'s echo test (verify.ts `echoCheck`) reads these
|
|
680
|
+
* apart from the call's other strings, because a URL's slug can spell a
|
|
681
|
+
* claim the page itself states — a reader returns the page, not its
|
|
682
|
+
* address — and only a body that PRINTS the address (a 404 page repeating
|
|
683
|
+
* the path it was asked for, a page headed with its own URL) can be
|
|
684
|
+
* reflecting it.
|
|
685
|
+
*/
|
|
686
|
+
export function originFieldValues(record, roots = {}) {
|
|
687
|
+
const { toolName, input } = record;
|
|
688
|
+
if (!isRecordObject(input))
|
|
689
|
+
return [];
|
|
690
|
+
const reader = READERS[toolName];
|
|
691
|
+
if (reader !== undefined) {
|
|
692
|
+
const value = input[reader.field];
|
|
693
|
+
return typeof value === "string" && readerOrigins(reader, toolName, input, roots).length > 0
|
|
694
|
+
? [value]
|
|
695
|
+
: [];
|
|
696
|
+
}
|
|
697
|
+
if (builtinKey(toolName) !== undefined)
|
|
698
|
+
return [];
|
|
699
|
+
return namedOrigins(input, roots)
|
|
700
|
+
.filter((one) => record.origins.includes(one.origin))
|
|
701
|
+
.map((one) => one.value);
|
|
702
|
+
}
|
|
703
|
+
/** A JSON hit's text: its text-bearing fields, or every string field when it has none. */
|
|
704
|
+
function hitText(hit, originField) {
|
|
705
|
+
const texts = HIT_TEXT_FIELDS.map((field) => hit[field]).filter((value) => typeof value === "string" && value.trim() !== "");
|
|
706
|
+
if (texts.length > 0)
|
|
707
|
+
return texts.join("\n");
|
|
708
|
+
return Object.entries(hit)
|
|
709
|
+
.filter(([key, value]) => key !== originField && typeof value === "string")
|
|
710
|
+
.map(([, value]) => value)
|
|
711
|
+
.join("\n");
|
|
712
|
+
}
|
|
713
|
+
function collectJsonHits(value, roots, out, depth) {
|
|
714
|
+
if (out.length >= MAX_HITS || depth > MAX_HIT_DEPTH)
|
|
715
|
+
return;
|
|
716
|
+
if (Array.isArray(value)) {
|
|
717
|
+
for (const item of value)
|
|
718
|
+
collectJsonHits(item, roots, out, depth + 1);
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
721
|
+
if (!isRecordObject(value))
|
|
722
|
+
return;
|
|
723
|
+
for (const field of HIT_ORIGIN_FIELDS) {
|
|
724
|
+
const named = value[field];
|
|
725
|
+
if (typeof named !== "string")
|
|
726
|
+
continue;
|
|
727
|
+
const origin = originOfValue(named, roots, field === "source");
|
|
728
|
+
if (origin === undefined)
|
|
729
|
+
continue;
|
|
730
|
+
const content = hitText(value, field);
|
|
731
|
+
if (content.trim() !== "")
|
|
732
|
+
out.push({ origin, content });
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
for (const child of Object.values(value)) {
|
|
736
|
+
if (typeof child === "object" && child !== null)
|
|
737
|
+
collectJsonHits(child, roots, out, depth + 1);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
/** Retrieve's hit header (`tool-retrieve` `formatHits`): `[1] id=… doc=<source> score=0.9123`. */
|
|
741
|
+
const RETRIEVE_HIT = /^\[(\d+)\] id=\S+ doc=(\S+) score=[\d.eE+-]+$/gm;
|
|
742
|
+
/**
|
|
743
|
+
* Retrieve's hits, each under its own document. A chunk preview is corpus
|
|
744
|
+
* text — a crawled third-party page among it — and is printed raw, so a
|
|
745
|
+
* line in it shaped like a hit header would otherwise start a "hit" under
|
|
746
|
+
* any origin it names. So the structure `formatHits` prints is required,
|
|
747
|
+
* and anything else reads as no hits at all (fail closed: a claim cited to
|
|
748
|
+
* a Retrieve hit is then `not_fetched`, never grounded in the wrong text):
|
|
749
|
+
*
|
|
750
|
+
* - a header counts only at the start of the result or after a blank line,
|
|
751
|
+
* the separator `formatHits` puts between hits;
|
|
752
|
+
* - the headers are numbered 1, 2, 3, … with no repeat and no gap;
|
|
753
|
+
* - there are no more of them than the call asked for (`k`), or than
|
|
754
|
+
* Retrieve ever returns.
|
|
755
|
+
*
|
|
756
|
+
* A forged header with a blank line before it inside the LAST hit's
|
|
757
|
+
* preview still reads as one more hit; closing that needs tool-retrieve to
|
|
758
|
+
* print a delimiter a preview cannot contain (reported, not done here).
|
|
759
|
+
*/
|
|
760
|
+
function retrieveHits(content, input, roots) {
|
|
761
|
+
const heads = [...content.matchAll(RETRIEVE_HIT)].filter((head) => head.index !== undefined && (head.index === 0 || content.endsWith("\n\n", head.index)));
|
|
762
|
+
if (heads.length === 0 || heads[0]?.index !== 0)
|
|
763
|
+
return [];
|
|
764
|
+
const k = isRecordObject(input) ? input["k"] : undefined;
|
|
765
|
+
const most = typeof k === "number" && Number.isInteger(k) && k > 0
|
|
766
|
+
? Math.min(k, MAX_RETRIEVE_HITS)
|
|
767
|
+
: MAX_RETRIEVE_HITS;
|
|
768
|
+
if (heads.length > most)
|
|
769
|
+
return [];
|
|
770
|
+
if (heads.some((head, i) => head[1] !== String(i + 1)))
|
|
771
|
+
return [];
|
|
772
|
+
const out = [];
|
|
773
|
+
heads.forEach((head, i) => {
|
|
774
|
+
if (head.index === undefined)
|
|
775
|
+
return;
|
|
776
|
+
const next = heads[i + 1]?.index ?? content.length;
|
|
777
|
+
const text = content.slice(head.index + head[0].length, next).trim();
|
|
778
|
+
const origin = originOfValue(head[2], roots, true);
|
|
779
|
+
if (origin !== undefined && text !== "")
|
|
780
|
+
out.push({ origin, content: text });
|
|
781
|
+
});
|
|
782
|
+
return out;
|
|
783
|
+
}
|
|
784
|
+
/** Every string a call's input carries, at any depth (the grounded echo test reads them too). */
|
|
785
|
+
export function inputStrings(value, out = [], depth = 0) {
|
|
786
|
+
if (depth > MAX_HIT_DEPTH + 2)
|
|
787
|
+
return out;
|
|
788
|
+
if (typeof value === "string")
|
|
789
|
+
out.push(value);
|
|
790
|
+
else if (Array.isArray(value))
|
|
791
|
+
for (const item of value)
|
|
792
|
+
inputStrings(item, out, depth + 1);
|
|
793
|
+
else if (isRecordObject(value)) {
|
|
794
|
+
for (const item of Object.values(value))
|
|
795
|
+
inputStrings(item, out, depth + 1);
|
|
796
|
+
}
|
|
797
|
+
return out;
|
|
798
|
+
}
|
|
799
|
+
const collapse = (text) => text.replace(/\s+/g, " ").trim().toLowerCase();
|
|
800
|
+
/**
|
|
801
|
+
* Is `text` the call's own input handed back? A tool the gate cannot see
|
|
802
|
+
* into (an MCP server's, a plugin's) may reflect what the model sent it — a
|
|
803
|
+
* SQL tool returns the literal a SELECT names, an "annotate" tool repeats
|
|
804
|
+
* its note — and its result would then put the model's own words under any
|
|
805
|
+
* origin it names. Such text is not a body anyone else wrote, so it is not
|
|
806
|
+
* indexed: it is a substring of one of the input's strings, or its content
|
|
807
|
+
* words are covered by the input's at the grounded bar.
|
|
808
|
+
*
|
|
809
|
+
* A search hit is far longer than its query and keeps its origin. A
|
|
810
|
+
* reflection padded with text from somewhere else (a SELECT that appends a
|
|
811
|
+
* real row to its literal) is not wholly an echo and is indexed; `grounded`
|
|
812
|
+
* then refuses a claim whose words, at their tightest, are the input's own
|
|
813
|
+
* (verify.ts `echoedSpan`).
|
|
814
|
+
*/
|
|
815
|
+
function echoesInput(text, input) {
|
|
816
|
+
const strings = inputStrings(input);
|
|
817
|
+
if (strings.length === 0)
|
|
818
|
+
return false;
|
|
819
|
+
const body = collapse(text);
|
|
820
|
+
if (body === "")
|
|
821
|
+
return false;
|
|
822
|
+
if (strings.some((value) => collapse(value).includes(body)))
|
|
823
|
+
return true;
|
|
824
|
+
return coverage(contentTokens(text), contentTokens(strings.join("\n"))) >= ECHO_COVERAGE;
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* Hits a result carries — and only where the hits cannot be the model's own
|
|
828
|
+
* writing dressed as someone else's page. A hit names its own origin, so a
|
|
829
|
+
* result the model can author (a `Read` of a file it wrote, `Bash` echoing
|
|
830
|
+
* JSON, a fetch of an echo endpoint) could otherwise put any text under any
|
|
831
|
+
* URL and ground a citation of a page the run never read. So:
|
|
832
|
+
*
|
|
833
|
+
* - Retrieve's `[n] id=… doc=… score=…` blocks are read only from Retrieve,
|
|
834
|
+
* whose corpus the operator ingested, and only in the exact structure it
|
|
835
|
+
* prints (`retrieveHits`).
|
|
836
|
+
* - JSON hits (`url`/`uri`/`href`/`link`/`source` fields) are read only from
|
|
837
|
+
* an MCP server's tool (`mcp__…`, `isMcpTool`) whose input names no URL or
|
|
838
|
+
* path: a search returns hits, a fetch of one URL returns that URL's
|
|
839
|
+
* body, whatever the body looks like. Never from the runtime's own tools
|
|
840
|
+
* whose result is MODEL output — `Task` (a sub-agent's final message),
|
|
841
|
+
* `Consult`/`Escalate`, an A2A peer, `Skill`, the memory, plan and goal
|
|
842
|
+
* tools: a sub-agent that answers `[{"url": …, "text": …}]` would
|
|
843
|
+
* otherwise ground a citation of a page nobody fetched, and a child that
|
|
844
|
+
* did fetch it would have its summary indexed beside the page, where a
|
|
845
|
+
* wrong summary grounds what the page contradicts (design §8: child
|
|
846
|
+
* returns are inputs; the child's evidence is its own logged fetches,
|
|
847
|
+
* which the child-log walk indexes). A hit whose text is the call's own
|
|
848
|
+
* input handed back is dropped (`echoesInput`).
|
|
849
|
+
*/
|
|
850
|
+
function resultHits(call, fetchedSomething, roots) {
|
|
851
|
+
const { content } = call;
|
|
852
|
+
if (call.toolName === RETRIEVE)
|
|
853
|
+
return retrieveHits(content, call.input, roots);
|
|
854
|
+
if (fetchedSomething || !isMcpTool(call.toolName))
|
|
855
|
+
return [];
|
|
856
|
+
const trimmed = content.trim();
|
|
857
|
+
if (trimmed.length > MAX_JSON_HIT_CHARS || !/^[[{]/.test(trimmed))
|
|
858
|
+
return [];
|
|
859
|
+
let parsed;
|
|
860
|
+
try {
|
|
861
|
+
parsed = JSON.parse(trimmed);
|
|
862
|
+
}
|
|
863
|
+
catch {
|
|
864
|
+
return [];
|
|
865
|
+
}
|
|
866
|
+
const out = [];
|
|
867
|
+
collectJsonHits(parsed, roots, out, 0);
|
|
868
|
+
return out.filter((hit) => !echoesInput(hit.content, call.input));
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* The URL a record's body was served from, where the fetcher says so: the
|
|
872
|
+
* final URL a WebFetch header names on its first line, after any redirect
|
|
873
|
+
* (design §5.2), normalized. `undefined` for any other record — its body is
|
|
874
|
+
* what its origins name. A citation of the requested URL grounds in a body
|
|
875
|
+
* this URL served, so the host checks that hold a redirect hop (`sources.deny`,
|
|
876
|
+
* the private-address rule) and `grounded.independent` read the body by
|
|
877
|
+
* this (verify.ts `grounded`).
|
|
878
|
+
*/
|
|
879
|
+
export function servedFrom(record) {
|
|
880
|
+
if (record.toolName !== WEB_FETCH)
|
|
881
|
+
return undefined;
|
|
882
|
+
const header = webFetchHeader(record.body ?? record.content);
|
|
883
|
+
return header !== undefined ? normalizeUrl(header.url) : undefined;
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* One call as an evidence record: its origins read off the input — a
|
|
887
|
+
* builtin reader's one field (WebFetch's and Fetch's `url`, Read's `path`),
|
|
888
|
+
* or an MCP tool's `url`, `uri`, `href`, `file_path`, `path`, `file`,
|
|
889
|
+
* `filePath` as far as its declared schema lets them (`mcpOrigins`), unless
|
|
890
|
+
* its result is an echo of its input (`inputOrigins`, `echoesInput`) — and
|
|
891
|
+
* the URL a WebFetch result's header names on its
|
|
892
|
+
* first line (indexed BESIDE the requested URL, so that once tool-web
|
|
893
|
+
* prints the final URL there a citation of either grounds — see
|
|
894
|
+
* `webFetchHeader`), and the hits a multi-source result carries (Retrieve;
|
|
895
|
+
* JSON results with `url`/`uri`/`href`/`link`/`source` from an MCP tool —
|
|
896
|
+
* see `resultHits` for which tools). A fetcher's HTTP status is carried
|
|
897
|
+
* (`status`): a non-2xx body is not the page (`servesPage`). Error results
|
|
898
|
+
* are recorded too: the verifiers reject them, by reason.
|
|
899
|
+
*
|
|
900
|
+
* Everything is read off the whole output (`body`) where the log holds a
|
|
901
|
+
* spill preview of it; `content` stays what the log holds, which a replay
|
|
902
|
+
* is compared against.
|
|
903
|
+
*/
|
|
904
|
+
export function recordFromCall(call, opts = {}) {
|
|
905
|
+
const roots = {
|
|
906
|
+
...(opts.workspaceRoot !== undefined ? { workspaceRoot: opts.workspaceRoot } : {}),
|
|
907
|
+
};
|
|
908
|
+
const text = call.body ?? call.content;
|
|
909
|
+
const { origins: named, pointed } = inputOrigins(call.toolName, call.input, roots, opts.inputSchemaOf ?? catalogSchema);
|
|
910
|
+
// An MCP tool that names an origin returns whatever it returns; when that
|
|
911
|
+
// is its own input, it read nothing.
|
|
912
|
+
const origins = named.length > 0 && READERS[call.toolName] === undefined && echoesInput(text, call.input)
|
|
913
|
+
? []
|
|
914
|
+
: [...named];
|
|
915
|
+
// A call pointed at an address returns that address's body, whatever the
|
|
916
|
+
// body looks like — never hits, whether or not the address was declared.
|
|
917
|
+
const fetchedSomething = pointed;
|
|
918
|
+
// The URL line is tool-web's own header; read off any other tool it would
|
|
919
|
+
// let a file or a page that merely opens with "URL: …" claim a URL.
|
|
920
|
+
const header = call.toolName === WEB_FETCH ? webFetchHeader(text) : undefined;
|
|
921
|
+
if (header !== undefined) {
|
|
922
|
+
const final = normalizeUrl(header.url);
|
|
923
|
+
if (final !== undefined)
|
|
924
|
+
origins.push(final);
|
|
925
|
+
}
|
|
926
|
+
const status = responseStatus(call.toolName, text);
|
|
927
|
+
const hits = resultHits({ ...call, content: text }, fetchedSomething, roots);
|
|
928
|
+
const sessionId = call.sessionId ?? opts.sessionId;
|
|
929
|
+
return {
|
|
930
|
+
toolUseId: call.toolUseId,
|
|
931
|
+
toolName: call.toolName,
|
|
932
|
+
input: call.input,
|
|
933
|
+
content: call.content,
|
|
934
|
+
...(call.body !== undefined ? { body: call.body } : {}),
|
|
935
|
+
isError: call.isError,
|
|
936
|
+
...(sessionId !== undefined ? { sessionId } : {}),
|
|
937
|
+
origins: [...new Set(origins)],
|
|
938
|
+
...(status !== undefined ? { status } : {}),
|
|
939
|
+
...(call.turnNumber !== undefined ? { turnNumber: call.turnNumber } : {}),
|
|
940
|
+
...(hits.length > 0 ? { hits } : {}),
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
/**
|
|
944
|
+
* The spill tail tool-result-store puts after a result's head when it saved
|
|
945
|
+
* the output to a file (`storeAndPreview`: an optional part line, then
|
|
946
|
+
* `[truncated, full output at <path>]`), read from the END of `content` line
|
|
947
|
+
* by line — each pattern anchored to one short line, never run over the
|
|
948
|
+
* body. Returns the head and the path, or `undefined`. A tail saying the
|
|
949
|
+
* output could not be saved has no file to read.
|
|
950
|
+
*/
|
|
951
|
+
function spillTail(content) {
|
|
952
|
+
const last = content.lastIndexOf("\n");
|
|
953
|
+
if (last === -1)
|
|
954
|
+
return undefined;
|
|
955
|
+
const saved = /^\[truncated, full output at ([^\n]+)\]$/.exec(content.slice(last + 1))?.[1];
|
|
956
|
+
if (saved === undefined)
|
|
957
|
+
return undefined;
|
|
958
|
+
const prior = content.lastIndexOf("\n", last - 1);
|
|
959
|
+
const part = prior === -1 ? "" : content.slice(prior + 1, last);
|
|
960
|
+
const partLine = /^\[part 1 of \d+; Read .+ for part 2\]$/.test(part) ||
|
|
961
|
+
/^\[the rest could not also be saved in parts to Read one at a time: .*\]$/.test(part);
|
|
962
|
+
return { head: content.slice(0, partLine ? prior : last), saved };
|
|
963
|
+
}
|
|
964
|
+
/**
|
|
965
|
+
* The whole output of a call whose logged result is tool-result-store's
|
|
966
|
+
* preview of it, or `undefined` (design §5.2: grounding needs the bodies
|
|
967
|
+
* the run read). runtime-core logs, and shows the model, only the first 100
|
|
968
|
+
* lines or 10 KB of a result over the threshold, then the path the whole
|
|
969
|
+
* was saved at; the claim the model reads further down a long page (the
|
|
970
|
+
* preview tells it to `Read` the parts) is in that file, not in the log.
|
|
971
|
+
*
|
|
972
|
+
* Read back only where it can be nothing but this call's own output:
|
|
973
|
+
*
|
|
974
|
+
* - the path is `<toolResultsRoot>/<runId>/<toolUseId>.txt` (or the
|
|
975
|
+
* `<toolUseId>.<n>.txt` tool-result-store moves on to when the name is
|
|
976
|
+
* taken) — THIS call's id, one run directory down, under the root the
|
|
977
|
+
* runtime spills to; a logged result is the tool's output verbatim, so a
|
|
978
|
+
* page that ends with a forged tail naming some other file reads nothing;
|
|
979
|
+
* - the file is a regular file opened inside that root, no link followed
|
|
980
|
+
* (`@crewhaus/tool-safety/fs`), at most {@link MAX_SPILL_BYTES};
|
|
981
|
+
* - it begins with the head the log holds.
|
|
982
|
+
*
|
|
983
|
+
* Anything else — a file gone, a tenant root not passed here — leaves the
|
|
984
|
+
* preview as the record's text: fewer words, a claim further down unmet,
|
|
985
|
+
* never more than the run read.
|
|
986
|
+
*/
|
|
987
|
+
export function spilledOutput(content, toolUseId, opts = {}) {
|
|
988
|
+
const tail = spillTail(content);
|
|
989
|
+
if (tail === undefined || tail.head === "")
|
|
990
|
+
return undefined;
|
|
991
|
+
const workspace = path.resolve(opts.workspaceRoot ?? process.cwd());
|
|
992
|
+
const root = path.resolve(workspace, opts.toolResultsRoot ?? DEFAULT_TOOL_RESULTS_DIR);
|
|
993
|
+
const rel = path.relative(root, path.resolve(workspace, tail.saved));
|
|
994
|
+
const parts = rel.split(path.sep);
|
|
995
|
+
if (parts.length !== 2 || parts.some((part) => part === "" || part === ".."))
|
|
996
|
+
return undefined;
|
|
997
|
+
const name = parts[1];
|
|
998
|
+
const own = `${toolUseId}.`;
|
|
999
|
+
if (!name.startsWith(own) || !/^(?:\d{1,4}\.)?txt$/.test(name.slice(own.length))) {
|
|
1000
|
+
return undefined;
|
|
1001
|
+
}
|
|
1002
|
+
const read = openForReadSync(root, rel, { maxBytes: MAX_SPILL_BYTES, followLeafSymlink: false });
|
|
1003
|
+
if (!read.ok || read.truncated)
|
|
1004
|
+
return undefined;
|
|
1005
|
+
const whole = read.text;
|
|
1006
|
+
return whole.startsWith(tail.head) && whole.length > tail.head.length ? whole : undefined;
|
|
1007
|
+
}
|
|
1008
|
+
/** A tool result's text: a string as is, the text blocks of an array joined. */
|
|
1009
|
+
function resultText(content) {
|
|
1010
|
+
if (typeof content === "string")
|
|
1011
|
+
return content;
|
|
1012
|
+
if (!Array.isArray(content))
|
|
1013
|
+
return "";
|
|
1014
|
+
const texts = [];
|
|
1015
|
+
for (const block of content) {
|
|
1016
|
+
if (isRecordObject(block) && block["type"] === "text" && typeof block["text"] === "string") {
|
|
1017
|
+
texts.push(block["text"]);
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
return texts.join("\n");
|
|
1021
|
+
}
|
|
1022
|
+
function turnOf(payload) {
|
|
1023
|
+
const turn = payload["turnNumber"];
|
|
1024
|
+
return typeof turn === "number" && Number.isInteger(turn) && turn >= 0 ? turn : undefined;
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Records for every completed tool call in a transcript, in call order. A
|
|
1028
|
+
* `tool_use` with no `tool_result` yet is not a record (the call never
|
|
1029
|
+
* finished); a result whose call is not in the transcript is skipped. An id
|
|
1030
|
+
* that appears twice (a provider that reuses ids) pairs its LAST call with
|
|
1031
|
+
* its LAST result, as the log walk does (`evidenceFromSessionLog`). A
|
|
1032
|
+
* result the runtime spilled is read whole (`spilledOutput`).
|
|
1033
|
+
*/
|
|
1034
|
+
export function evidenceFromMessages(messages, opts = {}) {
|
|
1035
|
+
const order = [];
|
|
1036
|
+
const uses = new Map();
|
|
1037
|
+
const results = new Map();
|
|
1038
|
+
for (const message of messages) {
|
|
1039
|
+
if (!Array.isArray(message.content))
|
|
1040
|
+
continue;
|
|
1041
|
+
for (const block of message.content) {
|
|
1042
|
+
if (!isRecordObject(block))
|
|
1043
|
+
continue;
|
|
1044
|
+
if (block["type"] === "tool_use" && typeof block["id"] === "string") {
|
|
1045
|
+
if (!uses.has(block["id"]))
|
|
1046
|
+
order.push(block["id"]);
|
|
1047
|
+
uses.set(block["id"], {
|
|
1048
|
+
name: typeof block["name"] === "string" ? block["name"] : "unknown",
|
|
1049
|
+
input: block["input"],
|
|
1050
|
+
});
|
|
1051
|
+
}
|
|
1052
|
+
else if (block["type"] === "tool_result" && typeof block["tool_use_id"] === "string") {
|
|
1053
|
+
results.set(block["tool_use_id"], {
|
|
1054
|
+
content: resultText(block["content"]),
|
|
1055
|
+
isError: block["is_error"] === true,
|
|
1056
|
+
});
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
const out = [];
|
|
1061
|
+
for (const id of order) {
|
|
1062
|
+
const use = uses.get(id);
|
|
1063
|
+
const result = results.get(id);
|
|
1064
|
+
if (result === undefined)
|
|
1065
|
+
continue;
|
|
1066
|
+
const body = spilledOutput(result.content, id, opts);
|
|
1067
|
+
out.push(recordFromCall({
|
|
1068
|
+
toolUseId: id,
|
|
1069
|
+
toolName: use.name,
|
|
1070
|
+
input: use.input,
|
|
1071
|
+
...result,
|
|
1072
|
+
...(body !== undefined ? { body } : {}),
|
|
1073
|
+
}, opts));
|
|
1074
|
+
}
|
|
1075
|
+
return out;
|
|
1076
|
+
}
|
|
1077
|
+
/**
|
|
1078
|
+
* Records for every completed tool call in a session's event log, child
|
|
1079
|
+
* sessions included, in call order: a child's calls are placed where its
|
|
1080
|
+
* `sub_agent_start` bracket sits in the parent, because that is when they
|
|
1081
|
+
* ran. The child walk follows the brackets `continuity-store`'s
|
|
1082
|
+
* `resolveEvidence` follows — `sub_agent_start` payloads name a
|
|
1083
|
+
* `childSessionId`, a session is read once, the depth is bounded — so a
|
|
1084
|
+
* researcher child's fetches ground the parent's answer. (It is not that
|
|
1085
|
+
* function: this reads every call of the tree once, in call order, where
|
|
1086
|
+
* `resolveEvidence` looks one id up breadth-first.)
|
|
1087
|
+
*
|
|
1088
|
+
* One id logged more than once in a session pairs its LAST `tool_use` with
|
|
1089
|
+
* its LAST `tool_result`, at the position it first appeared — the rule
|
|
1090
|
+
* continuity-store's `scanSession` applies, so `PlanComplete` and the
|
|
1091
|
+
* `checked` rung read one log the same way (design §6.3). An approval
|
|
1092
|
+
* replay logs the `tool_use` twice (the second line records the input that
|
|
1093
|
+
* RAN) and the result once; a provider that reuses ids (an
|
|
1094
|
+
* OpenAI-compatible server's `functions.<name>:<n>`) logs both twice, and
|
|
1095
|
+
* the last call is paired with its own result rather than with the first
|
|
1096
|
+
* call's. A result the runtime spilled is read whole (`spilledOutput`).
|
|
1097
|
+
* `turnNumber` is carried when a payload records one. A missing log
|
|
1098
|
+
* — the session's or a child's — yields no records, never a throw. A log
|
|
1099
|
+
* that cannot be read to its end (`@crewhaus/event-log` throws at a
|
|
1100
|
+
* malformed line — a crash mid-append leaves one — or refuses a link at
|
|
1101
|
+
* the log's name) keeps the records read before that point and reports it
|
|
1102
|
+
* through `onUnreadable`: fewer records fails closed — a claim whose
|
|
1103
|
+
* evidence was lost is unmet — where a throw would fail OPEN, every later
|
|
1104
|
+
* turn of the session served `unverified` because one child's log is torn.
|
|
1105
|
+
* Reading never creates the log directory.
|
|
1106
|
+
*/
|
|
1107
|
+
export async function evidenceFromSessionLog(opts) {
|
|
1108
|
+
if (!SESSION_ID_REGEX.test(opts.sessionId)) {
|
|
1109
|
+
throw new ProvabilityError("config", `provability: invalid sessionId "${opts.sessionId}" — expected sess_<16 hex>`);
|
|
1110
|
+
}
|
|
1111
|
+
const rootDir = opts.rootDir ?? DEFAULT_ROOT_DIR;
|
|
1112
|
+
const includeChildren = opts.includeChildren ?? true;
|
|
1113
|
+
const maxDepth = opts.maxDepth ?? DEFAULT_MAX_DEPTH;
|
|
1114
|
+
const roots = {
|
|
1115
|
+
...(opts.workspaceRoot !== undefined ? { workspaceRoot: opts.workspaceRoot } : {}),
|
|
1116
|
+
...(opts.toolResultsRoot !== undefined ? { toolResultsRoot: opts.toolResultsRoot } : {}),
|
|
1117
|
+
...(opts.inputSchemaOf !== undefined ? { inputSchemaOf: opts.inputSchemaOf } : {}),
|
|
1118
|
+
};
|
|
1119
|
+
const visited = new Set();
|
|
1120
|
+
const out = [];
|
|
1121
|
+
const walk = async (sessionId, depth) => {
|
|
1122
|
+
if (visited.has(sessionId))
|
|
1123
|
+
return;
|
|
1124
|
+
visited.add(sessionId);
|
|
1125
|
+
// openEventLog creates its root directory; a read must not.
|
|
1126
|
+
if (!existsSync(path.join(rootDir, `${sessionId}.jsonl`)))
|
|
1127
|
+
return;
|
|
1128
|
+
const log = await openEventLog(sessionId, { rootDir });
|
|
1129
|
+
const slots = [];
|
|
1130
|
+
const uses = new Map();
|
|
1131
|
+
const results = new Map();
|
|
1132
|
+
try {
|
|
1133
|
+
for await (const event of log.read()) {
|
|
1134
|
+
if (!isRecordObject(event.payload))
|
|
1135
|
+
continue;
|
|
1136
|
+
const p = event.payload;
|
|
1137
|
+
if (event.kind === "tool_use" && typeof p["id"] === "string") {
|
|
1138
|
+
const turnNumber = turnOf(p);
|
|
1139
|
+
const prior = uses.get(p["id"]);
|
|
1140
|
+
if (prior === undefined)
|
|
1141
|
+
slots.push({ kind: "call", id: p["id"] });
|
|
1142
|
+
uses.set(p["id"], {
|
|
1143
|
+
name: typeof p["name"] === "string" ? p["name"] : (prior?.name ?? "unknown"),
|
|
1144
|
+
input: p["input"],
|
|
1145
|
+
...(turnNumber !== undefined
|
|
1146
|
+
? { turnNumber }
|
|
1147
|
+
: prior?.turnNumber !== undefined
|
|
1148
|
+
? { turnNumber: prior.turnNumber }
|
|
1149
|
+
: {}),
|
|
1150
|
+
});
|
|
1151
|
+
}
|
|
1152
|
+
else if (event.kind === "tool_result" && typeof p["toolUseId"] === "string") {
|
|
1153
|
+
const turnNumber = turnOf(p);
|
|
1154
|
+
results.set(p["toolUseId"], {
|
|
1155
|
+
content: resultText(p["content"]),
|
|
1156
|
+
isError: p["isError"] === true,
|
|
1157
|
+
...(turnNumber !== undefined ? { turnNumber } : {}),
|
|
1158
|
+
});
|
|
1159
|
+
}
|
|
1160
|
+
else if (event.kind === "sub_agent_start") {
|
|
1161
|
+
const child = p["childSessionId"];
|
|
1162
|
+
if (typeof child === "string" && SESSION_ID_REGEX.test(child)) {
|
|
1163
|
+
slots.push({ kind: "child", sessionId: child });
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
catch (err) {
|
|
1169
|
+
opts.onUnreadable?.({
|
|
1170
|
+
sessionId,
|
|
1171
|
+
message: err instanceof Error ? err.message : String(err),
|
|
1172
|
+
});
|
|
1173
|
+
}
|
|
1174
|
+
finally {
|
|
1175
|
+
await log.close();
|
|
1176
|
+
}
|
|
1177
|
+
for (const slot of slots) {
|
|
1178
|
+
if (slot.kind === "child") {
|
|
1179
|
+
if (includeChildren && depth < maxDepth)
|
|
1180
|
+
await walk(slot.sessionId, depth + 1);
|
|
1181
|
+
continue;
|
|
1182
|
+
}
|
|
1183
|
+
const use = uses.get(slot.id);
|
|
1184
|
+
const result = results.get(slot.id);
|
|
1185
|
+
if (use === undefined || result === undefined)
|
|
1186
|
+
continue;
|
|
1187
|
+
const turnNumber = use.turnNumber ?? result.turnNumber;
|
|
1188
|
+
const body = spilledOutput(result.content, slot.id, roots);
|
|
1189
|
+
out.push(recordFromCall({
|
|
1190
|
+
toolUseId: slot.id,
|
|
1191
|
+
toolName: use.name,
|
|
1192
|
+
input: use.input,
|
|
1193
|
+
content: result.content,
|
|
1194
|
+
...(body !== undefined ? { body } : {}),
|
|
1195
|
+
isError: result.isError,
|
|
1196
|
+
sessionId,
|
|
1197
|
+
...(turnNumber !== undefined ? { turnNumber } : {}),
|
|
1198
|
+
}, roots));
|
|
1199
|
+
}
|
|
1200
|
+
};
|
|
1201
|
+
await walk(opts.sessionId, 0);
|
|
1202
|
+
return out;
|
|
1203
|
+
}
|
|
1204
|
+
/**
|
|
1205
|
+
* runtime-core logs a non-string `tool_result` (image or multi-block
|
|
1206
|
+
* content) as a summary — `[2 text blocks, 1834 chars]` — not the text. When
|
|
1207
|
+
* the same call also arrives from the transcript, the transcript's text wins
|
|
1208
|
+
* over that placeholder.
|
|
1209
|
+
*/
|
|
1210
|
+
const LOGGED_BLOCK_SUMMARY = /^\[(?:\d+ (?:image|text|other) blocks?)(?:, \d+ (?:image|text|other) blocks?)*, \d+ chars\]$/;
|
|
1211
|
+
function mergeRecords(first, later) {
|
|
1212
|
+
const takeLater = (first.content === "" && later.content !== "") ||
|
|
1213
|
+
(LOGGED_BLOCK_SUMMARY.test(first.content) && !LOGGED_BLOCK_SUMMARY.test(later.content));
|
|
1214
|
+
const body = takeLater ? later : first;
|
|
1215
|
+
const hits = first.hits ?? later.hits;
|
|
1216
|
+
const sessionId = first.sessionId ?? later.sessionId;
|
|
1217
|
+
const turnNumber = first.turnNumber ?? later.turnNumber;
|
|
1218
|
+
const whole = body.body ?? (takeLater ? first.body : later.body);
|
|
1219
|
+
return {
|
|
1220
|
+
toolUseId: first.toolUseId,
|
|
1221
|
+
toolName: first.toolName === "unknown" ? later.toolName : first.toolName,
|
|
1222
|
+
input: first.input,
|
|
1223
|
+
content: body.content,
|
|
1224
|
+
...(whole !== undefined ? { body: whole } : {}),
|
|
1225
|
+
isError: body.isError,
|
|
1226
|
+
...(sessionId !== undefined ? { sessionId } : {}),
|
|
1227
|
+
origins: [...new Set([...first.origins, ...later.origins])],
|
|
1228
|
+
...(body.status !== undefined ? { status: body.status } : {}),
|
|
1229
|
+
...(turnNumber !== undefined ? { turnNumber } : {}),
|
|
1230
|
+
...(takeLater && later.hits !== undefined
|
|
1231
|
+
? { hits: later.hits }
|
|
1232
|
+
: hits !== undefined
|
|
1233
|
+
? { hits }
|
|
1234
|
+
: {}),
|
|
1235
|
+
};
|
|
1236
|
+
}
|
|
1237
|
+
/** A view of `record` as one hit's own evidence: the hit's text, under the hit's origin. */
|
|
1238
|
+
function hitView(record, hit) {
|
|
1239
|
+
// The hit's text alone: never the whole output it was read from.
|
|
1240
|
+
const { hits: _hits, body: _body, status: _status, ...call } = record;
|
|
1241
|
+
return { ...call, content: hit.content, origins: [hit.origin] };
|
|
1242
|
+
}
|
|
1243
|
+
/** One call's key: its session and its id, the pair that names it (design §5.2). */
|
|
1244
|
+
const callKey = (sessionId, toolUseId) => `${sessionId ?? ""}\u0000${toolUseId}`;
|
|
1245
|
+
/** Did two records come from the same call: one tool, one input? */
|
|
1246
|
+
function sameCall(a, b) {
|
|
1247
|
+
if (a.toolName !== b.toolName)
|
|
1248
|
+
return false;
|
|
1249
|
+
try {
|
|
1250
|
+
return JSON.stringify(a.input) === JSON.stringify(b.input);
|
|
1251
|
+
}
|
|
1252
|
+
catch {
|
|
1253
|
+
return false;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
/**
|
|
1257
|
+
* The index over `records`, one record per call: a call is its session and
|
|
1258
|
+
* its `toolUseId` together (design §5.2: the index is keyed per call), so
|
|
1259
|
+
* the log's record and the transcript's of ONE call merge into one, in the
|
|
1260
|
+
* order each call first appears, and two sessions' calls never do. The
|
|
1261
|
+
* log's record wins where the two disagree, except where the log holds
|
|
1262
|
+
* only runtime-core's summary of block content and the transcript holds
|
|
1263
|
+
* the text.
|
|
1264
|
+
*
|
|
1265
|
+
* An id is unique only within a conversation: an OpenAI-compatible server
|
|
1266
|
+
* numbers its calls per conversation (`functions.<name>:<n>`, see
|
|
1267
|
+
* `evidenceFromSessionLog`), and every sub-agent child is a fresh one, so a
|
|
1268
|
+
* parent's first fetch and its child's share an id. Merged by id alone, the
|
|
1269
|
+
* two became one record — one call's body, indexed under both calls' URLs:
|
|
1270
|
+
* a claim the other page contradicts grounded, and the honest one failed.
|
|
1271
|
+
* A record that names no session is the parent's (see
|
|
1272
|
+
* {@link EvidenceIndexOptions.sessionId}); where the parent has no call by
|
|
1273
|
+
* its id, it is the twin of the one call that has its tool and its input,
|
|
1274
|
+
* and otherwise a call of its own.
|
|
1275
|
+
*
|
|
1276
|
+
* `byToolUseId` answers a `tool:<id>` the way continuity-store's
|
|
1277
|
+
* `resolveEvidence` does — the parent's call first, then a child's — and
|
|
1278
|
+
* an id only several children share (and the parent does not) is
|
|
1279
|
+
* ambiguous: `undefined`, with every candidate on `callsById`, so the
|
|
1280
|
+
* gate says so rather than grade one sub-agent's result for another's.
|
|
1281
|
+
*
|
|
1282
|
+
* `bySource` takes a normalized origin; a raw URL is normalized for the
|
|
1283
|
+
* caller. Error records are indexed like any other — the verifiers reject
|
|
1284
|
+
* them by reason, so "fetched, but it failed" reads differently from "never
|
|
1285
|
+
* fetched".
|
|
1286
|
+
*/
|
|
1287
|
+
export function buildEvidenceIndex(records, opts = {}) {
|
|
1288
|
+
const list = [...records];
|
|
1289
|
+
const root = opts.sessionId ?? list.find((record) => record.sessionId !== undefined)?.sessionId;
|
|
1290
|
+
const byKey = new Map();
|
|
1291
|
+
/** Each id's calls, by key, in the order they first appear. */
|
|
1292
|
+
const keysById = new Map();
|
|
1293
|
+
const order = [];
|
|
1294
|
+
const keyFor = (record) => {
|
|
1295
|
+
if (record.sessionId !== undefined)
|
|
1296
|
+
return callKey(record.sessionId, record.toolUseId);
|
|
1297
|
+
const own = callKey(root, record.toolUseId);
|
|
1298
|
+
const keys = keysById.get(record.toolUseId) ?? [];
|
|
1299
|
+
if (keys.length === 0 || (keys.length === 1 && keys[0] === own))
|
|
1300
|
+
return own;
|
|
1301
|
+
const twin = keys.find((key) => sameCall(byKey.get(key), record));
|
|
1302
|
+
// The parent's call by this id — unless the parent was guessed wrong:
|
|
1303
|
+
// another session holds this very call, and the guessed one's differs.
|
|
1304
|
+
if (keys.includes(own) && (twin === undefined || twin === own))
|
|
1305
|
+
return own;
|
|
1306
|
+
return twin ?? own;
|
|
1307
|
+
};
|
|
1308
|
+
for (const record of list) {
|
|
1309
|
+
const key = keyFor(record);
|
|
1310
|
+
const prior = byKey.get(key);
|
|
1311
|
+
if (prior === undefined) {
|
|
1312
|
+
byKey.set(key, record);
|
|
1313
|
+
order.push(key);
|
|
1314
|
+
const keys = keysById.get(record.toolUseId);
|
|
1315
|
+
if (keys === undefined)
|
|
1316
|
+
keysById.set(record.toolUseId, [key]);
|
|
1317
|
+
else
|
|
1318
|
+
keys.push(key);
|
|
1319
|
+
}
|
|
1320
|
+
else {
|
|
1321
|
+
byKey.set(key, mergeRecords(prior, record));
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
const all = order.map((key) => byKey.get(key));
|
|
1325
|
+
const callsOf = (toolUseId) => {
|
|
1326
|
+
const keys = keysById.get(toolUseId) ?? [];
|
|
1327
|
+
const own = callKey(root, toolUseId);
|
|
1328
|
+
// The parent's first, then the children's in the order they ran.
|
|
1329
|
+
return [...keys.filter((key) => key === own), ...keys.filter((key) => key !== own)].map((key) => byKey.get(key));
|
|
1330
|
+
};
|
|
1331
|
+
const sources = new Map();
|
|
1332
|
+
const add = (origin, record) => {
|
|
1333
|
+
const list = sources.get(origin);
|
|
1334
|
+
if (list === undefined)
|
|
1335
|
+
sources.set(origin, [record]);
|
|
1336
|
+
else
|
|
1337
|
+
list.push(record);
|
|
1338
|
+
};
|
|
1339
|
+
for (const record of all) {
|
|
1340
|
+
for (const origin of new Set(record.origins))
|
|
1341
|
+
add(origin, record);
|
|
1342
|
+
for (const hit of record.hits ?? [])
|
|
1343
|
+
add(hit.origin, hitView(record, hit));
|
|
1344
|
+
}
|
|
1345
|
+
return {
|
|
1346
|
+
bySource(origin) {
|
|
1347
|
+
const direct = sources.get(origin);
|
|
1348
|
+
if (direct !== undefined)
|
|
1349
|
+
return direct;
|
|
1350
|
+
const normalized = normalizeUrl(origin);
|
|
1351
|
+
return (normalized !== undefined ? sources.get(normalized) : undefined) ?? [];
|
|
1352
|
+
},
|
|
1353
|
+
byToolUseId(toolUseId) {
|
|
1354
|
+
const own = byKey.get(callKey(root, toolUseId));
|
|
1355
|
+
if (own !== undefined)
|
|
1356
|
+
return own;
|
|
1357
|
+
const calls = callsOf(toolUseId);
|
|
1358
|
+
return calls.length === 1 ? calls[0] : undefined;
|
|
1359
|
+
},
|
|
1360
|
+
callsById(toolUseId) {
|
|
1361
|
+
return callsOf(toolUseId);
|
|
1362
|
+
},
|
|
1363
|
+
all() {
|
|
1364
|
+
return all;
|
|
1365
|
+
},
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
/**
|
|
1369
|
+
* The index restricted to calls made in `turnNumbers` (`scope: turn`). A
|
|
1370
|
+
* record with no `turnNumber` is dropped: under a turn scope, evidence whose
|
|
1371
|
+
* turn is unknown cannot be shown to be this turn's.
|
|
1372
|
+
*/
|
|
1373
|
+
export function restrictToTurns(index, turnNumbers) {
|
|
1374
|
+
const keep = (record) => record.turnNumber !== undefined && turnNumbers.has(record.turnNumber);
|
|
1375
|
+
return {
|
|
1376
|
+
bySource: (origin) => index.bySource(origin).filter(keep),
|
|
1377
|
+
byToolUseId: (toolUseId) => {
|
|
1378
|
+
const record = index.byToolUseId(toolUseId);
|
|
1379
|
+
if (record !== undefined && keep(record))
|
|
1380
|
+
return record;
|
|
1381
|
+
if (index.callsById === undefined)
|
|
1382
|
+
return undefined;
|
|
1383
|
+
// The id's other calls, in scope: the one left answers for it, and
|
|
1384
|
+
// several stay ambiguous (`buildEvidenceIndex`).
|
|
1385
|
+
const kept = index.callsById(toolUseId).filter((call) => call !== record && keep(call));
|
|
1386
|
+
return kept.length === 1 ? kept[0] : undefined;
|
|
1387
|
+
},
|
|
1388
|
+
callsById: (toolUseId) => (index.callsById?.(toolUseId) ?? []).filter(keep),
|
|
1389
|
+
all: () => index.all().filter(keep),
|
|
1390
|
+
};
|
|
1391
|
+
}
|