@kyo-so/cli 0.12.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.ja.md +64 -34
- package/README.md +64 -34
- package/README.zh-CN.md +64 -34
- package/dist/acp/ndJsonLineLimit.d.ts +6 -0
- package/dist/acp/normalize.d.ts +1 -0
- package/dist/acp/prompts.d.ts +1 -0
- package/dist/bin/kyoso.js +12933 -11867
- package/dist/cli/pluginRuntimeContract.d.ts +8 -8
- package/dist/config/schema.d.ts +1 -0
- package/dist/core/constants.d.ts +3 -2
- package/dist/core/modelExecutionIdentity.d.ts +9 -0
- package/dist/core/requestFingerprint.d.ts +2 -2
- package/dist/core/reviewBudget.d.ts +28 -5
- package/dist/core/types.d.ts +44 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +781 -129
- package/dist/judge/anthropic.d.ts +2 -0
- package/dist/judge/openai.d.ts +2 -0
- package/dist/judge/provider.d.ts +9 -1
- package/dist/utils/env.d.ts +12 -2
- package/examples/codex-config.toml +1 -1
- package/package.json +6 -3
- package/scripts/review-budget-report.mjs +1114 -0
|
@@ -0,0 +1,1114 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { spawn } from "node:child_process";
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { constants as fsConstants, realpathSync } from "node:fs";
|
|
6
|
+
import { lstat, open, opendir } from "node:fs/promises";
|
|
7
|
+
import { isAbsolute, resolve } from "node:path";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
|
|
10
|
+
const MODEL_CALL_KINDS = new Set(["primary", "verifier", "judge"]);
|
|
11
|
+
const AGENT_STREAM_CALL_KINDS = new Set(["primary", "verifier"]);
|
|
12
|
+
const OPTIONAL_CALL_KINDS = new Set(["verifier", "judge"]);
|
|
13
|
+
const PROVIDER_ROUTES = new Set([
|
|
14
|
+
"codex_default",
|
|
15
|
+
"claude_default",
|
|
16
|
+
"openrouter",
|
|
17
|
+
"openai",
|
|
18
|
+
"anthropic",
|
|
19
|
+
]);
|
|
20
|
+
const TOKEN_USAGE_KEYS = [
|
|
21
|
+
"totalTokens",
|
|
22
|
+
"inputTokens",
|
|
23
|
+
"outputTokens",
|
|
24
|
+
"thoughtTokens",
|
|
25
|
+
"cachedReadTokens",
|
|
26
|
+
"cachedWriteTokens",
|
|
27
|
+
];
|
|
28
|
+
const CREDENTIAL_PATTERNS = [
|
|
29
|
+
/\bsk-(?:proj-|ant-)?[A-Za-z0-9_-]{8,}\b/i,
|
|
30
|
+
/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{8,}\b/i,
|
|
31
|
+
/\bgithub_pat_[A-Za-z0-9_]{20,}\b/i,
|
|
32
|
+
/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/,
|
|
33
|
+
/\bAIza[0-9A-Za-z_-]{35}\b/,
|
|
34
|
+
/\bxox[baprs]-[A-Za-z0-9-]{8,}\b/i,
|
|
35
|
+
/\bsk_(?:live|test)_[A-Za-z0-9]{8,}\b/i,
|
|
36
|
+
/\bglpat-[A-Za-z0-9_-]{8,}\b/i,
|
|
37
|
+
/\bnpm_[A-Za-z0-9]{16,}\b/i,
|
|
38
|
+
/\bpypi-[A-Za-z0-9_-]{16,}\b/i,
|
|
39
|
+
/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/,
|
|
40
|
+
/\b(?:bearer|basic)\s+[A-Za-z0-9._~+\/-]{8,}\b/i,
|
|
41
|
+
new RegExp(
|
|
42
|
+
["-{5}BEGIN ", "(?:RSA |OPENSSH |DSA |EC |PGP )?", "PRIVATE KEY-{5}"].join(
|
|
43
|
+
"",
|
|
44
|
+
),
|
|
45
|
+
"i",
|
|
46
|
+
),
|
|
47
|
+
];
|
|
48
|
+
const UNSAFE_METADATA_PATTERN =
|
|
49
|
+
/(?:https?|wss?):\/\/|\b(?:api[_-]?key|access[_-]?key|authorization|base[_-]?url|client[_-]?secret|credential|private[_ -]?key|secret|token|password)\b|[{}=]/i;
|
|
50
|
+
const MAX_METADATA_CHARS = 160;
|
|
51
|
+
const SCRIPT_PATH = fileURLToPath(import.meta.url);
|
|
52
|
+
const INTERNAL_WORKER_FLAG = "--internal-anchored-worker";
|
|
53
|
+
const MAX_WORKER_REPORT_BYTES = 64 * 1024 * 1024;
|
|
54
|
+
const MAX_WORKER_ERROR_BYTES = 64 * 1024;
|
|
55
|
+
const DEFAULT_TRACE_ROOT_FS = Object.freeze({
|
|
56
|
+
lstat,
|
|
57
|
+
open,
|
|
58
|
+
});
|
|
59
|
+
export const REPORT_LIMITS = Object.freeze({
|
|
60
|
+
jsonlFiles: 10_000,
|
|
61
|
+
directories: 10_000,
|
|
62
|
+
directoryEntries: 100_000,
|
|
63
|
+
traceFileBytes: 16 * 1024 * 1024,
|
|
64
|
+
totalTraceBytes: 256 * 1024 * 1024,
|
|
65
|
+
jsonlLines: 1_000_000,
|
|
66
|
+
jsonlLineBytes: 1024 * 1024,
|
|
67
|
+
parsedEvents: 250_000,
|
|
68
|
+
completedCalls: 100_000,
|
|
69
|
+
completedReviews: 100_000,
|
|
70
|
+
outputWarningEvents: 100_000,
|
|
71
|
+
correlationKeys: 100_000,
|
|
72
|
+
executionGroups: 10_000,
|
|
73
|
+
distinctReasons: 10_000,
|
|
74
|
+
});
|
|
75
|
+
const WORKER_FAILURE_MESSAGE = "Anchored trace worker rejected the input.";
|
|
76
|
+
const SAFE_WORKER_ERROR_MESSAGES = new Set([
|
|
77
|
+
"The trace directory identity is invalid.",
|
|
78
|
+
"Trace directory changed during traversal; report aborted.",
|
|
79
|
+
"Trace directories exceed the report input limit.",
|
|
80
|
+
"Trace directory entries exceed the report input limit.",
|
|
81
|
+
"A trace directory entry name is invalid.",
|
|
82
|
+
"A trace directory does not expose a stable file identity.",
|
|
83
|
+
"A trace file exceeds the per-file byte limit.",
|
|
84
|
+
"Trace files exceed the report input limit.",
|
|
85
|
+
"Trace files exceed the total report byte limit.",
|
|
86
|
+
"JSONL lines exceed the report input limit.",
|
|
87
|
+
"A JSONL line exceeds the report input byte limit.",
|
|
88
|
+
"Trace events exceed the report input limit.",
|
|
89
|
+
"Secure trace-file open capability is unavailable.",
|
|
90
|
+
"A trace file changed after discovery; report aborted.",
|
|
91
|
+
"A trace file exceeds its discovered byte limit.",
|
|
92
|
+
"A trace file changed while reading; report aborted.",
|
|
93
|
+
"Output warning events exceed the report input limit.",
|
|
94
|
+
"Completed reviews exceed the report input limit.",
|
|
95
|
+
"Completed model calls exceed the report input limit.",
|
|
96
|
+
"Execution groups exceed the report input limit.",
|
|
97
|
+
"Distinct trace reasons exceed the report input limit.",
|
|
98
|
+
"Trace correlation keys exceed the report input limit.",
|
|
99
|
+
]);
|
|
100
|
+
|
|
101
|
+
export async function buildReviewBudgetReport(traceDir) {
|
|
102
|
+
const rootIdentity = await inspectTraceRoot(traceDir);
|
|
103
|
+
return runAnchoredTraceWorker(traceDir, rootIdentity);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function runAnchoredTraceWorker(traceDir, rootIdentity) {
|
|
107
|
+
return new Promise((resolveReport, rejectReport) => {
|
|
108
|
+
const child = spawn(
|
|
109
|
+
process.execPath,
|
|
110
|
+
[
|
|
111
|
+
SCRIPT_PATH,
|
|
112
|
+
INTERNAL_WORKER_FLAG,
|
|
113
|
+
String(rootIdentity.device),
|
|
114
|
+
String(rootIdentity.inode),
|
|
115
|
+
],
|
|
116
|
+
{
|
|
117
|
+
cwd: traceDir,
|
|
118
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
119
|
+
},
|
|
120
|
+
);
|
|
121
|
+
const outputChunks = [];
|
|
122
|
+
const errorChunks = [];
|
|
123
|
+
let outputBytes = 0;
|
|
124
|
+
let errorBytes = 0;
|
|
125
|
+
let outputExceeded = false;
|
|
126
|
+
let errorExceeded = false;
|
|
127
|
+
let settled = false;
|
|
128
|
+
|
|
129
|
+
child.stdout.on("data", (chunk) => {
|
|
130
|
+
outputBytes += chunk.length;
|
|
131
|
+
if (outputBytes > MAX_WORKER_REPORT_BYTES) {
|
|
132
|
+
outputExceeded = true;
|
|
133
|
+
child.kill();
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
outputChunks.push(chunk);
|
|
137
|
+
});
|
|
138
|
+
child.stderr.on("data", (chunk) => {
|
|
139
|
+
errorBytes += chunk.length;
|
|
140
|
+
if (errorBytes > MAX_WORKER_ERROR_BYTES) {
|
|
141
|
+
errorExceeded = true;
|
|
142
|
+
child.kill();
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
errorChunks.push(chunk);
|
|
146
|
+
});
|
|
147
|
+
child.once("error", () => {
|
|
148
|
+
if (settled) return;
|
|
149
|
+
settled = true;
|
|
150
|
+
rejectReport(new Error(WORKER_FAILURE_MESSAGE));
|
|
151
|
+
});
|
|
152
|
+
child.once("close", (code) => {
|
|
153
|
+
if (settled) return;
|
|
154
|
+
settled = true;
|
|
155
|
+
if (outputExceeded || errorExceeded) {
|
|
156
|
+
rejectReport(new Error(WORKER_FAILURE_MESSAGE));
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
if (code !== 0) {
|
|
160
|
+
const workerMessage = Buffer.concat(errorChunks, errorBytes)
|
|
161
|
+
.toString("utf8")
|
|
162
|
+
.trim();
|
|
163
|
+
rejectReport(
|
|
164
|
+
new Error(
|
|
165
|
+
SAFE_WORKER_ERROR_MESSAGES.has(workerMessage)
|
|
166
|
+
? workerMessage
|
|
167
|
+
: WORKER_FAILURE_MESSAGE,
|
|
168
|
+
),
|
|
169
|
+
);
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
const report = JSON.parse(
|
|
174
|
+
Buffer.concat(outputChunks, outputBytes).toString("utf8"),
|
|
175
|
+
);
|
|
176
|
+
if (!isRecord(report) || !isRecord(report.source)) {
|
|
177
|
+
throw new Error(WORKER_FAILURE_MESSAGE);
|
|
178
|
+
}
|
|
179
|
+
resolveReport(report);
|
|
180
|
+
} catch {
|
|
181
|
+
rejectReport(new Error(WORKER_FAILURE_MESSAGE));
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function parseReportArgs(argv) {
|
|
188
|
+
let traceDir;
|
|
189
|
+
let json = false;
|
|
190
|
+
let help = false;
|
|
191
|
+
|
|
192
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
193
|
+
const argument = argv[index];
|
|
194
|
+
if (argument === "--json") {
|
|
195
|
+
json = true;
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
if (argument === "--help" || argument === "-h") {
|
|
199
|
+
help = true;
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
if (argument === "--trace-dir") {
|
|
203
|
+
if (traceDir !== undefined) {
|
|
204
|
+
throw new Error("--trace-dir may be specified only once.");
|
|
205
|
+
}
|
|
206
|
+
const value = argv[index + 1];
|
|
207
|
+
if (!value || value.startsWith("--")) {
|
|
208
|
+
throw new Error("Missing value for --trace-dir.");
|
|
209
|
+
}
|
|
210
|
+
traceDir = value;
|
|
211
|
+
index += 1;
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
throw new Error(`Unknown argument: ${argument}`);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (help) return { help: true, json, traceDir };
|
|
218
|
+
if (!traceDir) throw new Error("--trace-dir is required.");
|
|
219
|
+
if (!isAbsolute(traceDir)) {
|
|
220
|
+
throw new Error("--trace-dir must be an absolute path.");
|
|
221
|
+
}
|
|
222
|
+
return { help: false, json, traceDir };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
export function renderHumanReport(report) {
|
|
226
|
+
return [
|
|
227
|
+
"Kyoso review budget report",
|
|
228
|
+
`Trace files: ${report.source.jsonlFiles}`,
|
|
229
|
+
`Completed reviews: ${report.reviews.completed}`,
|
|
230
|
+
`Completed model-call events: ${report.calls.completed}`,
|
|
231
|
+
`Output-signal-eligible calls: ${report.calls.outputSignalEligible}`,
|
|
232
|
+
`Normal-path model calls: ${report.calls.normalPath}`,
|
|
233
|
+
`Output warning events: ${report.outputSignals.agentOutputWarning.events}`,
|
|
234
|
+
`Correlated output-warning calls: ${report.outputSignals.agentOutputWarning.calls}`,
|
|
235
|
+
`Uncorrelated output-warning events: ${report.outputSignals.agentOutputWarning.uncorrelatedEvents}`,
|
|
236
|
+
`Output-limit calls: ${report.outputSignals.agentOutputLimit.calls}`,
|
|
237
|
+
"Token and cost values are not estimated from bytes.",
|
|
238
|
+
].join("\n");
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export async function inspectTraceRoot(
|
|
242
|
+
traceDir,
|
|
243
|
+
fileSystem = DEFAULT_TRACE_ROOT_FS,
|
|
244
|
+
) {
|
|
245
|
+
if (typeof traceDir !== "string" || !isAbsolute(traceDir)) {
|
|
246
|
+
throw new Error("--trace-dir must be an absolute path.");
|
|
247
|
+
}
|
|
248
|
+
const rootInfo = await fileSystem.lstat(traceDir, { bigint: true });
|
|
249
|
+
if (rootInfo.isSymbolicLink() || !rootInfo.isDirectory()) {
|
|
250
|
+
throw new Error("--trace-dir must name a real directory, not a symlink.");
|
|
251
|
+
}
|
|
252
|
+
if (!hasStableDirectoryIdentity(rootInfo)) {
|
|
253
|
+
throw new Error(
|
|
254
|
+
"The trace directory does not expose a stable file identity.",
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const rootFlags =
|
|
259
|
+
fsConstants.O_RDONLY |
|
|
260
|
+
(fsConstants.O_DIRECTORY ?? 0) |
|
|
261
|
+
(fsConstants.O_NOFOLLOW ?? 0);
|
|
262
|
+
let rootHandle;
|
|
263
|
+
try {
|
|
264
|
+
rootHandle = await fileSystem.open(traceDir, rootFlags);
|
|
265
|
+
} catch {
|
|
266
|
+
throw new Error("The trace directory could not be opened securely.");
|
|
267
|
+
}
|
|
268
|
+
try {
|
|
269
|
+
const openedRootInfo = await rootHandle.stat({ bigint: true });
|
|
270
|
+
assertTraceRootIdentity(rootInfo, openedRootInfo);
|
|
271
|
+
const currentRootInfo = await fileSystem.lstat(traceDir, { bigint: true });
|
|
272
|
+
assertTraceRootIdentity(openedRootInfo, currentRootInfo);
|
|
273
|
+
return {
|
|
274
|
+
device: openedRootInfo.dev,
|
|
275
|
+
inode: openedRootInfo.ino,
|
|
276
|
+
};
|
|
277
|
+
} finally {
|
|
278
|
+
await rootHandle.close();
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export async function buildAnchoredReviewBudgetReport(rootIdentity) {
|
|
283
|
+
if (!hasStableIdentity(rootIdentity)) {
|
|
284
|
+
throw new Error("The trace directory identity is invalid.");
|
|
285
|
+
}
|
|
286
|
+
await assertCurrentDirectoryIdentity(rootIdentity);
|
|
287
|
+
const state = createAggregationState();
|
|
288
|
+
await walkAnchoredTraceDirectory(state, [], rootIdentity);
|
|
289
|
+
await assertCurrentDirectoryIdentity(rootIdentity);
|
|
290
|
+
return finalizeReport(state);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
async function walkAnchoredTraceDirectory(state, pathSegments, identity) {
|
|
294
|
+
await assertCurrentDirectoryIdentity(identity);
|
|
295
|
+
state.source.directories += 1;
|
|
296
|
+
if (state.source.directories > REPORT_LIMITS.directories) {
|
|
297
|
+
throw new Error("Trace directories exceed the report input limit.");
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const entries = [];
|
|
301
|
+
const directoryHandle = await opendir(".");
|
|
302
|
+
for await (const entry of directoryHandle) {
|
|
303
|
+
state.source.directoryEntries += 1;
|
|
304
|
+
if (state.source.directoryEntries > REPORT_LIMITS.directoryEntries) {
|
|
305
|
+
throw new Error("Trace directory entries exceed the report input limit.");
|
|
306
|
+
}
|
|
307
|
+
entries.push(entry);
|
|
308
|
+
}
|
|
309
|
+
entries.sort((left, right) => left.name.localeCompare(right.name, "en"));
|
|
310
|
+
|
|
311
|
+
for (const entry of entries) {
|
|
312
|
+
await assertCurrentDirectoryIdentity(identity);
|
|
313
|
+
if (!isSafeDirectoryEntryName(entry.name)) {
|
|
314
|
+
throw new Error("A trace directory entry name is invalid.");
|
|
315
|
+
}
|
|
316
|
+
const info = await lstat(entry.name, { bigint: true });
|
|
317
|
+
if (info.isSymbolicLink()) {
|
|
318
|
+
state.source.skippedSymlinks += 1;
|
|
319
|
+
continue;
|
|
320
|
+
}
|
|
321
|
+
if (info.isDirectory()) {
|
|
322
|
+
if (!hasStableDirectoryIdentity(info)) {
|
|
323
|
+
throw new Error(
|
|
324
|
+
"A trace directory does not expose a stable file identity.",
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
const childIdentity = { device: info.dev, inode: info.ino };
|
|
328
|
+
process.chdir(entry.name);
|
|
329
|
+
try {
|
|
330
|
+
await assertCurrentDirectoryIdentity(childIdentity);
|
|
331
|
+
await walkAnchoredTraceDirectory(
|
|
332
|
+
state,
|
|
333
|
+
[...pathSegments, entry.name],
|
|
334
|
+
childIdentity,
|
|
335
|
+
);
|
|
336
|
+
} finally {
|
|
337
|
+
process.chdir("..");
|
|
338
|
+
await assertCurrentDirectoryIdentity(identity);
|
|
339
|
+
}
|
|
340
|
+
continue;
|
|
341
|
+
}
|
|
342
|
+
if (!info.isFile() || !entry.name.endsWith(".jsonl")) continue;
|
|
343
|
+
|
|
344
|
+
if (info.size > BigInt(REPORT_LIMITS.traceFileBytes)) {
|
|
345
|
+
throw new Error("A trace file exceeds the per-file byte limit.");
|
|
346
|
+
}
|
|
347
|
+
if (state.source.jsonlFiles >= REPORT_LIMITS.jsonlFiles) {
|
|
348
|
+
throw new Error("Trace files exceed the report input limit.");
|
|
349
|
+
}
|
|
350
|
+
const size = Number(info.size);
|
|
351
|
+
if (state.source.jsonlBytes + size > REPORT_LIMITS.totalTraceBytes) {
|
|
352
|
+
throw new Error("Trace files exceed the total report byte limit.");
|
|
353
|
+
}
|
|
354
|
+
const file = {
|
|
355
|
+
name: entry.name,
|
|
356
|
+
key: [...pathSegments, entry.name].join("/"),
|
|
357
|
+
size,
|
|
358
|
+
device: info.dev,
|
|
359
|
+
inode: info.ino,
|
|
360
|
+
modifiedAtNs: info.mtimeNs,
|
|
361
|
+
changedAtNs: info.ctimeNs,
|
|
362
|
+
};
|
|
363
|
+
state.source.jsonlFiles += 1;
|
|
364
|
+
state.source.jsonlBytes += size;
|
|
365
|
+
const contents = await readBoundedTraceFile(file);
|
|
366
|
+
aggregateTraceContents(state, contents, file.key);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
await assertCurrentDirectoryIdentity(identity);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function aggregateTraceContents(state, contents, fileKey) {
|
|
373
|
+
for (const line of jsonlLines(contents)) {
|
|
374
|
+
if (state.source.jsonlLines >= REPORT_LIMITS.jsonlLines) {
|
|
375
|
+
throw new Error("JSONL lines exceed the report input limit.");
|
|
376
|
+
}
|
|
377
|
+
state.source.jsonlLines += 1;
|
|
378
|
+
if (Buffer.byteLength(line, "utf8") > REPORT_LIMITS.jsonlLineBytes) {
|
|
379
|
+
throw new Error("A JSONL line exceeds the report input byte limit.");
|
|
380
|
+
}
|
|
381
|
+
if (!line || line.trim().length === 0) {
|
|
382
|
+
state.source.emptyLines += 1;
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
385
|
+
let event;
|
|
386
|
+
try {
|
|
387
|
+
event = JSON.parse(line);
|
|
388
|
+
} catch {
|
|
389
|
+
state.source.malformedLines += 1;
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
if (!isRecord(event)) {
|
|
393
|
+
state.source.malformedLines += 1;
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
if (state.source.parsedEvents >= REPORT_LIMITS.parsedEvents) {
|
|
397
|
+
throw new Error("Trace events exceed the report input limit.");
|
|
398
|
+
}
|
|
399
|
+
state.source.parsedEvents += 1;
|
|
400
|
+
aggregateEvent(state, event, fileKey);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
export async function readBoundedTraceFile(file) {
|
|
405
|
+
const noFollow = fsConstants.O_NOFOLLOW;
|
|
406
|
+
const nonBlock = fsConstants.O_NONBLOCK;
|
|
407
|
+
if (
|
|
408
|
+
!isSafeDirectoryEntryName(file.name) ||
|
|
409
|
+
typeof noFollow !== "number" ||
|
|
410
|
+
noFollow <= 0 ||
|
|
411
|
+
typeof nonBlock !== "number" ||
|
|
412
|
+
nonBlock <= 0
|
|
413
|
+
) {
|
|
414
|
+
throw new Error("Secure trace-file open capability is unavailable.");
|
|
415
|
+
}
|
|
416
|
+
const handle = await open(
|
|
417
|
+
file.name,
|
|
418
|
+
fsConstants.O_RDONLY | noFollow | nonBlock,
|
|
419
|
+
);
|
|
420
|
+
try {
|
|
421
|
+
const info = await handle.stat({ bigint: true });
|
|
422
|
+
if (!matchesDiscoveredTraceFile(info, file)) {
|
|
423
|
+
throw new Error("A trace file changed after discovery; report aborted.");
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
const chunks = [];
|
|
427
|
+
let bytes = 0;
|
|
428
|
+
if (file.size > 0) {
|
|
429
|
+
const stream = handle.createReadStream({
|
|
430
|
+
autoClose: false,
|
|
431
|
+
start: 0,
|
|
432
|
+
end: file.size - 1,
|
|
433
|
+
});
|
|
434
|
+
for await (const chunk of stream) {
|
|
435
|
+
bytes += chunk.length;
|
|
436
|
+
if (bytes > file.size || bytes > REPORT_LIMITS.traceFileBytes) {
|
|
437
|
+
throw new Error("A trace file exceeds its discovered byte limit.");
|
|
438
|
+
}
|
|
439
|
+
chunks.push(chunk);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
const finalInfo = await handle.stat({ bigint: true });
|
|
443
|
+
if (bytes !== file.size || !matchesDiscoveredTraceFile(finalInfo, file)) {
|
|
444
|
+
throw new Error("A trace file changed while reading; report aborted.");
|
|
445
|
+
}
|
|
446
|
+
return Buffer.concat(chunks, bytes).toString("utf8");
|
|
447
|
+
} finally {
|
|
448
|
+
await handle.close();
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
async function assertCurrentDirectoryIdentity(expected) {
|
|
453
|
+
const current = await lstat(".", { bigint: true });
|
|
454
|
+
if (
|
|
455
|
+
current.isSymbolicLink() ||
|
|
456
|
+
!hasStableDirectoryIdentity(current) ||
|
|
457
|
+
current.dev !== expected.device ||
|
|
458
|
+
current.ino !== expected.inode
|
|
459
|
+
) {
|
|
460
|
+
throw new Error(
|
|
461
|
+
"Trace directory changed during traversal; report aborted.",
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
function assertTraceRootIdentity(expected, actual) {
|
|
467
|
+
if (
|
|
468
|
+
actual.isSymbolicLink() ||
|
|
469
|
+
!hasStableDirectoryIdentity(actual) ||
|
|
470
|
+
actual.dev !== expected.dev ||
|
|
471
|
+
actual.ino !== expected.ino
|
|
472
|
+
) {
|
|
473
|
+
throw new Error(
|
|
474
|
+
"Trace directory changed during validation; report aborted.",
|
|
475
|
+
);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
function hasStableDirectoryIdentity(info) {
|
|
480
|
+
return info.isDirectory() && info.dev !== 0n && info.ino !== 0n;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
function hasStableIdentity(identity) {
|
|
484
|
+
return (
|
|
485
|
+
isRecord(identity) &&
|
|
486
|
+
typeof identity.device === "bigint" &&
|
|
487
|
+
identity.device !== 0n &&
|
|
488
|
+
typeof identity.inode === "bigint" &&
|
|
489
|
+
identity.inode !== 0n
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function isSafeDirectoryEntryName(name) {
|
|
494
|
+
return (
|
|
495
|
+
typeof name === "string" &&
|
|
496
|
+
name.length > 0 &&
|
|
497
|
+
name !== "." &&
|
|
498
|
+
name !== ".." &&
|
|
499
|
+
!name.includes("/") &&
|
|
500
|
+
!name.includes("\\")
|
|
501
|
+
);
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
function matchesDiscoveredTraceFile(info, file) {
|
|
505
|
+
return (
|
|
506
|
+
info.isFile() &&
|
|
507
|
+
info.size === BigInt(file.size) &&
|
|
508
|
+
info.size <= BigInt(REPORT_LIMITS.traceFileBytes) &&
|
|
509
|
+
(file.device === 0n || info.dev === file.device) &&
|
|
510
|
+
(file.inode === 0n || info.ino === file.inode) &&
|
|
511
|
+
info.mtimeNs === file.modifiedAtNs &&
|
|
512
|
+
info.ctimeNs === file.changedAtNs
|
|
513
|
+
);
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
function createAggregationState() {
|
|
517
|
+
return {
|
|
518
|
+
source: {
|
|
519
|
+
jsonlFiles: 0,
|
|
520
|
+
jsonlBytes: 0,
|
|
521
|
+
jsonlLines: 0,
|
|
522
|
+
parsedEvents: 0,
|
|
523
|
+
malformedLines: 0,
|
|
524
|
+
emptyLines: 0,
|
|
525
|
+
ignoredEvents: 0,
|
|
526
|
+
invalidEvents: 0,
|
|
527
|
+
skippedSymlinks: 0,
|
|
528
|
+
directories: 0,
|
|
529
|
+
directoryEntries: 0,
|
|
530
|
+
},
|
|
531
|
+
completedReviews: new Set(),
|
|
532
|
+
completionReasons: new Map(),
|
|
533
|
+
reviewTokenStatuses: new Map([
|
|
534
|
+
["reported", 0],
|
|
535
|
+
["partial", 0],
|
|
536
|
+
["unknown", 0],
|
|
537
|
+
]),
|
|
538
|
+
completedCalls: 0,
|
|
539
|
+
outputSignalEligibleCalls: 0,
|
|
540
|
+
identityStatuses: new Map([
|
|
541
|
+
["reported", 0],
|
|
542
|
+
["requested_only", 0],
|
|
543
|
+
["unknown", 0],
|
|
544
|
+
["missing", 0],
|
|
545
|
+
]),
|
|
546
|
+
callGroups: new Map(),
|
|
547
|
+
agentStreamByteValues: createByteValues(),
|
|
548
|
+
normalPathAgentStreamByteValues: createByteValues(),
|
|
549
|
+
outputWarningEvents: 0,
|
|
550
|
+
completedCallEvents: new Map(),
|
|
551
|
+
outputWarningCallEvents: new Map(),
|
|
552
|
+
outputLimitCalls: 0,
|
|
553
|
+
outputLimitReviews: new Set(),
|
|
554
|
+
optionalSkipTotal: 0,
|
|
555
|
+
optionalSkipReasons: new Map(),
|
|
556
|
+
optionalSkipKindReasons: new Map(),
|
|
557
|
+
};
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
function aggregateEvent(state, event, fileKey) {
|
|
561
|
+
const type = event.type;
|
|
562
|
+
const reviewKey = eventKey(event, fileKey);
|
|
563
|
+
|
|
564
|
+
if (type === "model_call_completed") {
|
|
565
|
+
if (!MODEL_CALL_KINDS.has(event.kind)) {
|
|
566
|
+
state.source.invalidEvents += 1;
|
|
567
|
+
return;
|
|
568
|
+
}
|
|
569
|
+
aggregateCompletedCall(state, event);
|
|
570
|
+
incrementCorrelation(
|
|
571
|
+
state.completedCallEvents,
|
|
572
|
+
callEventKey(event, reviewKey),
|
|
573
|
+
reviewKey,
|
|
574
|
+
);
|
|
575
|
+
if (
|
|
576
|
+
AGENT_STREAM_CALL_KINDS.has(event.kind) &&
|
|
577
|
+
event.errorCode === "AGENT_OUTPUT_LIMIT"
|
|
578
|
+
) {
|
|
579
|
+
state.outputLimitCalls += 1;
|
|
580
|
+
state.outputLimitReviews.add(reviewKey);
|
|
581
|
+
}
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
if (type === "agent_output_warning") {
|
|
586
|
+
if (
|
|
587
|
+
!AGENT_STREAM_CALL_KINDS.has(event.kind) ||
|
|
588
|
+
typeof event.agent !== "string" ||
|
|
589
|
+
event.agent.length === 0
|
|
590
|
+
) {
|
|
591
|
+
state.source.invalidEvents += 1;
|
|
592
|
+
return;
|
|
593
|
+
}
|
|
594
|
+
if (state.outputWarningEvents >= REPORT_LIMITS.outputWarningEvents) {
|
|
595
|
+
throw new Error("Output warning events exceed the report input limit.");
|
|
596
|
+
}
|
|
597
|
+
state.outputWarningEvents += 1;
|
|
598
|
+
incrementCorrelation(
|
|
599
|
+
state.outputWarningCallEvents,
|
|
600
|
+
callEventKey(event, reviewKey),
|
|
601
|
+
reviewKey,
|
|
602
|
+
);
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
if (type === "review_budget_completed") {
|
|
607
|
+
if (state.completedReviews.has(reviewKey)) return;
|
|
608
|
+
if (state.completedReviews.size >= REPORT_LIMITS.completedReviews) {
|
|
609
|
+
throw new Error("Completed reviews exceed the report input limit.");
|
|
610
|
+
}
|
|
611
|
+
state.completedReviews.add(reviewKey);
|
|
612
|
+
const completion = isRecord(event.completion) ? event.completion : {};
|
|
613
|
+
if (Array.isArray(completion.reasons)) {
|
|
614
|
+
for (const reason of completion.reasons) {
|
|
615
|
+
incrementReason(
|
|
616
|
+
state.completionReasons,
|
|
617
|
+
safeMetadata(reason) ?? "unknown",
|
|
618
|
+
);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
const tokenUsage = isRecord(event.tokenUsage) ? event.tokenUsage : {};
|
|
622
|
+
const status = ["reported", "partial", "unknown"].includes(
|
|
623
|
+
tokenUsage.status,
|
|
624
|
+
)
|
|
625
|
+
? tokenUsage.status
|
|
626
|
+
: "unknown";
|
|
627
|
+
increment(state.reviewTokenStatuses, status);
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
if (type === "model_call_skipped") {
|
|
632
|
+
if (!OPTIONAL_CALL_KINDS.has(event.kind)) return;
|
|
633
|
+
const reason = safeMetadata(event.reason) ?? "unknown";
|
|
634
|
+
state.optionalSkipTotal += 1;
|
|
635
|
+
incrementReason(state.optionalSkipReasons, reason);
|
|
636
|
+
incrementReason(
|
|
637
|
+
state.optionalSkipKindReasons,
|
|
638
|
+
`${event.kind}\u0000${reason}`,
|
|
639
|
+
);
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
state.source.ignoredEvents += 1;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
function aggregateCompletedCall(state, event) {
|
|
647
|
+
if (state.completedCalls >= REPORT_LIMITS.completedCalls) {
|
|
648
|
+
throw new Error("Completed model calls exceed the report input limit.");
|
|
649
|
+
}
|
|
650
|
+
const identity = normalizeExecutionIdentity(event.executionIdentity);
|
|
651
|
+
const agent = safeMetadata(event.agent);
|
|
652
|
+
const kind = event.kind;
|
|
653
|
+
const agentStreamCall = AGENT_STREAM_CALL_KINDS.has(kind);
|
|
654
|
+
const normalPath = isNormalPathCall(event);
|
|
655
|
+
const groupFields = {
|
|
656
|
+
agent,
|
|
657
|
+
kind,
|
|
658
|
+
providerRoute: identity.providerRoute,
|
|
659
|
+
requestedModel: identity.requestedModel,
|
|
660
|
+
reportingStatus: identity.reportingStatus,
|
|
661
|
+
reportedProvider: identity.reportedProvider,
|
|
662
|
+
reportedModel: identity.reportedModel,
|
|
663
|
+
};
|
|
664
|
+
const key = JSON.stringify(Object.values(groupFields));
|
|
665
|
+
let group = state.callGroups.get(key);
|
|
666
|
+
if (!group) {
|
|
667
|
+
if (state.callGroups.size >= REPORT_LIMITS.executionGroups) {
|
|
668
|
+
throw new Error("Execution groups exceed the report input limit.");
|
|
669
|
+
}
|
|
670
|
+
group = {
|
|
671
|
+
...groupFields,
|
|
672
|
+
calls: 0,
|
|
673
|
+
normalPathCalls: 0,
|
|
674
|
+
byteValues: createByteValues(),
|
|
675
|
+
normalPathByteValues: createByteValues(),
|
|
676
|
+
reportedUsageCalls: 0,
|
|
677
|
+
unknownUsageCalls: 0,
|
|
678
|
+
};
|
|
679
|
+
state.callGroups.set(key, group);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
state.completedCalls += 1;
|
|
683
|
+
if (agentStreamCall) {
|
|
684
|
+
state.outputSignalEligibleCalls += 1;
|
|
685
|
+
}
|
|
686
|
+
group.calls += 1;
|
|
687
|
+
if (normalPath) group.normalPathCalls += 1;
|
|
688
|
+
increment(state.identityStatuses, identity.reportingStatus);
|
|
689
|
+
for (const keyName of ["messageBytes", "thoughtBytes", "outputBytes"]) {
|
|
690
|
+
const value = nonNegativeNumber(event[keyName]);
|
|
691
|
+
if (value === undefined) continue;
|
|
692
|
+
group.byteValues[keyName].push(value);
|
|
693
|
+
if (agentStreamCall) {
|
|
694
|
+
state.agentStreamByteValues[keyName].push(value);
|
|
695
|
+
}
|
|
696
|
+
if (normalPath) {
|
|
697
|
+
group.normalPathByteValues[keyName].push(value);
|
|
698
|
+
if (agentStreamCall) {
|
|
699
|
+
state.normalPathAgentStreamByteValues[keyName].push(value);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
if (hasReportedUsage(event.usage)) {
|
|
704
|
+
group.reportedUsageCalls += 1;
|
|
705
|
+
} else {
|
|
706
|
+
group.unknownUsageCalls += 1;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
function finalizeReport(state) {
|
|
711
|
+
const completedReviews = state.completedReviews.size;
|
|
712
|
+
const completedCalls = state.completedCalls;
|
|
713
|
+
const warningCorrelation = correlateOutputWarnings(state);
|
|
714
|
+
const groups = [...state.callGroups.values()]
|
|
715
|
+
.map((group) => ({
|
|
716
|
+
agent: group.agent,
|
|
717
|
+
kind: group.kind,
|
|
718
|
+
providerRoute: group.providerRoute,
|
|
719
|
+
requestedModel: group.requestedModel,
|
|
720
|
+
reportingStatus: group.reportingStatus,
|
|
721
|
+
reportedProvider: group.reportedProvider,
|
|
722
|
+
reportedModel: group.reportedModel,
|
|
723
|
+
calls: group.calls,
|
|
724
|
+
normalPathCalls: group.normalPathCalls,
|
|
725
|
+
bytes: {
|
|
726
|
+
allCalls: summarizeByteValues(group.byteValues),
|
|
727
|
+
normalPath: summarizeByteValues(group.normalPathByteValues),
|
|
728
|
+
},
|
|
729
|
+
tokenUsage: {
|
|
730
|
+
reportedCalls: group.reportedUsageCalls,
|
|
731
|
+
unknownCalls: group.unknownUsageCalls,
|
|
732
|
+
reportedRate: rate(group.reportedUsageCalls, group.calls),
|
|
733
|
+
unknownRate: rate(group.unknownUsageCalls, group.calls),
|
|
734
|
+
},
|
|
735
|
+
}))
|
|
736
|
+
.sort(compareExecutionGroups);
|
|
737
|
+
|
|
738
|
+
return {
|
|
739
|
+
schemaVersion: 1,
|
|
740
|
+
inputLimits: REPORT_LIMITS,
|
|
741
|
+
dataProvenance: {
|
|
742
|
+
bytes: "measured_trace_fields",
|
|
743
|
+
tokenUsage: "provider_reported_trace_fields",
|
|
744
|
+
tokenEstimatesIncluded: false,
|
|
745
|
+
costEstimatesIncluded: false,
|
|
746
|
+
},
|
|
747
|
+
source: state.source,
|
|
748
|
+
reviews: {
|
|
749
|
+
completed: completedReviews,
|
|
750
|
+
completionReasons: countEntries(state.completionReasons, "reason"),
|
|
751
|
+
},
|
|
752
|
+
calls: {
|
|
753
|
+
completed: completedCalls,
|
|
754
|
+
outputSignalEligible: state.outputSignalEligibleCalls,
|
|
755
|
+
normalPath: groups.reduce(
|
|
756
|
+
(total, group) => total + group.normalPathCalls,
|
|
757
|
+
0,
|
|
758
|
+
),
|
|
759
|
+
identityReporting: Object.fromEntries(
|
|
760
|
+
["reported", "requested_only", "unknown", "missing"].map((status) => {
|
|
761
|
+
const calls = state.identityStatuses.get(status) ?? 0;
|
|
762
|
+
return [status, { calls, rate: rate(calls, completedCalls) }];
|
|
763
|
+
}),
|
|
764
|
+
),
|
|
765
|
+
byExecution: groups,
|
|
766
|
+
},
|
|
767
|
+
bytes: {
|
|
768
|
+
allCalls: summarizeByteValues(state.agentStreamByteValues),
|
|
769
|
+
normalPath: summarizeByteValues(state.normalPathAgentStreamByteValues),
|
|
770
|
+
},
|
|
771
|
+
tokenUsage: {
|
|
772
|
+
reviewStatus: Object.fromEntries(
|
|
773
|
+
["reported", "partial", "unknown"].map((status) => {
|
|
774
|
+
const reviews = state.reviewTokenStatuses.get(status) ?? 0;
|
|
775
|
+
return [status, { reviews, rate: rate(reviews, completedReviews) }];
|
|
776
|
+
}),
|
|
777
|
+
),
|
|
778
|
+
},
|
|
779
|
+
outputSignals: {
|
|
780
|
+
agentOutputWarning: {
|
|
781
|
+
events: state.outputWarningEvents,
|
|
782
|
+
calls: warningCorrelation.calls,
|
|
783
|
+
uncorrelatedEvents:
|
|
784
|
+
state.outputWarningEvents - warningCorrelation.calls,
|
|
785
|
+
callRate: rate(
|
|
786
|
+
warningCorrelation.calls,
|
|
787
|
+
state.outputSignalEligibleCalls,
|
|
788
|
+
),
|
|
789
|
+
reviews: warningCorrelation.completedReviews,
|
|
790
|
+
reviewRate: rate(warningCorrelation.completedReviews, completedReviews),
|
|
791
|
+
},
|
|
792
|
+
agentOutputLimit: {
|
|
793
|
+
calls: state.outputLimitCalls,
|
|
794
|
+
callRate: rate(state.outputLimitCalls, state.outputSignalEligibleCalls),
|
|
795
|
+
reviews: intersectionSize(
|
|
796
|
+
state.outputLimitReviews,
|
|
797
|
+
state.completedReviews,
|
|
798
|
+
),
|
|
799
|
+
reviewRate: rate(
|
|
800
|
+
intersectionSize(state.outputLimitReviews, state.completedReviews),
|
|
801
|
+
completedReviews,
|
|
802
|
+
),
|
|
803
|
+
},
|
|
804
|
+
},
|
|
805
|
+
optionalPhaseSkips: {
|
|
806
|
+
total: state.optionalSkipTotal,
|
|
807
|
+
byReason: countEntries(state.optionalSkipReasons, "reason"),
|
|
808
|
+
byKindAndReason: [...state.optionalSkipKindReasons.entries()]
|
|
809
|
+
.map(([key, count]) => {
|
|
810
|
+
const [kind, reason] = key.split("\u0000");
|
|
811
|
+
return { kind, reason, count };
|
|
812
|
+
})
|
|
813
|
+
.sort((left, right) =>
|
|
814
|
+
`${left.kind}\u0000${left.reason}`.localeCompare(
|
|
815
|
+
`${right.kind}\u0000${right.reason}`,
|
|
816
|
+
"en",
|
|
817
|
+
),
|
|
818
|
+
),
|
|
819
|
+
},
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
function normalizeExecutionIdentity(value) {
|
|
824
|
+
if (!isRecord(value)) {
|
|
825
|
+
return {
|
|
826
|
+
providerRoute: null,
|
|
827
|
+
requestedModel: null,
|
|
828
|
+
reportingStatus: "missing",
|
|
829
|
+
reportedProvider: null,
|
|
830
|
+
reportedModel: null,
|
|
831
|
+
};
|
|
832
|
+
}
|
|
833
|
+
const providerRoute = PROVIDER_ROUTES.has(value.providerRoute)
|
|
834
|
+
? value.providerRoute
|
|
835
|
+
: null;
|
|
836
|
+
const requestedModel = safeMetadata(value.requestedModel);
|
|
837
|
+
const reportedProvider = safeMetadata(value.reportedProvider);
|
|
838
|
+
const reportedModel = safeMetadata(value.reportedModel);
|
|
839
|
+
const reportingStatus =
|
|
840
|
+
reportedProvider !== null || reportedModel !== null
|
|
841
|
+
? "reported"
|
|
842
|
+
: requestedModel !== null
|
|
843
|
+
? "requested_only"
|
|
844
|
+
: "unknown";
|
|
845
|
+
return {
|
|
846
|
+
providerRoute,
|
|
847
|
+
requestedModel,
|
|
848
|
+
reportingStatus,
|
|
849
|
+
reportedProvider,
|
|
850
|
+
reportedModel,
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
function safeMetadata(value) {
|
|
855
|
+
if (typeof value !== "string") return null;
|
|
856
|
+
const compact = value
|
|
857
|
+
.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
858
|
+
.replace(/[\u0000-\u001f\u007f-\u009f]/g, "")
|
|
859
|
+
.replace(/\s+/g, " ")
|
|
860
|
+
.trim();
|
|
861
|
+
if (
|
|
862
|
+
compact.length === 0 ||
|
|
863
|
+
CREDENTIAL_PATTERNS.some((pattern) => pattern.test(compact)) ||
|
|
864
|
+
UNSAFE_METADATA_PATTERN.test(compact) ||
|
|
865
|
+
compact.includes("[KYOSO_REDACTED]")
|
|
866
|
+
) {
|
|
867
|
+
return null;
|
|
868
|
+
}
|
|
869
|
+
return compact.length <= MAX_METADATA_CHARS
|
|
870
|
+
? compact
|
|
871
|
+
: `${compact.slice(0, MAX_METADATA_CHARS - 3)}...`;
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
function summarizeByteValues(values) {
|
|
875
|
+
return {
|
|
876
|
+
messageBytes: percentileSummary(values.messageBytes),
|
|
877
|
+
thoughtBytes: percentileSummary(values.thoughtBytes),
|
|
878
|
+
outputBytes: percentileSummary(values.outputBytes),
|
|
879
|
+
};
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
function percentileSummary(values) {
|
|
883
|
+
if (values.length === 0) {
|
|
884
|
+
return { samples: 0, p50: null, p95: null, p99: null, max: null };
|
|
885
|
+
}
|
|
886
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
887
|
+
return {
|
|
888
|
+
samples: sorted.length,
|
|
889
|
+
p50: nearestRank(sorted, 0.5),
|
|
890
|
+
p95: nearestRank(sorted, 0.95),
|
|
891
|
+
p99: nearestRank(sorted, 0.99),
|
|
892
|
+
max: sorted.at(-1),
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
function nearestRank(sorted, percentile) {
|
|
897
|
+
const index = Math.max(0, Math.ceil(percentile * sorted.length) - 1);
|
|
898
|
+
return sorted[index];
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
function hasReportedUsage(value) {
|
|
902
|
+
if (!isRecord(value)) return false;
|
|
903
|
+
return TOKEN_USAGE_KEYS.some(
|
|
904
|
+
(key) => nonNegativeNumber(value[key]) !== undefined,
|
|
905
|
+
);
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
function isNormalPathCall(event) {
|
|
909
|
+
return event.resultStatus === "completed" && event.errorCode === undefined;
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
function nonNegativeNumber(value) {
|
|
913
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0
|
|
914
|
+
? value
|
|
915
|
+
: undefined;
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
function countEntries(counts, label) {
|
|
919
|
+
return [...counts.entries()]
|
|
920
|
+
.map(([value, count]) => ({ [label]: value, count }))
|
|
921
|
+
.sort((left, right) =>
|
|
922
|
+
String(left[label]).localeCompare(String(right[label]), "en"),
|
|
923
|
+
);
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
function compareExecutionGroups(left, right) {
|
|
927
|
+
return [
|
|
928
|
+
left.kind,
|
|
929
|
+
left.agent ?? "",
|
|
930
|
+
left.providerRoute ?? "",
|
|
931
|
+
left.requestedModel ?? "",
|
|
932
|
+
left.reportingStatus,
|
|
933
|
+
left.reportedProvider ?? "",
|
|
934
|
+
left.reportedModel ?? "",
|
|
935
|
+
]
|
|
936
|
+
.join("\u0000")
|
|
937
|
+
.localeCompare(
|
|
938
|
+
[
|
|
939
|
+
right.kind,
|
|
940
|
+
right.agent ?? "",
|
|
941
|
+
right.providerRoute ?? "",
|
|
942
|
+
right.requestedModel ?? "",
|
|
943
|
+
right.reportingStatus,
|
|
944
|
+
right.reportedProvider ?? "",
|
|
945
|
+
right.reportedModel ?? "",
|
|
946
|
+
].join("\u0000"),
|
|
947
|
+
"en",
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
function createByteValues() {
|
|
952
|
+
return { messageBytes: [], thoughtBytes: [], outputBytes: [] };
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
function increment(map, key) {
|
|
956
|
+
map.set(key, (map.get(key) ?? 0) + 1);
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
function incrementReason(map, key) {
|
|
960
|
+
if (!map.has(key) && map.size >= REPORT_LIMITS.distinctReasons) {
|
|
961
|
+
throw new Error("Distinct trace reasons exceed the report input limit.");
|
|
962
|
+
}
|
|
963
|
+
increment(map, key);
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
function incrementCorrelation(map, key, reviewKey) {
|
|
967
|
+
const current = map.get(key);
|
|
968
|
+
if (current) {
|
|
969
|
+
current.count += 1;
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
if (map.size >= REPORT_LIMITS.correlationKeys) {
|
|
973
|
+
throw new Error("Trace correlation keys exceed the report input limit.");
|
|
974
|
+
}
|
|
975
|
+
map.set(key, { count: 1, reviewKey });
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
function correlateOutputWarnings(state) {
|
|
979
|
+
let calls = 0;
|
|
980
|
+
const reviews = new Set();
|
|
981
|
+
for (const [key, warning] of state.outputWarningCallEvents) {
|
|
982
|
+
const completed = state.completedCallEvents.get(key);
|
|
983
|
+
if (!completed) continue;
|
|
984
|
+
const correlated = Math.min(warning.count, completed.count);
|
|
985
|
+
calls += correlated;
|
|
986
|
+
if (correlated > 0 && state.completedReviews.has(warning.reviewKey)) {
|
|
987
|
+
reviews.add(warning.reviewKey);
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
return { calls, completedReviews: reviews.size };
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
function rate(numerator, denominator) {
|
|
994
|
+
if (denominator === 0) return 0;
|
|
995
|
+
return Math.round((numerator / denominator) * 1_000_000) / 1_000_000;
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
function intersectionSize(left, right) {
|
|
999
|
+
let count = 0;
|
|
1000
|
+
for (const value of left) {
|
|
1001
|
+
if (right.has(value)) count += 1;
|
|
1002
|
+
}
|
|
1003
|
+
return count;
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
function eventKey(event, fileKey) {
|
|
1007
|
+
const source =
|
|
1008
|
+
typeof event.traceId === "string" && event.traceId.length > 0
|
|
1009
|
+
? event.traceId
|
|
1010
|
+
: `file:${fileKey}`;
|
|
1011
|
+
return opaqueKey(["review", source]);
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
function callEventKey(event, reviewKey) {
|
|
1015
|
+
return opaqueKey([
|
|
1016
|
+
"call",
|
|
1017
|
+
reviewKey,
|
|
1018
|
+
typeof event.kind === "string" ? event.kind : "",
|
|
1019
|
+
typeof event.agent === "string" ? event.agent : "",
|
|
1020
|
+
]);
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
function opaqueKey(parts) {
|
|
1024
|
+
const hash = createHash("sha256");
|
|
1025
|
+
for (const part of parts) {
|
|
1026
|
+
hash.update(String(Buffer.byteLength(part, "utf8")));
|
|
1027
|
+
hash.update(":");
|
|
1028
|
+
hash.update(part);
|
|
1029
|
+
}
|
|
1030
|
+
return hash.digest("hex");
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
function isRecord(value) {
|
|
1034
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
function* jsonlLines(contents) {
|
|
1038
|
+
let start = 0;
|
|
1039
|
+
for (let index = 0; index < contents.length; index += 1) {
|
|
1040
|
+
if (contents[index] !== "\n") continue;
|
|
1041
|
+
const end =
|
|
1042
|
+
index > start && contents[index - 1] === "\r" ? index - 1 : index;
|
|
1043
|
+
yield contents.slice(start, end);
|
|
1044
|
+
start = index + 1;
|
|
1045
|
+
}
|
|
1046
|
+
if (start < contents.length) {
|
|
1047
|
+
const end = contents.endsWith("\r") ? contents.length - 1 : contents.length;
|
|
1048
|
+
yield contents.slice(start, end);
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
function usage() {
|
|
1053
|
+
return `Usage: node scripts/review-budget-report.mjs --trace-dir <absolute-directory> [--json]\n\nReads only .jsonl files below the explicitly supplied directory. Byte distributions use nearest-rank percentiles; token and cost values are never estimated from bytes.`;
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
async function main() {
|
|
1057
|
+
const options = parseReportArgs(process.argv.slice(2));
|
|
1058
|
+
if (options.help) {
|
|
1059
|
+
process.stdout.write(`${usage()}\n`);
|
|
1060
|
+
return;
|
|
1061
|
+
}
|
|
1062
|
+
const report = await buildReviewBudgetReport(options.traceDir);
|
|
1063
|
+
process.stdout.write(
|
|
1064
|
+
options.json
|
|
1065
|
+
? `${JSON.stringify(report, null, 2)}\n`
|
|
1066
|
+
: `${renderHumanReport(report)}\n`,
|
|
1067
|
+
);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
async function internalWorkerMain(argv) {
|
|
1071
|
+
if (
|
|
1072
|
+
argv.length !== 2 ||
|
|
1073
|
+
!/^[1-9][0-9]*$/.test(argv[0] ?? "") ||
|
|
1074
|
+
!/^[1-9][0-9]*$/.test(argv[1] ?? "")
|
|
1075
|
+
) {
|
|
1076
|
+
throw new Error("The trace directory identity is invalid.");
|
|
1077
|
+
}
|
|
1078
|
+
const report = await buildAnchoredReviewBudgetReport({
|
|
1079
|
+
device: BigInt(argv[0]),
|
|
1080
|
+
inode: BigInt(argv[1]),
|
|
1081
|
+
});
|
|
1082
|
+
process.stdout.write(JSON.stringify(report));
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
if (isMainModule()) {
|
|
1086
|
+
const internalWorker = process.argv[2] === INTERNAL_WORKER_FLAG;
|
|
1087
|
+
const entrypoint = internalWorker
|
|
1088
|
+
? internalWorkerMain(process.argv.slice(3))
|
|
1089
|
+
: main();
|
|
1090
|
+
entrypoint.catch((error) => {
|
|
1091
|
+
const rawMessage =
|
|
1092
|
+
error instanceof Error ? error.message : WORKER_FAILURE_MESSAGE;
|
|
1093
|
+
const message = internalWorker
|
|
1094
|
+
? SAFE_WORKER_ERROR_MESSAGES.has(rawMessage)
|
|
1095
|
+
? rawMessage
|
|
1096
|
+
: WORKER_FAILURE_MESSAGE
|
|
1097
|
+
: rawMessage;
|
|
1098
|
+
process.stderr.write(
|
|
1099
|
+
internalWorker
|
|
1100
|
+
? `${message}\n`
|
|
1101
|
+
: `review-budget-report failed: ${message}\n`,
|
|
1102
|
+
);
|
|
1103
|
+
process.exitCode = 1;
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
function isMainModule() {
|
|
1108
|
+
if (!process.argv[1]) return false;
|
|
1109
|
+
try {
|
|
1110
|
+
return realpathSync(resolve(process.argv[1])) === realpathSync(SCRIPT_PATH);
|
|
1111
|
+
} catch {
|
|
1112
|
+
return false;
|
|
1113
|
+
}
|
|
1114
|
+
}
|