@agent-finops/core 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/actionPlanner.d.ts +140 -0
- package/dist/actionPlanner.js +938 -0
- package/dist/actionVerification.d.ts +1240 -0
- package/dist/actionVerification.js +1028 -0
- package/dist/activitySnapshot.d.ts +101 -9
- package/dist/activitySnapshot.js +145 -6
- package/dist/activitySnapshotCache.d.ts +8 -1
- package/dist/activitySnapshotCache.js +103 -7
- package/dist/agentEconomicsReceipt.d.ts +58 -58
- package/dist/analyze.js +3 -1
- package/dist/cutList.js +1 -1
- package/dist/glance.d.ts +30 -2
- package/dist/glance.js +265 -84
- package/dist/index.d.ts +11 -2
- package/dist/index.js +10 -1
- package/dist/insights.js +3 -1
- package/dist/localAgentFormats/gemini.js +2 -2
- package/dist/localAgentFormats/registry.js +6 -2
- package/dist/localAgentFormats/runtimeRegistry.js +5 -2
- package/dist/localAgentFormats/types.d.ts +2 -1
- package/dist/localAgentLogs.d.ts +362 -3
- package/dist/localAgentLogs.js +1964 -165
- package/dist/modelPricing.d.ts +1 -1
- package/dist/modelPricing.js +4 -1
- package/dist/planMath.js +12 -7
- package/dist/projectEconomics.d.ts +617 -0
- package/dist/projectEconomics.js +620 -0
- package/dist/projectEconomicsBuilder.d.ts +89 -0
- package/dist/projectEconomicsBuilder.js +473 -0
- package/dist/projectIndexStore.d.ts +545 -0
- package/dist/projectIndexStore.js +606 -0
- package/dist/providerConnectors.d.ts +59 -1
- package/dist/providerConnectors.js +192 -12
- package/dist/qualitativeIndexCache.d.ts +494 -0
- package/dist/qualitativeIndexCache.js +930 -0
- package/dist/resultCard.d.ts +350 -0
- package/dist/resultCard.js +604 -0
- package/dist/runtimeCommands.d.ts +21 -0
- package/dist/runtimeCommands.js +27 -0
- package/dist/scanGuard.d.ts +3 -1
- package/dist/scanGuard.js +164 -4
- package/dist/schema.d.ts +31 -31
- package/dist/sessionVitals.d.ts +145 -0
- package/dist/sessionVitals.js +521 -0
- package/dist/sourceRegistry.js +90 -52
- package/dist/toolInvocations.d.ts +40 -1
- package/dist/toolInvocations.js +101 -20
- package/package.json +1 -1
package/dist/localAgentLogs.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { createReadStream } from "node:fs";
|
|
1
|
+
import { constants, createReadStream } from "node:fs";
|
|
2
2
|
import { lstat, open, readdir, readFile, stat } from "node:fs/promises";
|
|
3
3
|
import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { createInterface } from "node:readline";
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
6
7
|
import { canPriceTokenUsageAtScope, estimateTokenCostUsd, estimateTokenCostsUsd, promptTierThreshold, usesPromptTieredPricing } from "./modelPricing.js";
|
|
7
8
|
import { redactSecrets } from "./discovery.js";
|
|
8
|
-
import {
|
|
9
|
+
import { createCodexInvocationCollector, isCodexInvocationCollectorSnapshot } from "./toolInvocations.js";
|
|
10
|
+
import { localAgentFormatDescriptor, localAgentFormatDescriptors, localAgentFormatLabel, matchesLocalAgentDetectionFile, matchesLocalAgentFormatFile, validateLocalAgentFormatDescriptors } from "./localAgentFormats/registry.js";
|
|
9
11
|
import { parseGeminiSession } from "./localAgentFormats/gemini.js";
|
|
10
12
|
/**
|
|
11
13
|
* Resolve the repository root most recently observed in transcript metadata.
|
|
@@ -20,6 +22,56 @@ export function latestObservedWorkingDirectory(calls) {
|
|
|
20
22
|
.sort((left, right) => right.timestamp.localeCompare(left.timestamp))[0]
|
|
21
23
|
?.workingDirectory;
|
|
22
24
|
}
|
|
25
|
+
export const localAgentFinancialParserVersion = 1;
|
|
26
|
+
/**
|
|
27
|
+
* Qualitative parser contract version. Bumped to 2 with the checkpointed
|
|
28
|
+
* streaming path (A4b): entries and checkpoints written by the pre-streaming
|
|
29
|
+
* parser are never reinterpreted — a version mismatch is a miss (entries) or
|
|
30
|
+
* a discard (checkpoints), and the store schema pins this exact literal so
|
|
31
|
+
* both sides fail closed together. Bumped to 4 when Claude Code subagent
|
|
32
|
+
* transcripts gained their own session identity (`subagentId`) and
|
|
33
|
+
* cross-file completion evidence (`subagentCompletions`, from both Task tool
|
|
34
|
+
* results and background task-notifications): entries persisted by the
|
|
35
|
+
* collapsing parser must re-parse rather than silently keep merging subagent
|
|
36
|
+
* runs into their parent session.
|
|
37
|
+
*/
|
|
38
|
+
export const localAgentQualitativeParserVersion = 4;
|
|
39
|
+
/**
|
|
40
|
+
* Conservative launch defaults for action-capable qualitative evidence.
|
|
41
|
+
* Callers must still inspect `qualitativeCoverage` before deriving a finding:
|
|
42
|
+
* the limits protect responsiveness; they do not turn a partial scan into a
|
|
43
|
+
* representative sample.
|
|
44
|
+
*/
|
|
45
|
+
export const SAFE_QUALITATIVE_SCAN_POLICY = Object.freeze({
|
|
46
|
+
maxFileBytes: 64 * 1024 * 1024,
|
|
47
|
+
maxSourceBytes: 256 * 1024 * 1024
|
|
48
|
+
});
|
|
49
|
+
/**
|
|
50
|
+
* True only when every requested source was scanned under an explicit bounded
|
|
51
|
+
* policy without omitting an eligible file. Legacy unbounded results return
|
|
52
|
+
* false so an action caller cannot accidentally treat unknown coverage as a
|
|
53
|
+
* complete launch-safe scan.
|
|
54
|
+
*/
|
|
55
|
+
export function hasCompleteQualitativeCoverage(result, agents = ["claude-code", "codex"]) {
|
|
56
|
+
return agents.every((agent) => (result.sourceScans.find((scan) => scan.agent === agent)?.qualitativeCoverage === "complete"));
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Whether a bounded result contains exact calls from at least one completely
|
|
60
|
+
* parsed selected file for every requested source. This is deliberately
|
|
61
|
+
* weaker than global coverage: it can support a clearly scoped experiment,
|
|
62
|
+
* never a claim about the source's overall/main driver.
|
|
63
|
+
*/
|
|
64
|
+
export function hasExactSelectedQualitativeEvidence(result, agents) {
|
|
65
|
+
const requested = agents ?? [...new Set(result.calls
|
|
66
|
+
.filter((call) => localAgentFormatDescriptor(call.agent)?.capabilities.actionPlanning)
|
|
67
|
+
.map((call) => call.agent))];
|
|
68
|
+
return requested.length > 0 && requested.every((agent) => {
|
|
69
|
+
const scan = result.sourceScans.find((entry) => entry.agent === agent);
|
|
70
|
+
return scan?.qualitativeSelectedEvidence === "complete_files_only" &&
|
|
71
|
+
(scan.qualitativeFilesReadCompletely ?? 0) > 0 &&
|
|
72
|
+
result.calls.some((call) => call.agent === agent);
|
|
73
|
+
});
|
|
74
|
+
}
|
|
23
75
|
/**
|
|
24
76
|
* Codex rollout/compaction files can repeat the same session's cumulative
|
|
25
77
|
* token counter. Keep only the latest snapshot per session so financial value,
|
|
@@ -80,7 +132,7 @@ function stableTurnEvidenceFingerprint(call) {
|
|
|
80
132
|
timestamp: call.timestamp,
|
|
81
133
|
model: call.model,
|
|
82
134
|
project: call.project ?? null,
|
|
83
|
-
|
|
135
|
+
workingDirectoryRef: stableWorkingDirectoryRef(call),
|
|
84
136
|
usageSupport: call.usageSupport ?? null,
|
|
85
137
|
reportedTotalTokens: call.reportedTotalTokens ?? null,
|
|
86
138
|
usage: {
|
|
@@ -95,6 +147,27 @@ function stableTurnEvidenceFingerprint(call) {
|
|
|
95
147
|
geminiTokenEvidence: call.geminiTokenEvidence ?? null
|
|
96
148
|
});
|
|
97
149
|
}
|
|
150
|
+
function stableWorkingDirectoryRef(call) {
|
|
151
|
+
const supplied = call.workingDirectoryRef;
|
|
152
|
+
const derived = call.workingDirectory
|
|
153
|
+
? derivedWorkingDirectoryRef(call.workingDirectory)
|
|
154
|
+
: undefined;
|
|
155
|
+
if (supplied && derived && supplied !== derived)
|
|
156
|
+
return "conflicting-working-directory";
|
|
157
|
+
return supplied ?? derived ?? null;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* The one NUL-separated avref derivation for working directories. Every
|
|
161
|
+
* consumer — stable call refs, the financial probe re-attach, and header-pass
|
|
162
|
+
* attribution — must produce byte-identical refs for the same directory.
|
|
163
|
+
*/
|
|
164
|
+
function derivedWorkingDirectoryRef(workingDirectory) {
|
|
165
|
+
return `avref_${createHash("sha256")
|
|
166
|
+
.update("project-working-directory")
|
|
167
|
+
.update("\u0000")
|
|
168
|
+
.update(workingDirectory)
|
|
169
|
+
.digest("hex")}`;
|
|
170
|
+
}
|
|
98
171
|
function isStableTurnConflict(call) {
|
|
99
172
|
return call.model === "conflicting-local-evidence" &&
|
|
100
173
|
call.usageSupport === "unsupported_token_shape";
|
|
@@ -160,9 +233,37 @@ function parseClaudeFinancialUsage(value, onDiagnostic) {
|
|
|
160
233
|
cacheWrite1hTokens: write1hField.value ?? 0
|
|
161
234
|
};
|
|
162
235
|
if (componentsSupported) {
|
|
236
|
+
const cacheWriteEvidence = writeTotalField.value !== undefined ||
|
|
237
|
+
(write5mField.value !== undefined && write1hField.value !== undefined)
|
|
238
|
+
? "observed"
|
|
239
|
+
: write5mField.value !== undefined || write1hField.value !== undefined
|
|
240
|
+
? "partial"
|
|
241
|
+
: "not_separately_reported";
|
|
242
|
+
const cacheReadEvidence = cacheReadField.value === undefined
|
|
243
|
+
? "not_separately_reported"
|
|
244
|
+
: "observed";
|
|
245
|
+
const componentTotalComplete = cacheReadEvidence === "observed" &&
|
|
246
|
+
cacheWriteEvidence === "observed";
|
|
163
247
|
return {
|
|
164
248
|
usage,
|
|
165
|
-
latestTurnUsage: toTurnUsage(usage, "assistant_message_usage")
|
|
249
|
+
latestTurnUsage: toTurnUsage(usage, "assistant_message_usage"),
|
|
250
|
+
...(reportedTotalField.value !== undefined
|
|
251
|
+
? { reportedTotalTokens: reportedTotalField.value }
|
|
252
|
+
: {}),
|
|
253
|
+
tokenComponentEvidence: {
|
|
254
|
+
inputTokens: "observed",
|
|
255
|
+
outputTokens: "observed",
|
|
256
|
+
cacheReadTokens: cacheReadEvidence,
|
|
257
|
+
cacheWriteTokens: cacheWriteEvidence,
|
|
258
|
+
thoughtTokens: "not_separately_reported",
|
|
259
|
+
toolTokens: "not_separately_reported",
|
|
260
|
+
calculatedTotalTokens: componentTotalComplete
|
|
261
|
+
? "calculated_complete"
|
|
262
|
+
: "calculated_partial",
|
|
263
|
+
reportedTotalTokens: reportedTotalField.value === undefined
|
|
264
|
+
? "not_reported"
|
|
265
|
+
: "provider_reported"
|
|
266
|
+
}
|
|
166
267
|
};
|
|
167
268
|
}
|
|
168
269
|
onDiagnostic?.({ code: "unsupported_token_shape", count: 1 });
|
|
@@ -232,7 +333,28 @@ function parseCodexCumulativeUsage(current, baseline) {
|
|
|
232
333
|
cacheReadTokens: cached
|
|
233
334
|
},
|
|
234
335
|
supported: currentSupported && baselineSupported && monotonic,
|
|
235
|
-
...(reportedTotalTokens !== undefined ? { reportedTotalTokens } : {})
|
|
336
|
+
...(reportedTotalTokens !== undefined ? { reportedTotalTokens } : {}),
|
|
337
|
+
...(currentSupported && baselineSupported && monotonic
|
|
338
|
+
? {
|
|
339
|
+
tokenComponentEvidence: {
|
|
340
|
+
inputTokens: "observed",
|
|
341
|
+
outputTokens: "observed",
|
|
342
|
+
cacheReadTokens: currentCachedField.value === undefined
|
|
343
|
+
? "not_separately_reported"
|
|
344
|
+
: "observed",
|
|
345
|
+
cacheWriteTokens: "not_separately_reported",
|
|
346
|
+
thoughtTokens: "not_separately_reported",
|
|
347
|
+
toolTokens: "not_separately_reported",
|
|
348
|
+
// Codex input_tokens already contains cached input; subtracting
|
|
349
|
+
// the optional cache split and adding it back preserves the full
|
|
350
|
+
// observed input+output total even when the split is unavailable.
|
|
351
|
+
calculatedTotalTokens: "calculated_complete",
|
|
352
|
+
reportedTotalTokens: reportedTotalTokens === undefined
|
|
353
|
+
? "not_reported"
|
|
354
|
+
: "provider_reported"
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
: {})
|
|
236
358
|
};
|
|
237
359
|
}
|
|
238
360
|
function parseCodexTurnUsage(value) {
|
|
@@ -272,8 +394,18 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDia
|
|
|
272
394
|
let lastPrompt;
|
|
273
395
|
let latestActivityKey;
|
|
274
396
|
let isSubagent = filePath.split(sep).includes("subagents");
|
|
397
|
+
// Claude Code writes the parent's sessionId on every subagent line, so the
|
|
398
|
+
// line-level `agentId` (mirrored in the `subagents/agent-<id>.jsonl` file
|
|
399
|
+
// name) is the only truthful per-run identity for a subagent transcript.
|
|
400
|
+
const fileSubagentId = subagentTranscriptFileId(filePath);
|
|
401
|
+
const observedEntryAgentIds = new Set();
|
|
402
|
+
const subagentCompletionsBySession = new Map();
|
|
275
403
|
let parentSessionId;
|
|
276
404
|
let malformedLines = 0;
|
|
405
|
+
let entryOrdinal = 0;
|
|
406
|
+
let latestUnscopedWorkOrdinal = -1;
|
|
407
|
+
const latestWorkOrdinalBySession = new Map();
|
|
408
|
+
const completionBySession = new Map();
|
|
277
409
|
for (const line of content.split("\n")) {
|
|
278
410
|
if (!line.trim())
|
|
279
411
|
continue;
|
|
@@ -287,6 +419,27 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDia
|
|
|
287
419
|
}
|
|
288
420
|
if (!isRecord(entry))
|
|
289
421
|
continue;
|
|
422
|
+
entryOrdinal += 1;
|
|
423
|
+
const entrySessionId = stringOf(entry.sessionId);
|
|
424
|
+
if (entry.type === "user" || entry.type === "assistant") {
|
|
425
|
+
if (entrySessionId)
|
|
426
|
+
latestWorkOrdinalBySession.set(entrySessionId, entryOrdinal);
|
|
427
|
+
else
|
|
428
|
+
latestUnscopedWorkOrdinal = entryOrdinal;
|
|
429
|
+
}
|
|
430
|
+
const durationMs = numberOf(entry.durationMs);
|
|
431
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" &&
|
|
432
|
+
entrySessionId && Number.isSafeInteger(durationMs) && durationMs >= 0) {
|
|
433
|
+
const observedAt = toIso(stringOf(entry.timestamp));
|
|
434
|
+
if (observedAt) {
|
|
435
|
+
completionBySession.set(entrySessionId, {
|
|
436
|
+
status: "completed",
|
|
437
|
+
evidence: "claude_turn_duration",
|
|
438
|
+
observedAt,
|
|
439
|
+
ordinal: entryOrdinal
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
}
|
|
290
443
|
if (entry.type === "ai-title") {
|
|
291
444
|
title = stringOf(entry.aiTitle) ?? title;
|
|
292
445
|
}
|
|
@@ -295,7 +448,43 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDia
|
|
|
295
448
|
}
|
|
296
449
|
if (entry.isSidechain === true)
|
|
297
450
|
isSubagent = true;
|
|
451
|
+
const entryAgentId = stringOf(entry.agentId);
|
|
452
|
+
if (entryAgentId) {
|
|
453
|
+
// Line-level agentId only appears on sidechain lines; it is subagent
|
|
454
|
+
// evidence even when the sidechain flag or path marker is missing.
|
|
455
|
+
isSubagent = true;
|
|
456
|
+
observedEntryAgentIds.add(entryAgentId);
|
|
457
|
+
}
|
|
298
458
|
parentSessionId = stringOf(entry.parentSessionId) ?? parentSessionId;
|
|
459
|
+
// The host records each finished subagent run in the owning transcript:
|
|
460
|
+
// synchronous runs as a Task tool result, background runs as a
|
|
461
|
+
// task-notification queue operation. Subagent files carry no completion
|
|
462
|
+
// marker of their own, so these are the explicit completion evidence for
|
|
463
|
+
// those runs. Only an explicit "completed" status counts; failed, killed,
|
|
464
|
+
// or merely launched runs never read as comparable completed tasks.
|
|
465
|
+
if (entrySessionId) {
|
|
466
|
+
const toolUseResult = isRecord(entry.toolUseResult) ? entry.toolUseResult : undefined;
|
|
467
|
+
let completedSubagentId = toolUseResult && toolUseResult.status === "completed"
|
|
468
|
+
? stringOf(toolUseResult.agentId)
|
|
469
|
+
: undefined;
|
|
470
|
+
if (!completedSubagentId && entry.type === "queue-operation") {
|
|
471
|
+
const notification = parseTaskNotification(stringOf(entry.content));
|
|
472
|
+
if (notification?.status === "completed")
|
|
473
|
+
completedSubagentId = notification.taskId;
|
|
474
|
+
}
|
|
475
|
+
const completionObservedAt = completedSubagentId
|
|
476
|
+
? toIso(stringOf(entry.timestamp))
|
|
477
|
+
: undefined;
|
|
478
|
+
if (completedSubagentId && completionObservedAt) {
|
|
479
|
+
const bySubagent = subagentCompletionsBySession.get(entrySessionId) ??
|
|
480
|
+
new Map();
|
|
481
|
+
const prior = bySubagent.get(completedSubagentId);
|
|
482
|
+
if (!prior || Date.parse(completionObservedAt) > Date.parse(prior)) {
|
|
483
|
+
bySubagent.set(completedSubagentId, completionObservedAt);
|
|
484
|
+
}
|
|
485
|
+
subagentCompletionsBySession.set(entrySessionId, bySubagent);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
299
488
|
const message = isRecord(entry.message) ? entry.message : undefined;
|
|
300
489
|
if (entry.type === "user" && message) {
|
|
301
490
|
for (const prompt of textValues(message.content)) {
|
|
@@ -312,10 +501,11 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDia
|
|
|
312
501
|
if (stringOf(message.model) === "<synthetic>")
|
|
313
502
|
continue;
|
|
314
503
|
// Streaming/retries can write the same API response on multiple lines.
|
|
315
|
-
const
|
|
316
|
-
if (
|
|
504
|
+
const nativeResponse = claudeNativeResponseIdentity(message, entry);
|
|
505
|
+
if (nativeResponse && seen.has(nativeResponse.localDedupeKey))
|
|
317
506
|
continue;
|
|
318
|
-
|
|
507
|
+
if (nativeResponse)
|
|
508
|
+
seen.add(nativeResponse.localDedupeKey);
|
|
319
509
|
const timestamp = toIso(stringOf(entry.timestamp)) ?? new Date(0).toISOString();
|
|
320
510
|
if (typeof sinceMs === "number" && Date.parse(timestamp) < sinceMs) {
|
|
321
511
|
// These prompts led to a call outside the selected evidence window. Do
|
|
@@ -329,11 +519,14 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDia
|
|
|
329
519
|
const sessionId = stringOf(entry.sessionId);
|
|
330
520
|
const call = {
|
|
331
521
|
agent: "claude-code",
|
|
522
|
+
...(nativeResponse ? { callId: nativeResponse.callId } : {}),
|
|
332
523
|
model: stringOf(message.model) ?? "claude-code",
|
|
333
524
|
timestamp,
|
|
334
525
|
project,
|
|
335
526
|
workingDirectory,
|
|
336
527
|
sessionId,
|
|
528
|
+
...(entryAgentId ? { subagentId: entryAgentId } : {}),
|
|
529
|
+
...(stringOf(entry.version) ? { sourceVersion: stringOf(entry.version) } : {}),
|
|
337
530
|
...(parsedUsage.latestTurnUsage
|
|
338
531
|
? { latestTurnUsage: parsedUsage.latestTurnUsage }
|
|
339
532
|
: {}),
|
|
@@ -342,6 +535,9 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDia
|
|
|
342
535
|
...(parsedUsage.reportedTotalTokens !== undefined
|
|
343
536
|
? { reportedTotalTokens: parsedUsage.reportedTotalTokens }
|
|
344
537
|
: {}),
|
|
538
|
+
...(parsedUsage.tokenComponentEvidence
|
|
539
|
+
? { tokenComponentEvidence: parsedUsage.tokenComponentEvidence }
|
|
540
|
+
: {}),
|
|
345
541
|
usage: parsedUsage.usage
|
|
346
542
|
};
|
|
347
543
|
calls.push(call);
|
|
@@ -362,6 +558,31 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDia
|
|
|
362
558
|
activityEvidence.set(activityKey, evidence);
|
|
363
559
|
latestActivityKey = activityKey;
|
|
364
560
|
}
|
|
561
|
+
// One subagent transcript is one run: calls whose lines omitted agentId
|
|
562
|
+
// still belong to the file's single run identity (content agentId when one
|
|
563
|
+
// was observed, else the `subagents/` file name). Never invented for
|
|
564
|
+
// parent transcripts.
|
|
565
|
+
const defaultSubagentId = observedEntryAgentIds.size === 1
|
|
566
|
+
? [...observedEntryAgentIds][0]
|
|
567
|
+
: fileSubagentId;
|
|
568
|
+
if (isSubagent && defaultSubagentId) {
|
|
569
|
+
for (const call of calls)
|
|
570
|
+
call.subagentId ??= defaultSubagentId;
|
|
571
|
+
}
|
|
572
|
+
for (const [completionSessionId, bySubagent] of subagentCompletionsBySession) {
|
|
573
|
+
let target;
|
|
574
|
+
for (let index = calls.length - 1; index >= 0; index -= 1) {
|
|
575
|
+
if (calls[index].sessionId === completionSessionId) {
|
|
576
|
+
target = calls[index];
|
|
577
|
+
break;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
if (!target)
|
|
581
|
+
continue;
|
|
582
|
+
target.subagentCompletions = [...bySubagent.entries()]
|
|
583
|
+
.sort((left, right) => left[0].localeCompare(right[0]))
|
|
584
|
+
.map(([subagentId, observedAt]) => ({ subagentId, observedAt }));
|
|
585
|
+
}
|
|
365
586
|
const fallbackPrompt = lastPrompt && isHumanPrompt(lastPrompt) ? lastPrompt : undefined;
|
|
366
587
|
const activities = new Map();
|
|
367
588
|
for (const [activityKey, evidence] of activityEvidence) {
|
|
@@ -385,141 +606,179 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDia
|
|
|
385
606
|
}
|
|
386
607
|
for (const call of calls) {
|
|
387
608
|
call.activity = activities.get(localActivityScopeKey(call.sessionId, call.workingDirectory, call.project));
|
|
609
|
+
const completion = call.sessionId
|
|
610
|
+
? completionBySession.get(call.sessionId)
|
|
611
|
+
: undefined;
|
|
612
|
+
const latestWorkOrdinal = call.sessionId
|
|
613
|
+
? latestWorkOrdinalBySession.get(call.sessionId) ?? -1
|
|
614
|
+
: -1;
|
|
615
|
+
if (completion && completion.ordinal >= latestWorkOrdinal &&
|
|
616
|
+
completion.ordinal >= latestUnscopedWorkOrdinal &&
|
|
617
|
+
Date.parse(completion.observedAt) >= Date.parse(call.timestamp)) {
|
|
618
|
+
const { ordinal: _ordinal, ...evidence } = completion;
|
|
619
|
+
call.completion = evidence;
|
|
620
|
+
}
|
|
388
621
|
}
|
|
389
622
|
if (malformedLines > 0) {
|
|
390
623
|
onDiagnostic?.({ code: "malformed_jsonl", count: malformedLines });
|
|
391
624
|
}
|
|
392
625
|
return calls;
|
|
393
626
|
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
let
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
627
|
+
function createCodexRolloutParserState() {
|
|
628
|
+
return {
|
|
629
|
+
toolWorkdirs: new Map(),
|
|
630
|
+
rootSessionMetaSeen: false,
|
|
631
|
+
rootTaskStarted: false,
|
|
632
|
+
prompts: [],
|
|
633
|
+
restoredPromptOverflow: 0,
|
|
634
|
+
fileCounts: new Map(),
|
|
635
|
+
toolCallCount: 0,
|
|
636
|
+
isSubagent: false,
|
|
637
|
+
malformedLines: 0
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
/** One line of the shared Codex rollout reducer (blank/malformed included). */
|
|
641
|
+
function consumeCodexRolloutLine(state, line, onEntry) {
|
|
642
|
+
if (!line.trim())
|
|
643
|
+
return;
|
|
644
|
+
let entry;
|
|
645
|
+
try {
|
|
646
|
+
entry = JSON.parse(line);
|
|
647
|
+
}
|
|
648
|
+
catch {
|
|
649
|
+
state.malformedLines += 1;
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
if (!isRecord(entry))
|
|
653
|
+
return;
|
|
654
|
+
onEntry?.(entry);
|
|
655
|
+
const payload = isRecord(entry.payload) ? entry.payload : undefined;
|
|
656
|
+
if (entry.type === "session_meta" && payload && !state.rootSessionMetaSeen) {
|
|
657
|
+
// A forked/subagent rollout can embed the parent transcript, including
|
|
658
|
+
// many later session_meta records. The first metadata record belongs to
|
|
659
|
+
// this rollout file; later records are nested/history evidence and must
|
|
660
|
+
// never replace the financial session identity or root cwd.
|
|
661
|
+
state.rootSessionMetaSeen = true;
|
|
662
|
+
state.sessionId = stringOf(payload.id);
|
|
663
|
+
state.sourceVersion = stringOf(payload.cli_version);
|
|
664
|
+
state.rootCwd = stringOf(payload.cwd);
|
|
665
|
+
state.startedAt = toIso(stringOf(payload.timestamp) ?? stringOf(entry.timestamp));
|
|
666
|
+
state.rootStartedAtMs = timestampMilliseconds(payload.timestamp ?? entry.timestamp);
|
|
667
|
+
state.isSubagent = stringOf(payload.thread_source) === "subagent" ||
|
|
668
|
+
isRecord(payload.source) && "subagent" in payload.source;
|
|
669
|
+
state.parentSessionId = stringOf(payload.parent_thread_id);
|
|
670
|
+
}
|
|
671
|
+
if (entry.type === "turn_context" && payload) {
|
|
672
|
+
state.model = stringOf(payload.model) ?? state.model;
|
|
673
|
+
// The restored checkpoint already carries the session cwd decision when
|
|
674
|
+
// one was observed; the turn_context fallback applies only when neither
|
|
675
|
+
// a raw nor a restored session cwd exists (same ??= semantics).
|
|
676
|
+
if (state.rootCwd === undefined && !state.restoredRootCwd?.present) {
|
|
677
|
+
state.rootCwd = stringOf(payload.cwd);
|
|
421
678
|
}
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
679
|
+
}
|
|
680
|
+
if (state.isSubagent &&
|
|
681
|
+
!state.rootTaskStarted &&
|
|
682
|
+
payload?.type === "task_started" &&
|
|
683
|
+
isRootSpecificTaskStart(payload.started_at, state.rootStartedAtMs)) {
|
|
684
|
+
// Forked Codex rollouts copy the parent's complete event history after
|
|
685
|
+
// the child's root session_meta. The cumulative counter immediately
|
|
686
|
+
// before the child's first task is the inherited baseline, not child
|
|
687
|
+
// usage. Reset qualitative evidence at the same boundary so parent
|
|
688
|
+
// prompts/files cannot become the child's focus. Restored checkpoint
|
|
689
|
+
// evidence predating the boundary is parent history too and resets with
|
|
690
|
+
// it (the raw session cwd deliberately survives, exactly as rootCwd).
|
|
691
|
+
state.inheritedUsageBaseline = state.lastTotal;
|
|
692
|
+
state.lastTotal = undefined;
|
|
693
|
+
state.rootTaskStarted = true;
|
|
694
|
+
state.prompts.length = 0;
|
|
695
|
+
state.restoredPromptOverflow = 0;
|
|
696
|
+
state.fileCounts.clear();
|
|
697
|
+
state.toolWorkdirs.clear();
|
|
698
|
+
state.restoredWorkdirs = undefined;
|
|
699
|
+
state.toolCallCount = 0;
|
|
700
|
+
state.model = undefined;
|
|
701
|
+
state.lastTurn = undefined;
|
|
702
|
+
state.lastRateLimits = undefined;
|
|
703
|
+
state.lastActivityAt = toIso(stringOf(entry.timestamp)) ?? state.startedAt;
|
|
704
|
+
}
|
|
705
|
+
if (payload?.type === "task_started" && (!state.isSubagent || state.rootTaskStarted)) {
|
|
706
|
+
state.pendingTaskTurnId = stringOf(payload.turn_id);
|
|
707
|
+
state.completedTask = undefined;
|
|
708
|
+
}
|
|
709
|
+
if (entry.type === "event_msg" && payload?.type === "task_complete") {
|
|
710
|
+
const completedTurnId = stringOf(payload.turn_id);
|
|
711
|
+
const observedAt = toIso(stringOf(entry.timestamp));
|
|
712
|
+
if (state.pendingTaskTurnId && completedTurnId === state.pendingTaskTurnId && observedAt) {
|
|
713
|
+
state.completedTask = {
|
|
714
|
+
status: "completed",
|
|
715
|
+
evidence: "codex_task_complete",
|
|
716
|
+
observedAt
|
|
717
|
+
};
|
|
718
|
+
state.pendingTaskTurnId = undefined;
|
|
425
719
|
}
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
const payload = isRecord(entry.payload) ? entry.payload : undefined;
|
|
430
|
-
if (entry.type === "session_meta" && payload && !rootSessionMetaSeen) {
|
|
431
|
-
// A forked/subagent rollout can embed the parent transcript, including
|
|
432
|
-
// many later session_meta records. The first metadata record belongs to
|
|
433
|
-
// this rollout file; later records are nested/history evidence and must
|
|
434
|
-
// never replace the financial session identity or root cwd.
|
|
435
|
-
rootSessionMetaSeen = true;
|
|
436
|
-
sessionId = stringOf(payload.id);
|
|
437
|
-
rootCwd = stringOf(payload.cwd);
|
|
438
|
-
startedAt = toIso(stringOf(payload.timestamp) ?? stringOf(entry.timestamp));
|
|
439
|
-
rootStartedAtMs = timestampMilliseconds(payload.timestamp ?? entry.timestamp);
|
|
440
|
-
isSubagent = stringOf(payload.thread_source) === "subagent" || isRecord(payload.source) && "subagent" in payload.source;
|
|
441
|
-
parentSessionId = stringOf(payload.parent_thread_id);
|
|
442
|
-
}
|
|
443
|
-
if (entry.type === "turn_context" && payload) {
|
|
444
|
-
model = stringOf(payload.model) ?? model;
|
|
445
|
-
rootCwd ??= stringOf(payload.cwd);
|
|
446
|
-
}
|
|
447
|
-
if (isSubagent &&
|
|
448
|
-
!rootTaskStarted &&
|
|
449
|
-
payload?.type === "task_started" &&
|
|
450
|
-
isRootSpecificTaskStart(payload.started_at, rootStartedAtMs)) {
|
|
451
|
-
// Forked Codex rollouts copy the parent's complete event history after
|
|
452
|
-
// the child's root session_meta. The cumulative counter immediately
|
|
453
|
-
// before the child's first task is the inherited baseline, not child
|
|
454
|
-
// usage. Reset qualitative evidence at the same boundary so parent
|
|
455
|
-
// prompts/files cannot become the child's focus.
|
|
456
|
-
inheritedUsageBaseline = lastTotal;
|
|
457
|
-
lastTotal = undefined;
|
|
458
|
-
rootTaskStarted = true;
|
|
459
|
-
prompts.length = 0;
|
|
460
|
-
fileCounts.clear();
|
|
461
|
-
toolWorkdirs.clear();
|
|
462
|
-
toolCallCount = 0;
|
|
463
|
-
model = undefined;
|
|
464
|
-
lastTurn = undefined;
|
|
465
|
-
lastRateLimits = undefined;
|
|
466
|
-
lastActivityAt = toIso(stringOf(entry.timestamp)) ?? startedAt;
|
|
467
|
-
}
|
|
468
|
-
if (payload?.type === "function_call" || payload?.type === "custom_tool_call") {
|
|
469
|
-
toolCallCount += 1;
|
|
470
|
-
const args = jsonRecord(stringOf(payload.arguments));
|
|
471
|
-
const workdir = stringOf(args?.workdir) ?? stringOf(args?.cwd);
|
|
472
|
-
if (workdir && isAbsolute(workdir)) {
|
|
473
|
-
const normalized = resolve(workdir);
|
|
474
|
-
toolWorkdirs.set(normalized, (toolWorkdirs.get(normalized) ?? 0) + 1);
|
|
475
|
-
}
|
|
476
|
-
// Current Codex Desktop records the orchestration wrapper as a custom
|
|
477
|
-
// `exec` call whose input is JavaScript containing nested tool calls.
|
|
478
|
-
// Extract only quoted absolute workdir/cwd values; never evaluate it.
|
|
479
|
-
if (payload.type === "custom_tool_call" && stringOf(payload.name) === "exec") {
|
|
480
|
-
collectEmbeddedToolWorkdirs(stringOf(payload.input), toolWorkdirs);
|
|
481
|
-
}
|
|
482
|
-
collectToolFiles(args, fileCounts);
|
|
483
|
-
collectPatchFiles(stringOf(args?.patch) ?? stringOf(args?.input) ?? stringOf(payload.input), fileCounts);
|
|
720
|
+
else {
|
|
721
|
+
state.completedTask = undefined;
|
|
722
|
+
state.pendingTaskTurnId = undefined;
|
|
484
723
|
}
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
724
|
+
}
|
|
725
|
+
if (payload?.type === "function_call" || payload?.type === "custom_tool_call") {
|
|
726
|
+
state.toolCallCount += 1;
|
|
727
|
+
const args = jsonRecord(stringOf(payload.arguments));
|
|
728
|
+
const workdir = stringOf(args?.workdir) ?? stringOf(args?.cwd);
|
|
729
|
+
if (workdir && isAbsolute(workdir)) {
|
|
730
|
+
const normalized = resolve(workdir);
|
|
731
|
+
state.toolWorkdirs.set(normalized, (state.toolWorkdirs.get(normalized) ?? 0) + 1);
|
|
490
732
|
}
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
if (total) {
|
|
497
|
-
lastTotal = total;
|
|
498
|
-
lastActivityAt = eventTimestamp;
|
|
499
|
-
}
|
|
500
|
-
if (turn) {
|
|
501
|
-
lastTurn = turn;
|
|
502
|
-
lastActivityAt = eventTimestamp;
|
|
503
|
-
}
|
|
504
|
-
const rateLimits = parseCodexRateLimits(payload.rate_limits, eventTimestamp);
|
|
505
|
-
if (rateLimits) {
|
|
506
|
-
lastRateLimits = rateLimits;
|
|
507
|
-
}
|
|
733
|
+
// Current Codex Desktop records the orchestration wrapper as a custom
|
|
734
|
+
// `exec` call whose input is JavaScript containing nested tool calls.
|
|
735
|
+
// Extract only quoted absolute workdir/cwd values; never evaluate it.
|
|
736
|
+
if (payload.type === "custom_tool_call" && stringOf(payload.name) === "exec") {
|
|
737
|
+
collectEmbeddedToolWorkdirs(stringOf(payload.input), state.toolWorkdirs);
|
|
508
738
|
}
|
|
739
|
+
collectToolFiles(args, state.fileCounts);
|
|
740
|
+
collectPatchFiles(stringOf(args?.patch) ?? stringOf(args?.input) ?? stringOf(payload.input), state.fileCounts);
|
|
509
741
|
}
|
|
742
|
+
if (payload?.type === "message" && payload.role === "user") {
|
|
743
|
+
for (const prompt of textValues(payload.content)) {
|
|
744
|
+
if (isHumanPrompt(prompt))
|
|
745
|
+
state.prompts.push(prompt);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
if (entry.type === "event_msg" && payload?.type === "token_count") {
|
|
749
|
+
const eventTimestamp = toIso(stringOf(entry.timestamp)) ?? state.lastActivityAt ?? state.startedAt;
|
|
750
|
+
const info = isRecord(payload.info) ? payload.info : undefined;
|
|
751
|
+
const total = info && isRecord(info.total_token_usage) ? info.total_token_usage : undefined;
|
|
752
|
+
const turn = info && isRecord(info.last_token_usage) ? info.last_token_usage : undefined;
|
|
753
|
+
if (total) {
|
|
754
|
+
state.lastTotal = total;
|
|
755
|
+
state.lastActivityAt = eventTimestamp;
|
|
756
|
+
}
|
|
757
|
+
if (turn) {
|
|
758
|
+
state.lastTurn = turn;
|
|
759
|
+
state.lastActivityAt = eventTimestamp;
|
|
760
|
+
}
|
|
761
|
+
const rateLimits = parseCodexRateLimits(payload.rate_limits, eventTimestamp);
|
|
762
|
+
if (rateLimits) {
|
|
763
|
+
state.lastRateLimits = rateLimits;
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
/** Terminal step of the shared Codex rollout reducer. */
|
|
768
|
+
function finishCodexRolloutParse(state, onDiagnostic) {
|
|
510
769
|
// A fork without a recognized root-task boundary is ambiguous: older Codex
|
|
511
770
|
// formats may contain only inherited parent history. Omitting that child is
|
|
512
771
|
// safer than charging the parent cumulative counter again. Likewise, a
|
|
513
772
|
// recognized boundary with no later total_token_usage is not a financial
|
|
514
773
|
// call yet.
|
|
515
|
-
if (malformedLines > 0) {
|
|
516
|
-
onDiagnostic?.({ code: "malformed_jsonl", count: malformedLines });
|
|
774
|
+
if (state.malformedLines > 0) {
|
|
775
|
+
onDiagnostic?.({ code: "malformed_jsonl", count: state.malformedLines });
|
|
517
776
|
}
|
|
518
|
-
if (!lastTotal || isSubagent && !rootTaskStarted)
|
|
777
|
+
if (!state.lastTotal || state.isSubagent && !state.rootTaskStarted)
|
|
519
778
|
return [];
|
|
520
|
-
const parsedUsage = parseCodexCumulativeUsage(lastTotal, inheritedUsageBaseline);
|
|
521
|
-
const parsedTurn = lastTurn
|
|
522
|
-
? parseCodexTurnUsage(lastTurn)
|
|
779
|
+
const parsedUsage = parseCodexCumulativeUsage(state.lastTotal, state.inheritedUsageBaseline);
|
|
780
|
+
const parsedTurn = state.lastTurn
|
|
781
|
+
? parseCodexTurnUsage(state.lastTurn)
|
|
523
782
|
: { supported: true };
|
|
524
783
|
const usageSupport = parsedUsage.supported && parsedTurn.supported
|
|
525
784
|
? "complete"
|
|
@@ -527,25 +786,48 @@ export function parseCodexRollout(content, onEntry, onDiagnostic) {
|
|
|
527
786
|
if (usageSupport === "unsupported_token_shape") {
|
|
528
787
|
onDiagnostic?.({ code: "unsupported_token_shape", count: 1 });
|
|
529
788
|
}
|
|
530
|
-
const
|
|
531
|
-
|
|
789
|
+
const hasRestoredEvidence = state.restoredRootCwd !== undefined ||
|
|
790
|
+
(state.restoredWorkdirs !== undefined && state.restoredWorkdirs.length > 0);
|
|
791
|
+
let workingDirectory;
|
|
792
|
+
let workingDirectoryRef;
|
|
793
|
+
let project;
|
|
794
|
+
if (!hasRestoredEvidence) {
|
|
795
|
+
workingDirectory = absoluteWorkingDirectory(dominantCodexCwd(state.rootCwd, state.toolWorkdirs));
|
|
796
|
+
project = projectFromCwd(workingDirectory);
|
|
797
|
+
}
|
|
798
|
+
else {
|
|
799
|
+
const dominant = resolveStreamedDominantCwd(state);
|
|
800
|
+
workingDirectory = dominant.workingDirectory;
|
|
801
|
+
workingDirectoryRef = dominant.workingDirectoryRef;
|
|
802
|
+
project = dominant.project;
|
|
803
|
+
}
|
|
532
804
|
const activity = buildLocalAgentActivity({
|
|
533
|
-
prompts,
|
|
534
|
-
files: fileCounts,
|
|
535
|
-
toolCallCount,
|
|
805
|
+
prompts: state.prompts,
|
|
806
|
+
files: state.fileCounts,
|
|
807
|
+
toolCallCount: state.toolCallCount,
|
|
536
808
|
project,
|
|
537
|
-
isSubagent,
|
|
538
|
-
parentSessionId
|
|
809
|
+
isSubagent: state.isSubagent,
|
|
810
|
+
parentSessionId: state.parentSessionId,
|
|
811
|
+
...(state.restoredPromptOverflow > 0
|
|
812
|
+
? { priorPromptCount: state.restoredPromptOverflow }
|
|
813
|
+
: {})
|
|
539
814
|
});
|
|
540
815
|
return [{
|
|
541
816
|
agent: "codex",
|
|
542
|
-
model: model ?? "codex",
|
|
543
|
-
timestamp: lastActivityAt ?? startedAt ?? new Date(0).toISOString(),
|
|
544
|
-
startedAt,
|
|
817
|
+
model: state.model ?? "codex",
|
|
818
|
+
timestamp: state.lastActivityAt ?? state.startedAt ?? new Date(0).toISOString(),
|
|
819
|
+
startedAt: state.startedAt,
|
|
545
820
|
project,
|
|
546
821
|
workingDirectory,
|
|
547
|
-
|
|
548
|
-
|
|
822
|
+
...(workingDirectoryRef ? { workingDirectoryRef } : {}),
|
|
823
|
+
sessionId: state.sessionId,
|
|
824
|
+
...(state.sourceVersion ? { sourceVersion: state.sourceVersion } : {}),
|
|
825
|
+
...(state.completedTask && !state.pendingTaskTurnId &&
|
|
826
|
+
Date.parse(state.completedTask.observedAt) >=
|
|
827
|
+
Date.parse(state.lastActivityAt ?? state.startedAt ?? "")
|
|
828
|
+
? { completion: state.completedTask }
|
|
829
|
+
: {}),
|
|
830
|
+
rateLimits: state.lastRateLimits,
|
|
549
831
|
activity,
|
|
550
832
|
...(usageSupport === "complete" && parsedTurn.usage
|
|
551
833
|
? { latestTurnUsage: parsedTurn.usage }
|
|
@@ -555,9 +837,140 @@ export function parseCodexRollout(content, onEntry, onDiagnostic) {
|
|
|
555
837
|
...(parsedUsage.reportedTotalTokens !== undefined
|
|
556
838
|
? { reportedTotalTokens: parsedUsage.reportedTotalTokens }
|
|
557
839
|
: {}),
|
|
840
|
+
...(usageSupport === "complete" && parsedUsage.tokenComponentEvidence
|
|
841
|
+
? { tokenComponentEvidence: parsedUsage.tokenComponentEvidence }
|
|
842
|
+
: {}),
|
|
558
843
|
usage: parsedUsage.usage
|
|
559
844
|
}];
|
|
560
845
|
}
|
|
846
|
+
/**
|
|
847
|
+
* `dominantCodexCwd` over evidence that partially crossed a checkpoint
|
|
848
|
+
* boundary as hashes. Scores and depths reproduce the raw computation
|
|
849
|
+
* exactly (ancestor refs encode the descendant relation over resolved
|
|
850
|
+
* paths); only the terminal tie-break degrades from full-path localeCompare
|
|
851
|
+
* to basename-then-ref order when a restored entry is involved — the
|
|
852
|
+
* documented QA 27/28 divergence. A winner restored from hashes yields a
|
|
853
|
+
* ref-only working directory, matching warm cache-hit call shape.
|
|
854
|
+
*/
|
|
855
|
+
function resolveStreamedDominantCwd(state) {
|
|
856
|
+
if (state.rootCwd !== undefined) {
|
|
857
|
+
const sessionProject = projectFromCwd(state.rootCwd);
|
|
858
|
+
if (sessionProject && sessionProject !== "(home)") {
|
|
859
|
+
const workingDirectory = absoluteWorkingDirectory(state.rootCwd);
|
|
860
|
+
return { workingDirectory, project: projectFromCwd(workingDirectory) };
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
else if (state.restoredRootCwd?.present && state.restoredRootCwd.shortCircuits) {
|
|
864
|
+
return {
|
|
865
|
+
...(state.restoredRootCwd.resolvedRef
|
|
866
|
+
? { workingDirectoryRef: state.restoredRootCwd.resolvedRef }
|
|
867
|
+
: {}),
|
|
868
|
+
...(state.restoredRootCwd.resolvedProject
|
|
869
|
+
? { project: state.restoredRootCwd.resolvedProject }
|
|
870
|
+
: {})
|
|
871
|
+
};
|
|
872
|
+
}
|
|
873
|
+
const merged = mergeCodexWorkdirTallies(state);
|
|
874
|
+
if (merged.length === 0) {
|
|
875
|
+
if (state.rootCwd !== undefined) {
|
|
876
|
+
const workingDirectory = absoluteWorkingDirectory(state.rootCwd);
|
|
877
|
+
return { workingDirectory, project: projectFromCwd(workingDirectory) };
|
|
878
|
+
}
|
|
879
|
+
if (state.restoredRootCwd?.present) {
|
|
880
|
+
return {
|
|
881
|
+
...(state.restoredRootCwd.resolvedRef
|
|
882
|
+
? { workingDirectoryRef: state.restoredRootCwd.resolvedRef }
|
|
883
|
+
: {}),
|
|
884
|
+
...(state.restoredRootCwd.resolvedProject
|
|
885
|
+
? { project: state.restoredRootCwd.resolvedProject }
|
|
886
|
+
: {})
|
|
887
|
+
};
|
|
888
|
+
}
|
|
889
|
+
return {};
|
|
890
|
+
}
|
|
891
|
+
const scored = merged.map((candidate) => ({
|
|
892
|
+
candidate,
|
|
893
|
+
score: merged.reduce((total, entry) => (entry.ref === candidate.ref || entry.ancestorRefs.includes(candidate.ref)
|
|
894
|
+
? total + entry.count
|
|
895
|
+
: total), 0)
|
|
896
|
+
}));
|
|
897
|
+
scored.sort((left, right) => (right.score - left.score ||
|
|
898
|
+
left.candidate.depth - right.candidate.depth ||
|
|
899
|
+
(left.candidate.rawPath !== undefined && right.candidate.rawPath !== undefined
|
|
900
|
+
? left.candidate.rawPath.localeCompare(right.candidate.rawPath)
|
|
901
|
+
: left.candidate.base.localeCompare(right.candidate.base) ||
|
|
902
|
+
left.candidate.ref.localeCompare(right.candidate.ref))));
|
|
903
|
+
const winner = scored[0].candidate;
|
|
904
|
+
if (winner.rawPath !== undefined) {
|
|
905
|
+
return { workingDirectory: winner.rawPath, project: projectFromCwd(winner.rawPath) };
|
|
906
|
+
}
|
|
907
|
+
return {
|
|
908
|
+
workingDirectoryRef: winner.ref,
|
|
909
|
+
...(winner.base.length > 0 ? { project: winner.base } : {})
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
/** Merge restored hashed workdir tallies with this slice's raw observations. */
|
|
913
|
+
function mergeCodexWorkdirTallies(state) {
|
|
914
|
+
const home = resolve(homedir());
|
|
915
|
+
const byRef = new Map();
|
|
916
|
+
for (const restored of state.restoredWorkdirs ?? []) {
|
|
917
|
+
if (restored.isHome)
|
|
918
|
+
continue;
|
|
919
|
+
const existing = byRef.get(restored.ref);
|
|
920
|
+
if (existing)
|
|
921
|
+
existing.count += restored.count;
|
|
922
|
+
else
|
|
923
|
+
byRef.set(restored.ref, { ...restored, ancestorRefs: [...restored.ancestorRefs] });
|
|
924
|
+
}
|
|
925
|
+
for (const [rawPath, count] of state.toolWorkdirs) {
|
|
926
|
+
if (resolve(rawPath) === home)
|
|
927
|
+
continue;
|
|
928
|
+
const tally = hashedWorkdirTally(rawPath, count, home);
|
|
929
|
+
const existing = byRef.get(tally.ref);
|
|
930
|
+
if (existing) {
|
|
931
|
+
existing.count += count;
|
|
932
|
+
existing.rawPath = rawPath;
|
|
933
|
+
}
|
|
934
|
+
else {
|
|
935
|
+
byRef.set(tally.ref, { ...tally, rawPath });
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
return [...byRef.values()];
|
|
939
|
+
}
|
|
940
|
+
/** Hashed tally row for one observed (already resolved) tool workdir. */
|
|
941
|
+
function hashedWorkdirTally(path, count, home) {
|
|
942
|
+
return {
|
|
943
|
+
ref: derivedWorkingDirectoryRef(path),
|
|
944
|
+
ancestorRefs: properAncestorPaths(path).map(derivedWorkingDirectoryRef),
|
|
945
|
+
depth: path.split(sep).length,
|
|
946
|
+
base: basename(path),
|
|
947
|
+
isHome: resolve(path) === home,
|
|
948
|
+
count
|
|
949
|
+
};
|
|
950
|
+
}
|
|
951
|
+
/**
|
|
952
|
+
* Proper ancestors of a normalized absolute path, excluding the filesystem
|
|
953
|
+
* root: `dominantCodexCwd`'s descendant test (`startsWith(candidate + sep)`)
|
|
954
|
+
* never matches the bare root, so the root must not appear as an ancestor.
|
|
955
|
+
*/
|
|
956
|
+
function properAncestorPaths(path) {
|
|
957
|
+
const ancestors = [];
|
|
958
|
+
const segments = path.split(sep).filter(Boolean);
|
|
959
|
+
let current = "";
|
|
960
|
+
for (let index = 0; index < segments.length - 1; index += 1) {
|
|
961
|
+
current += `${sep}${segments[index]}`;
|
|
962
|
+
ancestors.push(current);
|
|
963
|
+
}
|
|
964
|
+
return ancestors;
|
|
965
|
+
}
|
|
966
|
+
/** Parse one Codex rollout file (JSONL event stream). Exported for tests. */
|
|
967
|
+
export function parseCodexRollout(content, onEntry, onDiagnostic) {
|
|
968
|
+
const state = createCodexRolloutParserState();
|
|
969
|
+
for (const line of content.split("\n")) {
|
|
970
|
+
consumeCodexRolloutLine(state, line, onEntry);
|
|
971
|
+
}
|
|
972
|
+
return finishCodexRolloutParse(state, onDiagnostic);
|
|
973
|
+
}
|
|
561
974
|
function collectEmbeddedToolWorkdirs(input, toolWorkdirs) {
|
|
562
975
|
if (!input)
|
|
563
976
|
return;
|
|
@@ -626,6 +1039,18 @@ function parseCodexRateLimitWindow(value) {
|
|
|
626
1039
|
export async function loadLocalAgentUsage(options = {}) {
|
|
627
1040
|
return loadLocalAgentUsageWithFormats(await localAgentFormatRuntimes(), options);
|
|
628
1041
|
}
|
|
1042
|
+
/**
|
|
1043
|
+
* Read only formats whose registry contract supports action planning.
|
|
1044
|
+
*
|
|
1045
|
+
* The launch action loop currently has truthful session/completion semantics
|
|
1046
|
+
* for Claude Code and Codex. Keeping this entrypoint registry-driven avoids an
|
|
1047
|
+
* unnecessary Gemini history walk and prevents presence-only/experimental
|
|
1048
|
+
* formats from silently entering a matched token experiment.
|
|
1049
|
+
*/
|
|
1050
|
+
export async function loadLocalAgentActionEvidence(options = {}) {
|
|
1051
|
+
const runtimes = await localAgentFormatRuntimes();
|
|
1052
|
+
return loadLocalAgentUsageWithFormats(runtimes.filter((runtime) => runtime.descriptor.capabilities.actionPlanning), options);
|
|
1053
|
+
}
|
|
629
1054
|
/**
|
|
630
1055
|
* Registry-driven ingestion engine. Exported from this module for registry
|
|
631
1056
|
* contract tests and format modules, but intentionally omitted from the
|
|
@@ -633,6 +1058,9 @@ export async function loadLocalAgentUsage(options = {}) {
|
|
|
633
1058
|
*/
|
|
634
1059
|
export async function loadLocalAgentUsageWithFormats(registry, options = {}) {
|
|
635
1060
|
validateLocalAgentFormatDescriptors(registry.map((entry) => entry.descriptor));
|
|
1061
|
+
const qualitativePolicy = options.qualitativeScan
|
|
1062
|
+
? validateQualitativeScanPolicy(options.qualitativeScan)
|
|
1063
|
+
: undefined;
|
|
636
1064
|
const home = homedir();
|
|
637
1065
|
const calls = [];
|
|
638
1066
|
const codexInvocationFiles = options.collectCodexInvocationEvidence
|
|
@@ -643,37 +1071,149 @@ export async function loadLocalAgentUsageWithFormats(registry, options = {}) {
|
|
|
643
1071
|
const sinceMs = typeof since === "number" && Number.isFinite(since) ? since : undefined;
|
|
644
1072
|
const diagnostics = [];
|
|
645
1073
|
const sourceScans = [];
|
|
1074
|
+
// One bounded header-pass budget per run. A malformed coverage ref is
|
|
1075
|
+
// dropped rather than compared: proven files then keep blocking (fail
|
|
1076
|
+
// closed) instead of reading as foreign to a garbage project.
|
|
1077
|
+
const headerPass = options.ownershipIndex
|
|
1078
|
+
? {
|
|
1079
|
+
ownershipIndex: options.ownershipIndex,
|
|
1080
|
+
...(options.coverageProjectRef &&
|
|
1081
|
+
/^avref_[a-f0-9]{64}$/.test(options.coverageProjectRef)
|
|
1082
|
+
? { coverageProjectRef: options.coverageProjectRef }
|
|
1083
|
+
: {}),
|
|
1084
|
+
probesRemaining: codexHeaderProbesPerScan
|
|
1085
|
+
}
|
|
1086
|
+
: undefined;
|
|
646
1087
|
for (const runtime of registry) {
|
|
647
1088
|
const { descriptor } = runtime;
|
|
648
1089
|
const scan = emptySourceScan(descriptor.id);
|
|
1090
|
+
if (qualitativePolicy)
|
|
1091
|
+
initializeQualitativeCoverage(scan);
|
|
649
1092
|
sourceScans.push(scan);
|
|
650
1093
|
const root = localAgentFormatRoot(descriptor, options, home);
|
|
651
|
-
|
|
1094
|
+
const discoveredFiles = await listFormatCandidateFiles(root, descriptor, scan, diagnostics);
|
|
1095
|
+
const files = qualitativePolicy
|
|
1096
|
+
? await selectBoundedQualitativeFiles(discoveredFiles, descriptor, sinceMs, qualitativePolicy,
|
|
1097
|
+
// Must match the loop's per-file derivation exactly (key alignment).
|
|
1098
|
+
Boolean(options.collectCodexInvocationEvidence) && descriptor.id === "codex" &&
|
|
1099
|
+
descriptor.capabilities.invocationEvidence, scan, diagnostics, options.qualitativeIndex, headerPass, options.streamCheckpoints)
|
|
1100
|
+
: discoveredFiles.map((filePath) => ({ filePath }));
|
|
1101
|
+
for (const selected of files) {
|
|
1102
|
+
const file = selected.filePath;
|
|
652
1103
|
if (!matchesLocalAgentFormatFile(descriptor, file))
|
|
653
1104
|
continue;
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
1105
|
+
const collectInvocationEvidence = Boolean(codexInvocationFiles) &&
|
|
1106
|
+
descriptor.id === "codex" && descriptor.capabilities.invocationEvidence;
|
|
1107
|
+
let indexed;
|
|
1108
|
+
if (qualitativePolicy && selected.streamedValue) {
|
|
1109
|
+
// A checkpointed stream completed this run; its index entry is
|
|
1110
|
+
// already persisted. Consume the value like a fresh complete read.
|
|
1111
|
+
indexed = selected.streamedValue;
|
|
1112
|
+
scan.qualitativeFilesReadCompletely =
|
|
1113
|
+
(scan.qualitativeFilesReadCompletely ?? 0) + 1;
|
|
657
1114
|
}
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
1115
|
+
else if (qualitativePolicy && options.qualitativeIndex && selected.indexKey) {
|
|
1116
|
+
try {
|
|
1117
|
+
const candidate = await options.qualitativeIndex.read(selected.indexKey);
|
|
1118
|
+
if (candidate && await qualitativeIndexKeyStillCurrent(file, selected.indexKey)) {
|
|
1119
|
+
if (isQualitativeIndexValue(candidate, descriptor, collectInvocationEvidence)) {
|
|
1120
|
+
indexed = candidate;
|
|
1121
|
+
scan.qualitativeIndexHits = (scan.qualitativeIndexHits ?? 0) + 1;
|
|
1122
|
+
scan.qualitativeBytesReused =
|
|
1123
|
+
(scan.qualitativeBytesReused ?? 0) + (selected.fileSize ?? 0);
|
|
1124
|
+
scan.qualitativeFilesReadCompletely =
|
|
1125
|
+
(scan.qualitativeFilesReadCompletely ?? 0) + 1;
|
|
1126
|
+
}
|
|
1127
|
+
else {
|
|
1128
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
// A candidate whose file identity moved on mid-scan is a benign
|
|
1132
|
+
// change, not an index failure; the fresh-read path fails it closed.
|
|
1133
|
+
}
|
|
1134
|
+
catch {
|
|
1135
|
+
if (!selected.probeErrored) {
|
|
1136
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
661
1139
|
}
|
|
662
|
-
|
|
1140
|
+
// A file admitted only because a validated selection-time hit exempted
|
|
1141
|
+
// it from the byte budget must never fall through to an unbudgeted
|
|
1142
|
+
// fresh read when the loop probe misses (concurrent GC, transient FS
|
|
1143
|
+
// error). Fail closed: record the skip, coverage stays "indexing", and
|
|
1144
|
+
// the next scan re-probes.
|
|
1145
|
+
if (!indexed && selected.budgetExempt) {
|
|
1146
|
+
recordQualitativeBudgetSkip(scan);
|
|
663
1147
|
continue;
|
|
1148
|
+
}
|
|
1149
|
+
let parsed;
|
|
1150
|
+
if (indexed) {
|
|
1151
|
+
parsed = indexed;
|
|
1152
|
+
}
|
|
1153
|
+
else {
|
|
1154
|
+
let content;
|
|
1155
|
+
try {
|
|
1156
|
+
if (qualitativePolicy) {
|
|
1157
|
+
const bounded = await readBoundedUtf8File(file, selected.maxReadBytes, selected.indexKey.fileIdentity);
|
|
1158
|
+
content = bounded.content;
|
|
1159
|
+
scan.qualitativeBytesRead = (scan.qualitativeBytesRead ?? 0) + bounded.bytesRead;
|
|
1160
|
+
scan.qualitativeFilesReadCompletely =
|
|
1161
|
+
(scan.qualitativeFilesReadCompletely ?? 0) + 1;
|
|
1162
|
+
}
|
|
1163
|
+
else {
|
|
1164
|
+
content = await readFile(file, "utf8");
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
catch (error) {
|
|
1168
|
+
if (qualitativePolicy && error instanceof QualitativeReadLimitError) {
|
|
1169
|
+
recordQualitativeBudgetSkip(scan);
|
|
1170
|
+
continue;
|
|
1171
|
+
}
|
|
1172
|
+
recordUnreadableFile(descriptor.id, scan, diagnostics, error);
|
|
1173
|
+
if (qualitativePolicy)
|
|
1174
|
+
scan.qualitativeCoverage = "partial";
|
|
1175
|
+
continue;
|
|
1176
|
+
}
|
|
1177
|
+
if (!content)
|
|
1178
|
+
continue;
|
|
1179
|
+
const fileDiagnostics = [];
|
|
1180
|
+
const fresh = runtime.parseFull({
|
|
1181
|
+
content,
|
|
1182
|
+
filePath: file,
|
|
1183
|
+
sinceMs,
|
|
1184
|
+
collectInvocationEvidence,
|
|
1185
|
+
onDiagnostic: (diagnostic) => fileDiagnostics.push(diagnostic)
|
|
1186
|
+
});
|
|
1187
|
+
parsed = {
|
|
1188
|
+
calls: fresh.calls,
|
|
1189
|
+
...(fresh.invocationFile ? { invocationFile: fresh.invocationFile } : {}),
|
|
1190
|
+
...(fresh.invocationWindowProof
|
|
1191
|
+
? { invocationWindowProof: fresh.invocationWindowProof }
|
|
1192
|
+
: {}),
|
|
1193
|
+
diagnostics: fileDiagnostics
|
|
1194
|
+
};
|
|
1195
|
+
// Never persist a registry ownership violation, even through a
|
|
1196
|
+
// caller-supplied adapter. Preserve the legacy fail-fast contract.
|
|
1197
|
+
assertFormatCallOwnership(descriptor, parsed.calls);
|
|
1198
|
+
assertInvocationOwnership(descriptor, parsed.invocationFile, collectInvocationEvidence);
|
|
1199
|
+
if (qualitativePolicy && options.qualitativeIndex && selected.indexKey) {
|
|
1200
|
+
try {
|
|
1201
|
+
await options.qualitativeIndex.write(selected.indexKey, parsed);
|
|
1202
|
+
}
|
|
1203
|
+
catch {
|
|
1204
|
+
// One broken store document should read as one failure: the
|
|
1205
|
+
// selection probe already counted it when it errored there.
|
|
1206
|
+
if (!selected.probeErrored) {
|
|
1207
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
664
1212
|
filesParsed += 1;
|
|
665
1213
|
scan.filesParsed += 1;
|
|
666
|
-
const
|
|
667
|
-
descriptor.id
|
|
668
|
-
|
|
669
|
-
content,
|
|
670
|
-
filePath: file,
|
|
671
|
-
sinceMs,
|
|
672
|
-
collectInvocationEvidence,
|
|
673
|
-
onDiagnostic: (diagnostic) => {
|
|
674
|
-
recordParseDiagnostic(descriptor.id, scan, diagnostics, diagnostic);
|
|
675
|
-
}
|
|
676
|
-
});
|
|
1214
|
+
for (const diagnostic of parsed.diagnostics) {
|
|
1215
|
+
recordParseDiagnostic(descriptor.id, scan, diagnostics, diagnostic);
|
|
1216
|
+
}
|
|
677
1217
|
assertFormatCallOwnership(descriptor, parsed.calls);
|
|
678
1218
|
assertInvocationOwnership(descriptor, parsed.invocationFile, collectInvocationEvidence);
|
|
679
1219
|
calls.push(...parsed.calls);
|
|
@@ -681,6 +1221,8 @@ export async function loadLocalAgentUsageWithFormats(registry, options = {}) {
|
|
|
681
1221
|
codexInvocationFiles.push(parsed.invocationFile);
|
|
682
1222
|
}
|
|
683
1223
|
}
|
|
1224
|
+
if (qualitativePolicy)
|
|
1225
|
+
finishQualitativeCoverage(scan, diagnostics);
|
|
684
1226
|
}
|
|
685
1227
|
const normalizedCalls = dedupeCumulativeSessionCalls(calls, (agent) => {
|
|
686
1228
|
const scan = sourceScans.find((entry) => entry.agent === agent);
|
|
@@ -729,9 +1271,71 @@ export async function loadLocalAgentFinancialUsageWithFormats(registry, options
|
|
|
729
1271
|
const scan = financialSourceScan(descriptor);
|
|
730
1272
|
const calls = [];
|
|
731
1273
|
const root = localAgentFormatRoot(descriptor, options, home);
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
1274
|
+
const candidateFiles = (await listFormatCandidateFiles(root, descriptor, scan, diagnostics))
|
|
1275
|
+
.filter((file) => matchesLocalAgentFormatFile(descriptor, file));
|
|
1276
|
+
// Per-file financial cache (req 5's financial half). The newest file per
|
|
1277
|
+
// source always parses fresh: the active session's evidence — including
|
|
1278
|
+
// its raw working directory for downstream context inference — must never
|
|
1279
|
+
// come from a privacy-reduced cache entry.
|
|
1280
|
+
const financialKeys = new Map();
|
|
1281
|
+
let newestFile;
|
|
1282
|
+
if (options.financialIndex) {
|
|
1283
|
+
let newestRecency = Number.NEGATIVE_INFINITY;
|
|
1284
|
+
for (const file of candidateFiles) {
|
|
1285
|
+
const metadata = await lstat(file).catch(() => undefined);
|
|
1286
|
+
if (!metadata || !metadata.isFile() || metadata.isSymbolicLink())
|
|
1287
|
+
continue;
|
|
1288
|
+
financialKeys.set(file, {
|
|
1289
|
+
schemaVersion: 2,
|
|
1290
|
+
section: "financial",
|
|
1291
|
+
agent: descriptor.id,
|
|
1292
|
+
pathHash: createHash("sha256").update(resolve(file)).digest("hex"),
|
|
1293
|
+
fileIdentity: qualitativeFileIdentity(metadata),
|
|
1294
|
+
financialParserVersion: localAgentFinancialParserVersion
|
|
1295
|
+
});
|
|
1296
|
+
const recency = Math.max(metadata.mtimeMs, metadata.ctimeMs, metadata.birthtimeMs);
|
|
1297
|
+
if (recency > newestRecency) {
|
|
1298
|
+
newestRecency = recency;
|
|
1299
|
+
newestFile = file;
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
for (const file of candidateFiles) {
|
|
1304
|
+
const financialKey = financialKeys.get(file);
|
|
1305
|
+
// Any newest file may reuse its cache when its identity is
|
|
1306
|
+
// byte-unchanged — cached evidence is then exact. Codex additionally
|
|
1307
|
+
// recovers its root working directory from the bounded session_meta
|
|
1308
|
+
// first line; Claude reuse serves refs only and context inference
|
|
1309
|
+
// falls back to the requested project path. An actively growing
|
|
1310
|
+
// session changes identity and always parses fresh.
|
|
1311
|
+
if (options.financialIndex && financialKey) {
|
|
1312
|
+
try {
|
|
1313
|
+
const cached = await options.financialIndex.read(financialKey);
|
|
1314
|
+
if (cached && file === newestFile &&
|
|
1315
|
+
!await financialIdentityStillCurrent(file, financialKey)) {
|
|
1316
|
+
throw new Error("stale newest identity");
|
|
1317
|
+
}
|
|
1318
|
+
if (cached) {
|
|
1319
|
+
if (file === newestFile && descriptor.id === "codex") {
|
|
1320
|
+
await attachProbedRootCwd(file, cached.calls, financialKey.fileIdentity);
|
|
1321
|
+
}
|
|
1322
|
+
const cachedCalls = cached.calls;
|
|
1323
|
+
assertFormatCallOwnership(descriptor, cachedCalls);
|
|
1324
|
+
scan.filesParsed += 1;
|
|
1325
|
+
for (const diagnostic of cached.diagnostics) {
|
|
1326
|
+
recordParseDiagnostic(descriptor.id, scan, diagnostics, diagnostic);
|
|
1327
|
+
}
|
|
1328
|
+
calls.push(...cachedCalls);
|
|
1329
|
+
continue;
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
catch {
|
|
1333
|
+
// A failing cache never blocks the authoritative fresh parse.
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
const diagnosticsBefore = diagnostics.length;
|
|
1337
|
+
const unreadableBefore = scan.unreadableFiles;
|
|
1338
|
+
const filesParsedBefore = scan.filesParsed;
|
|
735
1339
|
const parsedCalls = await runtime.parseFinancialFile({
|
|
736
1340
|
filePath: file,
|
|
737
1341
|
sinceMs,
|
|
@@ -741,6 +1345,23 @@ export async function loadLocalAgentFinancialUsageWithFormats(registry, options
|
|
|
741
1345
|
assertFormatCallOwnership(descriptor, parsedCalls);
|
|
742
1346
|
assertFinancialSourceOwnership(descriptor, scan, diagnostics);
|
|
743
1347
|
calls.push(...parsedCalls);
|
|
1348
|
+
// Cache only clean, content-bearing parses under the identity captured
|
|
1349
|
+
// BEFORE the read: a file that changed mid-parse fails the next
|
|
1350
|
+
// identity check instead of serving mixed evidence.
|
|
1351
|
+
if (options.financialIndex && financialKey &&
|
|
1352
|
+
scan.filesParsed === filesParsedBefore + 1 &&
|
|
1353
|
+
scan.unreadableFiles === unreadableBefore) {
|
|
1354
|
+
const parseDiagnostics = diagnostics.slice(diagnosticsBefore)
|
|
1355
|
+
.filter((entry) => (entry.code === "malformed_jsonl" ||
|
|
1356
|
+
entry.code === "malformed_session_file" ||
|
|
1357
|
+
entry.code === "unsupported_token_shape"))
|
|
1358
|
+
.map((entry) => ({ code: entry.code,
|
|
1359
|
+
count: entry.count }));
|
|
1360
|
+
await options.financialIndex.write(financialKey, {
|
|
1361
|
+
calls: parsedCalls,
|
|
1362
|
+
diagnostics: parseDiagnostics
|
|
1363
|
+
}).catch(() => undefined);
|
|
1364
|
+
}
|
|
744
1365
|
}
|
|
745
1366
|
return { calls, scan, diagnostics };
|
|
746
1367
|
}));
|
|
@@ -768,6 +1389,139 @@ export async function loadLocalAgentFinancialUsageWithFormats(registry, options
|
|
|
768
1389
|
diagnostics: scanned.flatMap((entry) => entry.diagnostics)
|
|
769
1390
|
};
|
|
770
1391
|
}
|
|
1392
|
+
/** Whether a financial cache key still matches the file's on-disk identity. */
|
|
1393
|
+
async function financialIdentityStillCurrent(filePath, key) {
|
|
1394
|
+
const metadata = await lstat(filePath).catch(() => undefined);
|
|
1395
|
+
if (!metadata || !metadata.isFile() || metadata.isSymbolicLink())
|
|
1396
|
+
return false;
|
|
1397
|
+
return qualitativeFileIdentity(metadata) === key.fileIdentity;
|
|
1398
|
+
}
|
|
1399
|
+
/**
|
|
1400
|
+
* Recover the newest Codex rollout's root working directory from its bounded
|
|
1401
|
+
* session_meta first line and re-attach it to cached calls whose privacy-
|
|
1402
|
+
* reduced ref matches. Reads at most 256 KiB; never guesses on mismatch.
|
|
1403
|
+
*/
|
|
1404
|
+
async function attachProbedRootCwd(filePath, calls, expectedIdentity) {
|
|
1405
|
+
const header = await probeCodexRolloutHeader(filePath, expectedIdentity);
|
|
1406
|
+
const cwd = header?.cwd;
|
|
1407
|
+
if (!cwd)
|
|
1408
|
+
return;
|
|
1409
|
+
const ref = derivedWorkingDirectoryRef(cwd);
|
|
1410
|
+
for (const call of calls) {
|
|
1411
|
+
if (call.workingDirectoryRef === ref && !call.workingDirectory) {
|
|
1412
|
+
call.workingDirectory = cwd;
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
const codexHeaderProbeBytes = 256 * 1024;
|
|
1417
|
+
/**
|
|
1418
|
+
* Bounded Codex rollout header probe (design section c). Opens the exact
|
|
1419
|
+
* inode (O_NOFOLLOW), binds both boundary stats to `expectedIdentity` when
|
|
1420
|
+
* one is supplied, reads at most the first 256 KiB, and abstractly parses
|
|
1421
|
+
* only the first complete line as a root `session_meta` record. Every
|
|
1422
|
+
* failure — oversized first line, non-session_meta first line, malformed
|
|
1423
|
+
* JSON, identity drift mid-probe — returns undefined: unknown, never an
|
|
1424
|
+
* error and never a guess. The raw cwd never escapes its callers beyond
|
|
1425
|
+
* deriving an avref or re-attaching evidence that already carried the
|
|
1426
|
+
* byte-identical ref.
|
|
1427
|
+
*/
|
|
1428
|
+
async function probeCodexRolloutHeader(filePath, expectedIdentity) {
|
|
1429
|
+
let firstLine;
|
|
1430
|
+
try {
|
|
1431
|
+
const handle = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
1432
|
+
try {
|
|
1433
|
+
const before = await handle.stat();
|
|
1434
|
+
if (!before.isFile() ||
|
|
1435
|
+
expectedIdentity !== undefined &&
|
|
1436
|
+
qualitativeFileIdentity(before) !== expectedIdentity) {
|
|
1437
|
+
return undefined;
|
|
1438
|
+
}
|
|
1439
|
+
const probe = Buffer.allocUnsafe(codexHeaderProbeBytes);
|
|
1440
|
+
let bytesRead = 0;
|
|
1441
|
+
let sawEof = false;
|
|
1442
|
+
while (bytesRead < probe.length) {
|
|
1443
|
+
const read = await handle.read(probe, bytesRead, probe.length - bytesRead, bytesRead);
|
|
1444
|
+
if (read.bytesRead === 0) {
|
|
1445
|
+
sawEof = true;
|
|
1446
|
+
break;
|
|
1447
|
+
}
|
|
1448
|
+
bytesRead += read.bytesRead;
|
|
1449
|
+
if (probe.subarray(0, bytesRead).includes(0x0a))
|
|
1450
|
+
break;
|
|
1451
|
+
}
|
|
1452
|
+
const window = probe.subarray(0, bytesRead);
|
|
1453
|
+
const newlineAt = window.indexOf(0x0a);
|
|
1454
|
+
if (newlineAt > 0) {
|
|
1455
|
+
firstLine = Buffer.from(window.subarray(0, newlineAt));
|
|
1456
|
+
}
|
|
1457
|
+
else if (newlineAt === -1 && sawEof && bytesRead > 0) {
|
|
1458
|
+
// EOF inside the window: the whole file is one complete first line.
|
|
1459
|
+
firstLine = Buffer.from(window);
|
|
1460
|
+
}
|
|
1461
|
+
const after = await handle.stat();
|
|
1462
|
+
if (!after.isFile() ||
|
|
1463
|
+
expectedIdentity !== undefined &&
|
|
1464
|
+
qualitativeFileIdentity(after) !== expectedIdentity) {
|
|
1465
|
+
return undefined;
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
finally {
|
|
1469
|
+
await handle.close().catch(() => undefined);
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
catch {
|
|
1473
|
+
return undefined;
|
|
1474
|
+
}
|
|
1475
|
+
if (!firstLine)
|
|
1476
|
+
return undefined;
|
|
1477
|
+
let record;
|
|
1478
|
+
try {
|
|
1479
|
+
record = JSON.parse(firstLine.toString("utf8"));
|
|
1480
|
+
}
|
|
1481
|
+
catch {
|
|
1482
|
+
return undefined;
|
|
1483
|
+
}
|
|
1484
|
+
if (!isRecord(record) || record.type !== "session_meta" || !isRecord(record.payload)) {
|
|
1485
|
+
return undefined;
|
|
1486
|
+
}
|
|
1487
|
+
const payload = record.payload;
|
|
1488
|
+
const cwd = stringOf(payload.cwd);
|
|
1489
|
+
const parentSessionId = stringOf(payload.parent_thread_id);
|
|
1490
|
+
const forkedFromId = stringOf(payload.forked_from_id);
|
|
1491
|
+
return {
|
|
1492
|
+
...(cwd && isAbsolute(cwd) ? { cwd } : {}),
|
|
1493
|
+
isSubagent: stringOf(payload.thread_source) === "subagent" ||
|
|
1494
|
+
isRecord(payload.source) && "subagent" in payload.source,
|
|
1495
|
+
...(parentSessionId ? { parentSessionId } : {}),
|
|
1496
|
+
...(forkedFromId ? { forkedFromId } : {})
|
|
1497
|
+
};
|
|
1498
|
+
}
|
|
1499
|
+
/**
|
|
1500
|
+
* MINOR-5 attribution parity: "proven" requires exactly the condition under
|
|
1501
|
+
* which `dominantCodexCwd` short-circuits to the session cwd — an absolute
|
|
1502
|
+
* path whose `projectFromCwd` label is a real project (not "(home)"; cwd "/"
|
|
1503
|
+
* has an empty basename and falls through to body evidence, QA case 36). A
|
|
1504
|
+
* full parse of such a rollout is guaranteed to emit this same directory as
|
|
1505
|
+
* its only call's working directory, so the derived ref matches byte-for-
|
|
1506
|
+
* byte. Everything else — home, "/", relative, absent, unparseable — stays
|
|
1507
|
+
* unknown and keeps blocking; unknown is never treated as foreign.
|
|
1508
|
+
*/
|
|
1509
|
+
function codexHeaderAttribution(header) {
|
|
1510
|
+
const marker = header ? { isSubagent: header.isSubagent } : {};
|
|
1511
|
+
const cwd = header?.cwd;
|
|
1512
|
+
if (!cwd || !isAbsolute(cwd))
|
|
1513
|
+
return { status: "unknown", ...marker };
|
|
1514
|
+
const project = projectFromCwd(cwd);
|
|
1515
|
+
if (!project || project === "(home)")
|
|
1516
|
+
return { status: "unknown", ...marker };
|
|
1517
|
+
// Refs derive from the resolved directory: `absoluteWorkingDirectory`
|
|
1518
|
+
// resolves the short-circuited session cwd before the call is emitted.
|
|
1519
|
+
return {
|
|
1520
|
+
status: "proven",
|
|
1521
|
+
projectRef: derivedWorkingDirectoryRef(resolve(cwd)),
|
|
1522
|
+
...marker
|
|
1523
|
+
};
|
|
1524
|
+
}
|
|
771
1525
|
/** @internal Runtime hook owned by the Claude Code registry entry. */
|
|
772
1526
|
export async function readClaudeCodeFinancialFileForRegistry(context) {
|
|
773
1527
|
const { filePath, sinceMs, scan, diagnostics } = context;
|
|
@@ -780,7 +1534,11 @@ export async function readClaudeCodeFinancialFileForRegistry(context) {
|
|
|
780
1534
|
let streamed;
|
|
781
1535
|
try {
|
|
782
1536
|
streamed = await streamJsonlRecords(filePath, (entry) => {
|
|
783
|
-
const call = parseClaudeFinancialEntry(entry, filePath,
|
|
1537
|
+
const call = parseClaudeFinancialEntry(entry, filePath,
|
|
1538
|
+
// Window-blind on purpose: cached values must contain the complete
|
|
1539
|
+
// file so a narrow-window run can never truncate a wider one. The
|
|
1540
|
+
// loader's final timestamp filter performs all narrowing.
|
|
1541
|
+
undefined, seen, (diagnostic) => fileDiagnostics.push(diagnostic));
|
|
784
1542
|
if (call)
|
|
785
1543
|
calls.push(call);
|
|
786
1544
|
});
|
|
@@ -920,6 +1678,975 @@ function assertInvocationOwnership(descriptor, invocationFile, collectionAllowed
|
|
|
920
1678
|
throw new Error(`Local-agent format ${descriptor.id} emitted invocation evidence for a different source.`);
|
|
921
1679
|
}
|
|
922
1680
|
}
|
|
1681
|
+
class QualitativeReadLimitError extends Error {
|
|
1682
|
+
constructor() {
|
|
1683
|
+
super("Qualitative transcript changed beyond the configured byte limit while being read.");
|
|
1684
|
+
this.name = "QualitativeReadLimitError";
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
function validateQualitativeScanPolicy(policy) {
|
|
1688
|
+
for (const [name, value] of Object.entries(policy)) {
|
|
1689
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
1690
|
+
throw new RangeError(`${name} must be a positive safe integer byte limit.`);
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
return policy;
|
|
1694
|
+
}
|
|
1695
|
+
function initializeQualitativeCoverage(scan) {
|
|
1696
|
+
scan.filesSkippedBeforeWindow = 0;
|
|
1697
|
+
scan.qualitativeCoverage = "complete";
|
|
1698
|
+
scan.qualitativeFilesEligible = 0;
|
|
1699
|
+
scan.qualitativeFilesSkippedForBudget = 0;
|
|
1700
|
+
scan.qualitativeFilesSelected = 0;
|
|
1701
|
+
scan.qualitativeFilesReadCompletely = 0;
|
|
1702
|
+
scan.qualitativeBytesEligible = 0;
|
|
1703
|
+
scan.qualitativeBytesSelected = 0;
|
|
1704
|
+
scan.qualitativeBytesRead = 0;
|
|
1705
|
+
scan.qualitativeBytesReused = 0;
|
|
1706
|
+
scan.qualitativeIndexHits = 0;
|
|
1707
|
+
scan.qualitativeIndexErrors = 0;
|
|
1708
|
+
scan.qualitativeFilesIndexed = 0;
|
|
1709
|
+
scan.qualitativeFilesForeignProven = 0;
|
|
1710
|
+
scan.qualitativeFilesOwnershipUnknown = 0;
|
|
1711
|
+
scan.qualitativeSelectedEvidence = "complete_files_only";
|
|
1712
|
+
}
|
|
1713
|
+
/**
|
|
1714
|
+
* Select complete files, never byte ranges. A partial session can make a
|
|
1715
|
+
* cumulative counter look like a turn or make inherited Codex history look
|
|
1716
|
+
* like child usage, so a file that does not fit is omitted as a whole.
|
|
1717
|
+
*/
|
|
1718
|
+
async function selectBoundedQualitativeFiles(discoveredFiles, descriptor, sinceMs, policy, collectInvocationEvidence, scan, diagnostics, qualitativeIndex, headerPass, streamCheckpoints) {
|
|
1719
|
+
const eligible = [];
|
|
1720
|
+
for (const filePath of discoveredFiles) {
|
|
1721
|
+
if (!matchesLocalAgentFormatFile(descriptor, filePath))
|
|
1722
|
+
continue;
|
|
1723
|
+
let metadata;
|
|
1724
|
+
try {
|
|
1725
|
+
metadata = await lstat(filePath);
|
|
1726
|
+
}
|
|
1727
|
+
catch (error) {
|
|
1728
|
+
recordUnreadableFile(descriptor.id, scan, diagnostics, error);
|
|
1729
|
+
scan.qualitativeCoverage = "partial";
|
|
1730
|
+
continue;
|
|
1731
|
+
}
|
|
1732
|
+
// Discovery accepts regular files only. Re-check immediately before the
|
|
1733
|
+
// read so a path replacement cannot redirect a local scan through a link.
|
|
1734
|
+
if (!metadata.isFile() || metadata.isSymbolicLink() ||
|
|
1735
|
+
!Number.isSafeInteger(metadata.size) || metadata.size < 0) {
|
|
1736
|
+
recordUnreadableFile(descriptor.id, scan, diagnostics, newErrorWithCode("EINVAL"));
|
|
1737
|
+
scan.qualitativeCoverage = "partial";
|
|
1738
|
+
continue;
|
|
1739
|
+
}
|
|
1740
|
+
// Match the financial reader's conservative proof: ctime catches a path
|
|
1741
|
+
// whose mtime was restored after replacement, and birthtime catches a
|
|
1742
|
+
// recent copy that preserved an older mtime.
|
|
1743
|
+
const newestFileEvidence = Math.max(metadata.mtimeMs, metadata.ctimeMs, metadata.birthtimeMs);
|
|
1744
|
+
if (typeof sinceMs === "number" && Number.isFinite(newestFileEvidence) &&
|
|
1745
|
+
newestFileEvidence < sinceMs) {
|
|
1746
|
+
scan.filesSkippedBeforeWindow = (scan.filesSkippedBeforeWindow ?? 0) + 1;
|
|
1747
|
+
continue;
|
|
1748
|
+
}
|
|
1749
|
+
scan.qualitativeFilesEligible = (scan.qualitativeFilesEligible ?? 0) + 1;
|
|
1750
|
+
scan.qualitativeBytesEligible = (scan.qualitativeBytesEligible ?? 0) + metadata.size;
|
|
1751
|
+
eligible.push({
|
|
1752
|
+
filePath,
|
|
1753
|
+
size: metadata.size,
|
|
1754
|
+
recencyMs: newestFileEvidence,
|
|
1755
|
+
dev: metadata.dev,
|
|
1756
|
+
ino: metadata.ino,
|
|
1757
|
+
mtimeMs: metadata.mtimeMs,
|
|
1758
|
+
ctimeMs: metadata.ctimeMs,
|
|
1759
|
+
birthtimeMs: metadata.birthtimeMs
|
|
1760
|
+
});
|
|
1761
|
+
}
|
|
1762
|
+
// Recent evidence is the most useful bounded prefix. Path order breaks
|
|
1763
|
+
// equal-mtime ties so repeated scans of an unchanged tree are deterministic.
|
|
1764
|
+
eligible.sort((left, right) => (right.recencyMs - left.recencyMs || left.filePath.localeCompare(right.filePath)));
|
|
1765
|
+
const selected = [];
|
|
1766
|
+
const streamCandidates = [];
|
|
1767
|
+
let remaining = policy.maxSourceBytes;
|
|
1768
|
+
for (const file of eligible) {
|
|
1769
|
+
let probeErrored = false;
|
|
1770
|
+
// Probe the private index before any budget decision (req 5): a file with
|
|
1771
|
+
// validated cached evidence is admitted regardless of size and consumes
|
|
1772
|
+
// no byte budget. The probe is trusted for budget exemption only after
|
|
1773
|
+
// the same identity re-check the parse loop performs.
|
|
1774
|
+
if (qualitativeIndex) {
|
|
1775
|
+
const candidateKey = qualitativeIndexKey(descriptor.id, file.filePath, file, sinceMs, collectInvocationEvidence);
|
|
1776
|
+
let cachedHit = false;
|
|
1777
|
+
try {
|
|
1778
|
+
const candidate = await qualitativeIndex.read(candidateKey);
|
|
1779
|
+
cachedHit = Boolean(candidate &&
|
|
1780
|
+
await qualitativeIndexKeyStillCurrent(file.filePath, candidateKey) &&
|
|
1781
|
+
isQualitativeIndexValue(candidate, descriptor, collectInvocationEvidence));
|
|
1782
|
+
}
|
|
1783
|
+
catch {
|
|
1784
|
+
probeErrored = true;
|
|
1785
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
1786
|
+
}
|
|
1787
|
+
if (cachedHit) {
|
|
1788
|
+
selected.push({
|
|
1789
|
+
filePath: file.filePath,
|
|
1790
|
+
maxReadBytes: file.size,
|
|
1791
|
+
fileSize: file.size,
|
|
1792
|
+
indexKey: candidateKey,
|
|
1793
|
+
budgetExempt: true,
|
|
1794
|
+
probeErrored
|
|
1795
|
+
});
|
|
1796
|
+
scan.qualitativeFilesSelected = (scan.qualitativeFilesSelected ?? 0) + 1;
|
|
1797
|
+
continue;
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
// Oversized Codex rollouts route to the bounded streaming pass instead
|
|
1801
|
+
// of a permanent skip whenever both index sections are available; they
|
|
1802
|
+
// never consume the whole-file byte budget (req 5).
|
|
1803
|
+
if (streamCheckpoints && qualitativeIndex && descriptor.id === "codex" &&
|
|
1804
|
+
file.size > policy.maxFileBytes) {
|
|
1805
|
+
streamCandidates.push(file);
|
|
1806
|
+
continue;
|
|
1807
|
+
}
|
|
1808
|
+
if (file.size > policy.maxFileBytes || file.size > remaining) {
|
|
1809
|
+
recordQualitativeBudgetSkip(scan);
|
|
1810
|
+
// A budget-skipped Codex file still gets a bounded header pass so a
|
|
1811
|
+
// proven-foreign attribution can honestly narrow the requested
|
|
1812
|
+
// project's blocking set without ever reading the body. Claude files
|
|
1813
|
+
// get no header shortcut (their cwd can change mid-file).
|
|
1814
|
+
if (headerPass && descriptor.id === "codex") {
|
|
1815
|
+
await recordBudgetSkippedCodexHeader(file, headerPass, scan);
|
|
1816
|
+
}
|
|
1817
|
+
continue;
|
|
1818
|
+
}
|
|
1819
|
+
selected.push({
|
|
1820
|
+
filePath: file.filePath,
|
|
1821
|
+
// Bind the read to the selected metadata snapshot. Growth is treated as
|
|
1822
|
+
// incomplete coverage rather than silently consuming another file's
|
|
1823
|
+
// reserved source budget.
|
|
1824
|
+
maxReadBytes: file.size,
|
|
1825
|
+
fileSize: file.size,
|
|
1826
|
+
indexKey: qualitativeIndexKey(descriptor.id, file.filePath, file, sinceMs, collectInvocationEvidence),
|
|
1827
|
+
probeErrored: probeErrored || undefined
|
|
1828
|
+
});
|
|
1829
|
+
scan.qualitativeFilesSelected = (scan.qualitativeFilesSelected ?? 0) + 1;
|
|
1830
|
+
scan.qualitativeBytesSelected = (scan.qualitativeBytesSelected ?? 0) + file.size;
|
|
1831
|
+
remaining -= file.size;
|
|
1832
|
+
}
|
|
1833
|
+
if (streamCandidates.length > 0 && streamCheckpoints && qualitativeIndex) {
|
|
1834
|
+
const streamed = await runCodexStreamingPass(streamCandidates, {
|
|
1835
|
+
descriptor,
|
|
1836
|
+
qualitativeIndex,
|
|
1837
|
+
streamCheckpoints,
|
|
1838
|
+
...(headerPass ? { ownershipIndex: headerPass.ownershipIndex } : {}),
|
|
1839
|
+
collectInvocationEvidence,
|
|
1840
|
+
sinceMs,
|
|
1841
|
+
scan,
|
|
1842
|
+
remainingBytes: policy.maxStreamedBytesPerRun ?? defaultStreamedBytesPerRun
|
|
1843
|
+
});
|
|
1844
|
+
for (const file of streamCandidates) {
|
|
1845
|
+
const value = streamed.get(file.filePath);
|
|
1846
|
+
if (value) {
|
|
1847
|
+
selected.push({
|
|
1848
|
+
filePath: file.filePath,
|
|
1849
|
+
fileSize: file.size,
|
|
1850
|
+
streamedValue: value
|
|
1851
|
+
});
|
|
1852
|
+
scan.qualitativeFilesSelected = (scan.qualitativeFilesSelected ?? 0) + 1;
|
|
1853
|
+
continue;
|
|
1854
|
+
}
|
|
1855
|
+
// Unconverged this run: same honest accounting as a budget skip (the
|
|
1856
|
+
// body was not fully parsed), plus explicit stream-progress counters
|
|
1857
|
+
// and the A4a header-pass ownership classification.
|
|
1858
|
+
recordQualitativeBudgetSkip(scan);
|
|
1859
|
+
scan.qualitativeFilesStreaming = (scan.qualitativeFilesStreaming ?? 0) + 1;
|
|
1860
|
+
if (headerPass) {
|
|
1861
|
+
await recordBudgetSkippedCodexHeader(file, headerPass, scan);
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
return selected;
|
|
1866
|
+
}
|
|
1867
|
+
/**
|
|
1868
|
+
* Fresh bounded header reads allowed per scan. Files beyond the cap simply
|
|
1869
|
+
* stay ownership-unknown ("indexing") this run — never claimed either way —
|
|
1870
|
+
* and are reached on a later run once earlier probes persist their results.
|
|
1871
|
+
*/
|
|
1872
|
+
export const codexHeaderProbesPerScan = 64;
|
|
1873
|
+
/**
|
|
1874
|
+
* Classify one budget-skipped Codex file from its persisted or freshly probed
|
|
1875
|
+
* header attribution. Only a "proven" attribution to a project OTHER than the
|
|
1876
|
+
* requested coverage ref counts as foreign; proven-owned and unknown files
|
|
1877
|
+
* keep blocking. Store failures count as index errors, which already force
|
|
1878
|
+
* the fail-closed "indexing" state.
|
|
1879
|
+
*/
|
|
1880
|
+
async function recordBudgetSkippedCodexHeader(file, headerPass, scan) {
|
|
1881
|
+
const pathHash = createHash("sha256").update(resolve(file.filePath)).digest("hex");
|
|
1882
|
+
const fileIdentity = qualitativeFileIdentity(file);
|
|
1883
|
+
let attribution;
|
|
1884
|
+
try {
|
|
1885
|
+
const stored = await headerPass.ownershipIndex.readOwnership("codex", pathHash);
|
|
1886
|
+
if (stored && stored.fileIdentity === fileIdentity) {
|
|
1887
|
+
attribution = stored.headerAttribution;
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
catch {
|
|
1891
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
1892
|
+
return;
|
|
1893
|
+
}
|
|
1894
|
+
if (!attribution) {
|
|
1895
|
+
// Honest carry-over: past the cap the file simply stays unknown this run.
|
|
1896
|
+
if (headerPass.probesRemaining <= 0)
|
|
1897
|
+
return;
|
|
1898
|
+
headerPass.probesRemaining -= 1;
|
|
1899
|
+
attribution = codexHeaderAttribution(await probeCodexRolloutHeader(file.filePath, fileIdentity));
|
|
1900
|
+
try {
|
|
1901
|
+
await headerPass.ownershipIndex.writeOwnership("codex", pathHash, {
|
|
1902
|
+
// Body-derived ownership is still unknown: only the header was read.
|
|
1903
|
+
status: "unknown",
|
|
1904
|
+
fileIdentity,
|
|
1905
|
+
projectRefs: [],
|
|
1906
|
+
headerAttribution: attribution
|
|
1907
|
+
});
|
|
1908
|
+
}
|
|
1909
|
+
catch {
|
|
1910
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
1911
|
+
return;
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
if (attribution.status === "proven" && attribution.projectRef &&
|
|
1915
|
+
headerPass.coverageProjectRef &&
|
|
1916
|
+
attribution.projectRef !== headerPass.coverageProjectRef) {
|
|
1917
|
+
scan.qualitativeFilesForeignProven = (scan.qualitativeFilesForeignProven ?? 0) + 1;
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
const codexStreamChunkBytes = 8 * 1_024 * 1_024;
|
|
1921
|
+
/** A single JSONL line above this bound is skipped as malformed (design e). */
|
|
1922
|
+
const codexStreamMaxLineBytes = 32 * 1_024 * 1_024;
|
|
1923
|
+
const codexStreamPrefixProbeBytes = 64 * 1_024;
|
|
1924
|
+
/**
|
|
1925
|
+
* Default per-run byte allowance for the streaming pass. Sized from measured
|
|
1926
|
+
* end-to-end throughput (~99 MB/s on the reference machine, QA probe8): 512
|
|
1927
|
+
* MiB keeps one slice at roughly 4-5 seconds of wall clock inside the cold
|
|
1928
|
+
* budget. Callers with different budgets tune `maxStreamedBytesPerRun`.
|
|
1929
|
+
*/
|
|
1930
|
+
export const defaultStreamedBytesPerRun = 512 * 1_024 * 1_024;
|
|
1931
|
+
/** Bounded per-run header probes used only to order the streaming schedule. */
|
|
1932
|
+
const codexStreamSchedulingProbesPerScan = 256;
|
|
1933
|
+
/** Documented retention caps for the privacy-reduced checkpoint state. */
|
|
1934
|
+
const checkpointRecentPromptLimit = 12;
|
|
1935
|
+
const checkpointPromptCharLimit = 4_096;
|
|
1936
|
+
const checkpointFileCountLimit = 256;
|
|
1937
|
+
const checkpointWorkdirLimit = 128;
|
|
1938
|
+
/**
|
|
1939
|
+
* Reduce live reducer state to its persistable form: prompts pass through
|
|
1940
|
+
* `sanitizeLocalActivityText` and keep only the last 12 survivors (earlier
|
|
1941
|
+
* survivors persist as a count); file basename tallies and hashed workdir
|
|
1942
|
+
* tallies are capped deterministically; the raw session cwd collapses to its
|
|
1943
|
+
* short-circuit decision plus resolved ref/project. Raw absolute paths and
|
|
1944
|
+
* raw prompt text never reach disk.
|
|
1945
|
+
*/
|
|
1946
|
+
function serializeCodexReducerState(state) {
|
|
1947
|
+
// Path spans are stripped BEFORE sanitization so persisted survivors carry
|
|
1948
|
+
// the same privacy guarantee as entries: raw absolute paths (including
|
|
1949
|
+
// home-anchored ones) never reach disk. Topic derivation is unaffected
|
|
1950
|
+
// (topicTokens strips the same spans); the residual divergence is that
|
|
1951
|
+
// inferAction can no longer see action words embedded inside path segments
|
|
1952
|
+
// of checkpoint-crossing prompts, and path-only prompts stop counting —
|
|
1953
|
+
// pinned by the streaming privacy fixture.
|
|
1954
|
+
const survivors = state.prompts
|
|
1955
|
+
.map((prompt) => sanitizeLocalActivityText(stripAbsolutePathSpans(prompt)))
|
|
1956
|
+
.filter(isHumanPrompt)
|
|
1957
|
+
.map((prompt) => prompt.length > checkpointPromptCharLimit
|
|
1958
|
+
? prompt.slice(0, checkpointPromptCharLimit)
|
|
1959
|
+
: prompt);
|
|
1960
|
+
const recentPrompts = survivors.slice(-checkpointRecentPromptLimit);
|
|
1961
|
+
const promptOverflow = state.restoredPromptOverflow +
|
|
1962
|
+
Math.max(0, survivors.length - checkpointRecentPromptLimit);
|
|
1963
|
+
let rootCwd;
|
|
1964
|
+
if (state.rootCwd !== undefined) {
|
|
1965
|
+
const sessionProject = projectFromCwd(state.rootCwd);
|
|
1966
|
+
const resolved = absoluteWorkingDirectory(state.rootCwd);
|
|
1967
|
+
rootCwd = {
|
|
1968
|
+
present: true,
|
|
1969
|
+
shortCircuits: Boolean(sessionProject && sessionProject !== "(home)"),
|
|
1970
|
+
...(resolved !== undefined
|
|
1971
|
+
? {
|
|
1972
|
+
resolvedRef: derivedWorkingDirectoryRef(resolved),
|
|
1973
|
+
...(projectFromCwd(resolved) !== undefined
|
|
1974
|
+
? { resolvedProject: projectFromCwd(resolved) }
|
|
1975
|
+
: {})
|
|
1976
|
+
}
|
|
1977
|
+
: {})
|
|
1978
|
+
};
|
|
1979
|
+
}
|
|
1980
|
+
else if (state.restoredRootCwd) {
|
|
1981
|
+
rootCwd = state.restoredRootCwd;
|
|
1982
|
+
}
|
|
1983
|
+
else {
|
|
1984
|
+
rootCwd = { present: false, shortCircuits: false };
|
|
1985
|
+
}
|
|
1986
|
+
const workdirs = mergeCodexWorkdirTallies(state)
|
|
1987
|
+
.map(({ rawPath: _rawPath, ...tally }) => tally)
|
|
1988
|
+
.sort((left, right) => right.count - left.count || left.ref.localeCompare(right.ref))
|
|
1989
|
+
.slice(0, checkpointWorkdirLimit);
|
|
1990
|
+
const fileCounts = [...state.fileCounts.entries()]
|
|
1991
|
+
.sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
|
|
1992
|
+
.slice(0, checkpointFileCountLimit);
|
|
1993
|
+
return {
|
|
1994
|
+
...(state.model !== undefined ? { model: state.model } : {}),
|
|
1995
|
+
...(state.sessionId !== undefined ? { sessionId: state.sessionId } : {}),
|
|
1996
|
+
...(state.sourceVersion !== undefined ? { sourceVersion: state.sourceVersion } : {}),
|
|
1997
|
+
rootSessionMetaSeen: state.rootSessionMetaSeen,
|
|
1998
|
+
...(state.startedAt !== undefined ? { startedAt: state.startedAt } : {}),
|
|
1999
|
+
...(state.rootStartedAtMs !== undefined ? { rootStartedAtMs: state.rootStartedAtMs } : {}),
|
|
2000
|
+
rootTaskStarted: state.rootTaskStarted,
|
|
2001
|
+
isSubagent: state.isSubagent,
|
|
2002
|
+
...(state.parentSessionId !== undefined ? { parentSessionId: state.parentSessionId } : {}),
|
|
2003
|
+
...(state.pendingTaskTurnId !== undefined ? { pendingTaskTurnId: state.pendingTaskTurnId } : {}),
|
|
2004
|
+
...(state.completedTask !== undefined ? { completedTask: state.completedTask } : {}),
|
|
2005
|
+
malformedLines: state.malformedLines,
|
|
2006
|
+
...(state.lastActivityAt !== undefined ? { lastActivityAt: state.lastActivityAt } : {}),
|
|
2007
|
+
...(state.lastTotal !== undefined ? { lastTotal: state.lastTotal } : {}),
|
|
2008
|
+
...(state.lastTurn !== undefined ? { lastTurn: state.lastTurn } : {}),
|
|
2009
|
+
...(state.inheritedUsageBaseline !== undefined
|
|
2010
|
+
? { inheritedUsageBaseline: state.inheritedUsageBaseline }
|
|
2011
|
+
: {}),
|
|
2012
|
+
...(state.lastRateLimits !== undefined ? { lastRateLimits: state.lastRateLimits } : {}),
|
|
2013
|
+
rootCwd,
|
|
2014
|
+
workdirs,
|
|
2015
|
+
promptOverflow,
|
|
2016
|
+
recentPrompts,
|
|
2017
|
+
fileCounts,
|
|
2018
|
+
toolCallCount: state.toolCallCount
|
|
2019
|
+
};
|
|
2020
|
+
}
|
|
2021
|
+
/** Fail-closed structural check for a restored reducer state. */
|
|
2022
|
+
function isPersistedCodexReducerState(value) {
|
|
2023
|
+
if (!isRecord(value))
|
|
2024
|
+
return false;
|
|
2025
|
+
const optionalString = (input) => input === undefined || typeof input === "string";
|
|
2026
|
+
const optionalRecord = (input) => input === undefined || isRecord(input) && !Array.isArray(input);
|
|
2027
|
+
const optionalFinite = (input) => input === undefined || typeof input === "number" && Number.isFinite(input);
|
|
2028
|
+
const nonnegativeInt = (input) => Number.isSafeInteger(input) && Number(input) >= 0;
|
|
2029
|
+
const rootCwd = value.rootCwd;
|
|
2030
|
+
const validRootCwd = isRecord(rootCwd) &&
|
|
2031
|
+
typeof rootCwd.present === "boolean" &&
|
|
2032
|
+
typeof rootCwd.shortCircuits === "boolean" &&
|
|
2033
|
+
optionalString(rootCwd.resolvedRef) &&
|
|
2034
|
+
optionalString(rootCwd.resolvedProject);
|
|
2035
|
+
const validWorkdirs = Array.isArray(value.workdirs) && value.workdirs.every((entry) => (isRecord(entry) &&
|
|
2036
|
+
typeof entry.ref === "string" &&
|
|
2037
|
+
Array.isArray(entry.ancestorRefs) &&
|
|
2038
|
+
entry.ancestorRefs.every((ref) => typeof ref === "string") &&
|
|
2039
|
+
nonnegativeInt(entry.depth) &&
|
|
2040
|
+
typeof entry.base === "string" &&
|
|
2041
|
+
typeof entry.isHome === "boolean" &&
|
|
2042
|
+
nonnegativeInt(entry.count)));
|
|
2043
|
+
const validCompletion = value.completedTask === undefined || (isRecord(value.completedTask) &&
|
|
2044
|
+
value.completedTask.status === "completed" &&
|
|
2045
|
+
value.completedTask.evidence === "codex_task_complete" &&
|
|
2046
|
+
typeof value.completedTask.observedAt === "string");
|
|
2047
|
+
const validRateLimits = value.lastRateLimits === undefined || (isRecord(value.lastRateLimits) &&
|
|
2048
|
+
typeof value.lastRateLimits.observedAt === "string" &&
|
|
2049
|
+
Array.isArray(value.lastRateLimits.windows));
|
|
2050
|
+
return typeof value.rootSessionMetaSeen === "boolean" &&
|
|
2051
|
+
typeof value.rootTaskStarted === "boolean" &&
|
|
2052
|
+
typeof value.isSubagent === "boolean" &&
|
|
2053
|
+
optionalString(value.model) &&
|
|
2054
|
+
optionalString(value.sessionId) &&
|
|
2055
|
+
optionalString(value.sourceVersion) &&
|
|
2056
|
+
optionalString(value.startedAt) &&
|
|
2057
|
+
optionalFinite(value.rootStartedAtMs) &&
|
|
2058
|
+
optionalString(value.parentSessionId) &&
|
|
2059
|
+
optionalString(value.pendingTaskTurnId) &&
|
|
2060
|
+
validCompletion &&
|
|
2061
|
+
nonnegativeInt(value.malformedLines) &&
|
|
2062
|
+
optionalString(value.lastActivityAt) &&
|
|
2063
|
+
optionalRecord(value.lastTotal) &&
|
|
2064
|
+
optionalRecord(value.lastTurn) &&
|
|
2065
|
+
optionalRecord(value.inheritedUsageBaseline) &&
|
|
2066
|
+
validRateLimits &&
|
|
2067
|
+
validRootCwd &&
|
|
2068
|
+
validWorkdirs &&
|
|
2069
|
+
nonnegativeInt(value.promptOverflow) &&
|
|
2070
|
+
Array.isArray(value.recentPrompts) &&
|
|
2071
|
+
value.recentPrompts.every((prompt) => typeof prompt === "string") &&
|
|
2072
|
+
Array.isArray(value.fileCounts) &&
|
|
2073
|
+
value.fileCounts.every((pair) => (Array.isArray(pair) && pair.length === 2 &&
|
|
2074
|
+
typeof pair[0] === "string" && nonnegativeInt(pair[1]))) &&
|
|
2075
|
+
nonnegativeInt(value.toolCallCount);
|
|
2076
|
+
}
|
|
2077
|
+
function restoreCodexReducerState(persisted) {
|
|
2078
|
+
const state = createCodexRolloutParserState();
|
|
2079
|
+
state.model = persisted.model;
|
|
2080
|
+
state.sessionId = persisted.sessionId;
|
|
2081
|
+
state.sourceVersion = persisted.sourceVersion;
|
|
2082
|
+
state.rootSessionMetaSeen = persisted.rootSessionMetaSeen;
|
|
2083
|
+
state.startedAt = persisted.startedAt;
|
|
2084
|
+
state.rootStartedAtMs = persisted.rootStartedAtMs;
|
|
2085
|
+
state.rootTaskStarted = persisted.rootTaskStarted;
|
|
2086
|
+
state.isSubagent = persisted.isSubagent;
|
|
2087
|
+
state.parentSessionId = persisted.parentSessionId;
|
|
2088
|
+
state.pendingTaskTurnId = persisted.pendingTaskTurnId;
|
|
2089
|
+
state.completedTask = persisted.completedTask;
|
|
2090
|
+
state.malformedLines = persisted.malformedLines;
|
|
2091
|
+
state.lastActivityAt = persisted.lastActivityAt;
|
|
2092
|
+
state.lastTotal = persisted.lastTotal;
|
|
2093
|
+
state.lastTurn = persisted.lastTurn;
|
|
2094
|
+
state.inheritedUsageBaseline = persisted.inheritedUsageBaseline;
|
|
2095
|
+
state.lastRateLimits = persisted.lastRateLimits;
|
|
2096
|
+
state.restoredRootCwd = persisted.rootCwd;
|
|
2097
|
+
state.restoredWorkdirs = persisted.workdirs;
|
|
2098
|
+
state.prompts = [...persisted.recentPrompts];
|
|
2099
|
+
state.restoredPromptOverflow = persisted.promptOverflow;
|
|
2100
|
+
state.fileCounts = new Map(persisted.fileCounts);
|
|
2101
|
+
state.toolCallCount = persisted.toolCallCount;
|
|
2102
|
+
return state;
|
|
2103
|
+
}
|
|
2104
|
+
/**
|
|
2105
|
+
* One run's bounded streaming pass over oversized Codex rollouts. Candidates
|
|
2106
|
+
* are ordered mainline-first (checkpoint or bounded header probe supplies the
|
|
2107
|
+
* subagent marker) so the 326-subagent reverse-scan pathology can never
|
|
2108
|
+
* starve mainline files, then newest-first within each class. Returns the
|
|
2109
|
+
* evidence of every file whose stream completed this run; everything else
|
|
2110
|
+
* advanced by at least its checkpoint and stays honestly unconverged.
|
|
2111
|
+
*/
|
|
2112
|
+
async function runCodexStreamingPass(files, context) {
|
|
2113
|
+
const scan = context.scan;
|
|
2114
|
+
const candidates = [];
|
|
2115
|
+
let schedulingProbes = codexStreamSchedulingProbesPerScan;
|
|
2116
|
+
for (const file of files) {
|
|
2117
|
+
const pathHash = createHash("sha256").update(resolve(file.filePath)).digest("hex");
|
|
2118
|
+
let checkpoint;
|
|
2119
|
+
try {
|
|
2120
|
+
checkpoint = await context.streamCheckpoints.readStreamCheckpoint("codex", pathHash);
|
|
2121
|
+
}
|
|
2122
|
+
catch {
|
|
2123
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
2124
|
+
}
|
|
2125
|
+
let isSubagent;
|
|
2126
|
+
if (checkpoint && isPersistedCodexReducerState(checkpoint.reducerState) &&
|
|
2127
|
+
checkpoint.reducerState.rootSessionMetaSeen) {
|
|
2128
|
+
isSubagent = checkpoint.reducerState.isSubagent;
|
|
2129
|
+
}
|
|
2130
|
+
else {
|
|
2131
|
+
if (context.ownershipIndex) {
|
|
2132
|
+
try {
|
|
2133
|
+
const stored = await context.ownershipIndex.readOwnership("codex", pathHash);
|
|
2134
|
+
if (stored && stored.fileIdentity === qualitativeFileIdentity(file)) {
|
|
2135
|
+
isSubagent = stored.headerAttribution?.isSubagent;
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
catch {
|
|
2139
|
+
// Scheduling is a hint; the ledger's own ownership reads count
|
|
2140
|
+
// store failures and force the fail-closed coverage state.
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
if (isSubagent === undefined && schedulingProbes > 0) {
|
|
2144
|
+
schedulingProbes -= 1;
|
|
2145
|
+
const header = await probeCodexRolloutHeader(file.filePath, qualitativeFileIdentity(file));
|
|
2146
|
+
isSubagent = header?.isSubagent;
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
candidates.push({
|
|
2150
|
+
file,
|
|
2151
|
+
pathHash,
|
|
2152
|
+
...(checkpoint ? { checkpoint } : {}),
|
|
2153
|
+
...(isSubagent !== undefined ? { isSubagent } : {})
|
|
2154
|
+
});
|
|
2155
|
+
}
|
|
2156
|
+
const classOf = (candidate) => candidate.isSubagent === false ? 0 : candidate.isSubagent === undefined ? 1 : 2;
|
|
2157
|
+
candidates.sort((left, right) => (classOf(left) - classOf(right) ||
|
|
2158
|
+
right.file.recencyMs - left.file.recencyMs ||
|
|
2159
|
+
left.file.filePath.localeCompare(right.file.filePath)));
|
|
2160
|
+
const completed = new Map();
|
|
2161
|
+
const budget = { remainingBytes: context.remainingBytes };
|
|
2162
|
+
for (const candidate of candidates) {
|
|
2163
|
+
// The policy validator guarantees a positive allowance, so the first
|
|
2164
|
+
// scheduled candidate always streams; later candidates run only while
|
|
2165
|
+
// allowance remains (mainline-first order makes this starvation-safe).
|
|
2166
|
+
if (budget.remainingBytes <= 0)
|
|
2167
|
+
break;
|
|
2168
|
+
const value = await streamCodexRolloutSlice(candidate, context, budget);
|
|
2169
|
+
if (value)
|
|
2170
|
+
completed.set(candidate.file.filePath, value);
|
|
2171
|
+
}
|
|
2172
|
+
return completed;
|
|
2173
|
+
}
|
|
2174
|
+
/**
|
|
2175
|
+
* Advance one oversized rollout by a bounded slice; complete it when EOF is
|
|
2176
|
+
* reached with a stable identity. Every failure path degrades to "no claim
|
|
2177
|
+
* this run": a broken resume proof restarts from byte zero, a mid-stream
|
|
2178
|
+
* append keeps the checkpoint and withholds the entry (design edge 4), and
|
|
2179
|
+
* store errors surface as index errors that force the fail-closed
|
|
2180
|
+
* "indexing" coverage state.
|
|
2181
|
+
*/
|
|
2182
|
+
async function streamCodexRolloutSlice(candidate, context, budget) {
|
|
2183
|
+
const scan = context.scan;
|
|
2184
|
+
const selectionIdentity = qualitativeFileIdentity(candidate.file);
|
|
2185
|
+
const runSinceIso = typeof context.sinceMs === "number"
|
|
2186
|
+
? new Date(context.sinceMs).toISOString()
|
|
2187
|
+
: null;
|
|
2188
|
+
let handle;
|
|
2189
|
+
try {
|
|
2190
|
+
handle = await open(candidate.file.filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
2191
|
+
const before = await handle.stat();
|
|
2192
|
+
if (!before.isFile() ||
|
|
2193
|
+
before.dev !== candidate.file.dev ||
|
|
2194
|
+
before.ino !== candidate.file.ino ||
|
|
2195
|
+
before.birthtimeMs !== candidate.file.birthtimeMs) {
|
|
2196
|
+
return undefined;
|
|
2197
|
+
}
|
|
2198
|
+
// Resume proof: pinned identity, monotonic size, matching prefix bytes,
|
|
2199
|
+
// matching contract, and structurally valid restored state. Anything
|
|
2200
|
+
// less discards the checkpoint and restarts from byte zero.
|
|
2201
|
+
let state = createCodexRolloutParserState();
|
|
2202
|
+
let collector = context.collectInvocationEvidence
|
|
2203
|
+
? createCodexInvocationCollector(context.sinceMs)
|
|
2204
|
+
: undefined;
|
|
2205
|
+
let offset = 0;
|
|
2206
|
+
let entrySinceIso = context.collectInvocationEvidence ? runSinceIso : null;
|
|
2207
|
+
const checkpoint = candidate.checkpoint;
|
|
2208
|
+
if (checkpoint) {
|
|
2209
|
+
let resumable = checkpoint.parserVersion === localAgentQualitativeParserVersion &&
|
|
2210
|
+
checkpoint.collectInvocationEvidence === context.collectInvocationEvidence &&
|
|
2211
|
+
checkpoint.pin.dev === candidate.file.dev &&
|
|
2212
|
+
checkpoint.pin.ino === candidate.file.ino &&
|
|
2213
|
+
checkpoint.pin.birthtimeMs === candidate.file.birthtimeMs &&
|
|
2214
|
+
checkpoint.offset <= candidate.file.size &&
|
|
2215
|
+
checkpoint.prefixProbe.bytes === Math.min(codexStreamPrefixProbeBytes, checkpoint.offset) &&
|
|
2216
|
+
isPersistedCodexReducerState(checkpoint.reducerState) &&
|
|
2217
|
+
(!context.collectInvocationEvidence ||
|
|
2218
|
+
isCodexInvocationCollectorSnapshot(checkpoint.collectorState));
|
|
2219
|
+
if (resumable && checkpoint.offset > 0) {
|
|
2220
|
+
const probeBytes = Math.min(codexStreamPrefixProbeBytes, checkpoint.offset);
|
|
2221
|
+
const probe = Buffer.allocUnsafe(probeBytes);
|
|
2222
|
+
let probeRead = 0;
|
|
2223
|
+
while (probeRead < probeBytes) {
|
|
2224
|
+
const read = await handle.read(probe, probeRead, probeBytes - probeRead, checkpoint.offset - probeBytes + probeRead);
|
|
2225
|
+
if (read.bytesRead === 0)
|
|
2226
|
+
break;
|
|
2227
|
+
probeRead += read.bytesRead;
|
|
2228
|
+
}
|
|
2229
|
+
resumable = probeRead === probeBytes &&
|
|
2230
|
+
createHash("sha256").update(probe).digest("hex") === checkpoint.prefixProbe.sha256;
|
|
2231
|
+
}
|
|
2232
|
+
if (resumable) {
|
|
2233
|
+
state = restoreCodexReducerState(checkpoint.reducerState);
|
|
2234
|
+
if (context.collectInvocationEvidence) {
|
|
2235
|
+
const pinnedSinceMs = checkpoint.sinceIso === null
|
|
2236
|
+
? undefined
|
|
2237
|
+
: Date.parse(checkpoint.sinceIso);
|
|
2238
|
+
collector = createCodexInvocationCollector(pinnedSinceMs, checkpoint.collectorState);
|
|
2239
|
+
entrySinceIso = checkpoint.sinceIso;
|
|
2240
|
+
}
|
|
2241
|
+
offset = checkpoint.offset;
|
|
2242
|
+
}
|
|
2243
|
+
else {
|
|
2244
|
+
await context.streamCheckpoints.deleteStreamCheckpoint("codex", candidate.pathHash)
|
|
2245
|
+
.catch(() => undefined);
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
// Bounded chunked line consumption from the resume offset.
|
|
2249
|
+
const collectorConsume = collector?.consume;
|
|
2250
|
+
let consumed = offset;
|
|
2251
|
+
let position = offset;
|
|
2252
|
+
let carry = Buffer.alloc(0);
|
|
2253
|
+
let skippingOversizedLine = false;
|
|
2254
|
+
let linesConsumed = 0;
|
|
2255
|
+
let sawEof = false;
|
|
2256
|
+
while (true) {
|
|
2257
|
+
// A candidate that was scheduled always consumes at least one complete
|
|
2258
|
+
// line (or reaches EOF/an oversized-line skip) before yielding to the
|
|
2259
|
+
// byte allowance — this is what guarantees convergence across runs.
|
|
2260
|
+
if (budget.remainingBytes <= 0 && linesConsumed > 0)
|
|
2261
|
+
break;
|
|
2262
|
+
const readLength = budget.remainingBytes > 0
|
|
2263
|
+
? Math.min(codexStreamChunkBytes, budget.remainingBytes)
|
|
2264
|
+
: codexStreamChunkBytes;
|
|
2265
|
+
const chunk = Buffer.allocUnsafe(readLength);
|
|
2266
|
+
const read = await handle.read(chunk, 0, readLength, position);
|
|
2267
|
+
if (read.bytesRead === 0) {
|
|
2268
|
+
sawEof = true;
|
|
2269
|
+
break;
|
|
2270
|
+
}
|
|
2271
|
+
budget.remainingBytes -= read.bytesRead;
|
|
2272
|
+
scan.qualitativeBytesStreamed = (scan.qualitativeBytesStreamed ?? 0) + read.bytesRead;
|
|
2273
|
+
const dataStart = position - carry.length;
|
|
2274
|
+
position += read.bytesRead;
|
|
2275
|
+
const data = carry.length > 0
|
|
2276
|
+
? Buffer.concat([carry, chunk.subarray(0, read.bytesRead)])
|
|
2277
|
+
: chunk.subarray(0, read.bytesRead);
|
|
2278
|
+
let lineStart = 0;
|
|
2279
|
+
while (true) {
|
|
2280
|
+
const newlineAt = data.indexOf(0x0a, lineStart);
|
|
2281
|
+
if (newlineAt === -1)
|
|
2282
|
+
break;
|
|
2283
|
+
if (skippingOversizedLine) {
|
|
2284
|
+
// The oversized line just terminated; count it once, exactly when
|
|
2285
|
+
// its bytes are durably consumed past the checkpoint offset.
|
|
2286
|
+
skippingOversizedLine = false;
|
|
2287
|
+
state.malformedLines += 1;
|
|
2288
|
+
}
|
|
2289
|
+
else if (newlineAt - lineStart > codexStreamMaxLineBytes) {
|
|
2290
|
+
// Exact bound: chunk alignment can let a just-over-cap line
|
|
2291
|
+
// accumulate fully before its newline arrives; it is still skipped
|
|
2292
|
+
// as malformed, matching the documented >32 MiB rule.
|
|
2293
|
+
state.malformedLines += 1;
|
|
2294
|
+
}
|
|
2295
|
+
else {
|
|
2296
|
+
consumeCodexRolloutLine(state, data.subarray(lineStart, newlineAt).toString("utf8"), collectorConsume);
|
|
2297
|
+
}
|
|
2298
|
+
lineStart = newlineAt + 1;
|
|
2299
|
+
linesConsumed += 1;
|
|
2300
|
+
consumed = dataStart + lineStart;
|
|
2301
|
+
}
|
|
2302
|
+
if (skippingOversizedLine) {
|
|
2303
|
+
carry = Buffer.alloc(0);
|
|
2304
|
+
}
|
|
2305
|
+
else {
|
|
2306
|
+
carry = Buffer.from(data.subarray(lineStart));
|
|
2307
|
+
if (carry.length > codexStreamMaxLineBytes) {
|
|
2308
|
+
skippingOversizedLine = true;
|
|
2309
|
+
carry = Buffer.alloc(0);
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
// A crashed or kill-9'd session leaves a final line with no trailing
|
|
2314
|
+
// newline, and that file never changes again. When the post-slice
|
|
2315
|
+
// identity still equals the selection identity, the tail is provably not
|
|
2316
|
+
// an in-flight append: consume it as the final complete line, exactly as
|
|
2317
|
+
// the whole-file parser's split("\\n") treats the same bytes. A torn tail
|
|
2318
|
+
// degrades to the malformed_jsonl diagnostic both paths share.
|
|
2319
|
+
if (sawEof && !skippingOversizedLine && carry.length > 0) {
|
|
2320
|
+
const settled = await handle.stat();
|
|
2321
|
+
if (settled.isFile() && settled.size === consumed + carry.length &&
|
|
2322
|
+
qualitativeFileIdentity(settled) === selectionIdentity) {
|
|
2323
|
+
if (carry.length > codexStreamMaxLineBytes) {
|
|
2324
|
+
state.malformedLines += 1;
|
|
2325
|
+
}
|
|
2326
|
+
else {
|
|
2327
|
+
consumeCodexRolloutLine(state, carry.toString("utf8"), collectorConsume);
|
|
2328
|
+
}
|
|
2329
|
+
consumed += carry.length;
|
|
2330
|
+
carry = Buffer.alloc(0);
|
|
2331
|
+
}
|
|
2332
|
+
}
|
|
2333
|
+
if (sawEof && !skippingOversizedLine && carry.length === 0 && consumed > 0) {
|
|
2334
|
+
const after = await handle.stat();
|
|
2335
|
+
if (after.isFile() && after.size === consumed &&
|
|
2336
|
+
qualitativeFileIdentity(after) === selectionIdentity) {
|
|
2337
|
+
return await completeStreamedRollout(candidate, context, state, collector, consumed, entrySinceIso, runSinceIso);
|
|
2338
|
+
}
|
|
2339
|
+
// The rollout changed while streaming (an active append): keep the
|
|
2340
|
+
// checkpoint at the last complete line and withhold the entry.
|
|
2341
|
+
}
|
|
2342
|
+
if (consumed > offset || (consumed > 0 && !candidate.checkpoint)) {
|
|
2343
|
+
await writeStreamCheckpointAt(candidate, context, handle, state, collector, consumed, entrySinceIso);
|
|
2344
|
+
}
|
|
2345
|
+
return undefined;
|
|
2346
|
+
}
|
|
2347
|
+
catch {
|
|
2348
|
+
// Unreadable mid-stream: no claim this run; selection re-treats the file
|
|
2349
|
+
// as skipped and the ledger keeps it blocking.
|
|
2350
|
+
return undefined;
|
|
2351
|
+
}
|
|
2352
|
+
finally {
|
|
2353
|
+
await handle?.close().catch(() => undefined);
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
async function writeStreamCheckpointAt(candidate, context, handle, state, collector, consumed, entrySinceIso) {
|
|
2357
|
+
const scan = context.scan;
|
|
2358
|
+
const probeBytes = Math.min(codexStreamPrefixProbeBytes, consumed);
|
|
2359
|
+
const probe = Buffer.allocUnsafe(probeBytes);
|
|
2360
|
+
let probeRead = 0;
|
|
2361
|
+
while (probeRead < probeBytes) {
|
|
2362
|
+
const read = await handle.read(probe, probeRead, probeBytes - probeRead, consumed - probeBytes + probeRead);
|
|
2363
|
+
if (read.bytesRead === 0)
|
|
2364
|
+
break;
|
|
2365
|
+
probeRead += read.bytesRead;
|
|
2366
|
+
}
|
|
2367
|
+
if (probeRead !== probeBytes) {
|
|
2368
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
2369
|
+
return;
|
|
2370
|
+
}
|
|
2371
|
+
try {
|
|
2372
|
+
await context.streamCheckpoints.writeStreamCheckpoint("codex", candidate.pathHash, {
|
|
2373
|
+
pin: {
|
|
2374
|
+
dev: candidate.file.dev,
|
|
2375
|
+
ino: candidate.file.ino,
|
|
2376
|
+
birthtimeMs: candidate.file.birthtimeMs
|
|
2377
|
+
},
|
|
2378
|
+
parserVersion: localAgentQualitativeParserVersion,
|
|
2379
|
+
collectInvocationEvidence: context.collectInvocationEvidence,
|
|
2380
|
+
sinceIso: entrySinceIso,
|
|
2381
|
+
offset: consumed,
|
|
2382
|
+
prefixProbe: {
|
|
2383
|
+
bytes: probeBytes,
|
|
2384
|
+
sha256: createHash("sha256").update(probe).digest("hex")
|
|
2385
|
+
},
|
|
2386
|
+
reducerState: serializeCodexReducerState(state),
|
|
2387
|
+
...(collector ? { collectorState: collector.snapshot() } : {})
|
|
2388
|
+
});
|
|
2389
|
+
}
|
|
2390
|
+
catch {
|
|
2391
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
async function completeStreamedRollout(candidate, context, state, collector, consumed, entrySinceIso, runSinceIso) {
|
|
2395
|
+
const scan = context.scan;
|
|
2396
|
+
const fileDiagnostics = [];
|
|
2397
|
+
const calls = finishCodexRolloutParse(state, (diagnostic) => fileDiagnostics.push(diagnostic));
|
|
2398
|
+
const invocationFile = collector?.finish();
|
|
2399
|
+
const invocationWindowProof = collector?.windowProof();
|
|
2400
|
+
const stored = {
|
|
2401
|
+
calls,
|
|
2402
|
+
...(invocationFile ? { invocationFile } : {}),
|
|
2403
|
+
...(collector ? { invocationWindowProof } : {}),
|
|
2404
|
+
diagnostics: fileDiagnostics
|
|
2405
|
+
};
|
|
2406
|
+
assertFormatCallOwnership(context.descriptor, stored.calls);
|
|
2407
|
+
assertInvocationOwnership(context.descriptor, stored.invocationFile, context.collectInvocationEvidence);
|
|
2408
|
+
const entryKey = {
|
|
2409
|
+
schemaVersion: 1,
|
|
2410
|
+
parserVersion: localAgentQualitativeParserVersion,
|
|
2411
|
+
agent: "codex",
|
|
2412
|
+
pathHash: candidate.pathHash,
|
|
2413
|
+
fileIdentity: qualitativeFileIdentity(candidate.file),
|
|
2414
|
+
sinceIso: entrySinceIso,
|
|
2415
|
+
collectInvocationEvidence: context.collectInvocationEvidence
|
|
2416
|
+
};
|
|
2417
|
+
let entryPersisted = false;
|
|
2418
|
+
try {
|
|
2419
|
+
await context.qualitativeIndex.write(entryKey, stored);
|
|
2420
|
+
entryPersisted = true;
|
|
2421
|
+
}
|
|
2422
|
+
catch {
|
|
2423
|
+
scan.qualitativeIndexErrors = (scan.qualitativeIndexErrors ?? 0) + 1;
|
|
2424
|
+
}
|
|
2425
|
+
if (entryPersisted) {
|
|
2426
|
+
// Only a durably indexed file may drop its resumable state.
|
|
2427
|
+
await context.streamCheckpoints.deleteStreamCheckpoint("codex", candidate.pathHash)
|
|
2428
|
+
.catch(() => undefined);
|
|
2429
|
+
}
|
|
2430
|
+
void consumed;
|
|
2431
|
+
// The stored entry carries the pinned collector window. This run may have
|
|
2432
|
+
// requested a newer window; aggregated invocation evidence is used in-run
|
|
2433
|
+
// only when it narrows exactly, otherwise this run honestly reports
|
|
2434
|
+
// partial coverage while the calls (window-blind, filtered by timestamp
|
|
2435
|
+
// downstream) remain exact.
|
|
2436
|
+
if (stored.invocationFile && entrySinceIso !== runSinceIso &&
|
|
2437
|
+
!streamedInvocationNarrowsExactly(entrySinceIso, runSinceIso, stored)) {
|
|
2438
|
+
scan.qualitativeCoverage = "partial";
|
|
2439
|
+
const { invocationFile: _omitted, invocationWindowProof: _proof, ...rest } = stored;
|
|
2440
|
+
return { ...rest };
|
|
2441
|
+
}
|
|
2442
|
+
return stored;
|
|
2443
|
+
}
|
|
2444
|
+
/**
|
|
2445
|
+
* In-run mirror of the store's `invocationWindowCanBeNarrowedExactly` for a
|
|
2446
|
+
* freshly completed stream whose collector window predates this run's
|
|
2447
|
+
* request (the loader cannot import the store module).
|
|
2448
|
+
*/
|
|
2449
|
+
function streamedInvocationNarrowsExactly(pinnedSinceIso, runSinceIso, value) {
|
|
2450
|
+
if (pinnedSinceIso === runSinceIso)
|
|
2451
|
+
return true;
|
|
2452
|
+
if (runSinceIso === null)
|
|
2453
|
+
return false;
|
|
2454
|
+
if (pinnedSinceIso !== null && Date.parse(pinnedSinceIso) > Date.parse(runSinceIso)) {
|
|
2455
|
+
return false;
|
|
2456
|
+
}
|
|
2457
|
+
const invocation = value.invocationFile;
|
|
2458
|
+
if (!invocation)
|
|
2459
|
+
return false;
|
|
2460
|
+
const proof = value.invocationWindowProof;
|
|
2461
|
+
if (!proof || !proof.allCountedEventsTimestamped)
|
|
2462
|
+
return false;
|
|
2463
|
+
const hasCountedEvidence = invocation.assistantTurns > 0 ||
|
|
2464
|
+
invocation.contextSignal.compactionEvents > 0 ||
|
|
2465
|
+
invocation.invocations.length > 0 ||
|
|
2466
|
+
invocation.invokedMcpTools.length > 0 ||
|
|
2467
|
+
invocation.invokedSkills.length > 0 ||
|
|
2468
|
+
invocation.invokedSubagents.length > 0 ||
|
|
2469
|
+
invocation.invokedCommands.length > 0 ||
|
|
2470
|
+
invocation.contextSignal.fileReads.length > 0 ||
|
|
2471
|
+
invocation.contextSignal.repeatedFileReads.length > 0;
|
|
2472
|
+
if (!hasCountedEvidence)
|
|
2473
|
+
return proof.earliestCountedAt === undefined;
|
|
2474
|
+
if (proof.earliestCountedAt === undefined)
|
|
2475
|
+
return false;
|
|
2476
|
+
return Date.parse(proof.earliestCountedAt) >= Date.parse(runSinceIso);
|
|
2477
|
+
}
|
|
2478
|
+
function qualitativeIndexKey(agent, filePath, metadata, sinceMs, collectInvocationEvidence) {
|
|
2479
|
+
return {
|
|
2480
|
+
schemaVersion: 1,
|
|
2481
|
+
parserVersion: localAgentQualitativeParserVersion,
|
|
2482
|
+
agent,
|
|
2483
|
+
pathHash: createHash("sha256").update(resolve(filePath)).digest("hex"),
|
|
2484
|
+
fileIdentity: qualitativeFileIdentity(metadata),
|
|
2485
|
+
sinceIso: typeof sinceMs === "number" ? new Date(sinceMs).toISOString() : null,
|
|
2486
|
+
collectInvocationEvidence
|
|
2487
|
+
};
|
|
2488
|
+
}
|
|
2489
|
+
function qualitativeFileIdentity(metadata) {
|
|
2490
|
+
return [
|
|
2491
|
+
metadata.dev,
|
|
2492
|
+
metadata.ino,
|
|
2493
|
+
metadata.size,
|
|
2494
|
+
metadata.mtimeMs,
|
|
2495
|
+
metadata.ctimeMs,
|
|
2496
|
+
metadata.birthtimeMs
|
|
2497
|
+
].join(":");
|
|
2498
|
+
}
|
|
2499
|
+
async function qualitativeIndexKeyStillCurrent(filePath, key) {
|
|
2500
|
+
try {
|
|
2501
|
+
const current = await lstat(filePath);
|
|
2502
|
+
return current.isFile() && !current.isSymbolicLink() &&
|
|
2503
|
+
qualitativeFileIdentity(current) === key.fileIdentity;
|
|
2504
|
+
}
|
|
2505
|
+
catch {
|
|
2506
|
+
return false;
|
|
2507
|
+
}
|
|
2508
|
+
}
|
|
2509
|
+
function isQualitativeIndexValue(value, descriptor, collectInvocationEvidence) {
|
|
2510
|
+
if (!isRecord(value) || !Array.isArray(value.calls) ||
|
|
2511
|
+
!Array.isArray(value.diagnostics)) {
|
|
2512
|
+
return false;
|
|
2513
|
+
}
|
|
2514
|
+
const calls = value.calls;
|
|
2515
|
+
if (!calls.every((call) => isIndexedLocalAgentCall(call, descriptor.id))) {
|
|
2516
|
+
return false;
|
|
2517
|
+
}
|
|
2518
|
+
const diagnostics = value.diagnostics;
|
|
2519
|
+
if (!diagnostics.every((entry) => (isRecord(entry) &&
|
|
2520
|
+
(entry.code === "malformed_jsonl" ||
|
|
2521
|
+
entry.code === "malformed_session_file" ||
|
|
2522
|
+
entry.code === "unsupported_token_shape") &&
|
|
2523
|
+
Number.isSafeInteger(entry.count) && Number(entry.count) > 0))) {
|
|
2524
|
+
return false;
|
|
2525
|
+
}
|
|
2526
|
+
const invocationFile = value.invocationFile;
|
|
2527
|
+
try {
|
|
2528
|
+
assertInvocationOwnership(descriptor, invocationFile, collectInvocationEvidence);
|
|
2529
|
+
}
|
|
2530
|
+
catch {
|
|
2531
|
+
return false;
|
|
2532
|
+
}
|
|
2533
|
+
return true;
|
|
2534
|
+
}
|
|
2535
|
+
function isIndexedLocalAgentCall(value, agent) {
|
|
2536
|
+
if (!isRecord(value) || value.agent !== agent ||
|
|
2537
|
+
typeof value.model !== "string" || value.model.length === 0 ||
|
|
2538
|
+
typeof value.timestamp !== "string" || !Number.isFinite(Date.parse(value.timestamp))) {
|
|
2539
|
+
return false;
|
|
2540
|
+
}
|
|
2541
|
+
const usage = value.usage;
|
|
2542
|
+
if (!isRecord(usage))
|
|
2543
|
+
return false;
|
|
2544
|
+
const tokenFields = [
|
|
2545
|
+
"inputTokens",
|
|
2546
|
+
"outputTokens",
|
|
2547
|
+
"cacheReadTokens",
|
|
2548
|
+
"cacheWrite5mTokens",
|
|
2549
|
+
"cacheWrite1hTokens",
|
|
2550
|
+
"thoughtTokens",
|
|
2551
|
+
"toolTokens"
|
|
2552
|
+
];
|
|
2553
|
+
return tokenFields.every((field) => {
|
|
2554
|
+
const tokenValue = usage[field];
|
|
2555
|
+
if (field === "inputTokens" || field === "outputTokens") {
|
|
2556
|
+
return Number.isSafeInteger(tokenValue) && Number(tokenValue) >= 0;
|
|
2557
|
+
}
|
|
2558
|
+
return tokenValue === undefined ||
|
|
2559
|
+
Number.isSafeInteger(tokenValue) && Number(tokenValue) >= 0;
|
|
2560
|
+
});
|
|
2561
|
+
}
|
|
2562
|
+
async function readBoundedUtf8File(filePath, maxBytes, expectedIdentity) {
|
|
2563
|
+
// Open the selected inode itself without following a last-moment link. The
|
|
2564
|
+
// metadata snapshot used for selection is then checked on the descriptor,
|
|
2565
|
+
// not on the path, both before and after reading. This prevents a path swap
|
|
2566
|
+
// from turning an out-of-root or actively rewritten file into "complete"
|
|
2567
|
+
// qualitative evidence.
|
|
2568
|
+
const handle = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
2569
|
+
const chunks = [];
|
|
2570
|
+
let bytesRead = 0;
|
|
2571
|
+
try {
|
|
2572
|
+
const before = await handle.stat();
|
|
2573
|
+
if (!before.isFile() || qualitativeFileIdentity(before) !== expectedIdentity) {
|
|
2574
|
+
throw newErrorWithCode("ESTALE");
|
|
2575
|
+
}
|
|
2576
|
+
const chunk = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes + 1));
|
|
2577
|
+
while (true) {
|
|
2578
|
+
const remainingWithSentinel = maxBytes - bytesRead + 1;
|
|
2579
|
+
const length = Math.min(chunk.length, remainingWithSentinel);
|
|
2580
|
+
const read = await handle.read(chunk, 0, length, null);
|
|
2581
|
+
if (read.bytesRead === 0)
|
|
2582
|
+
break;
|
|
2583
|
+
bytesRead += read.bytesRead;
|
|
2584
|
+
if (bytesRead > maxBytes) {
|
|
2585
|
+
throw new QualitativeReadLimitError();
|
|
2586
|
+
}
|
|
2587
|
+
chunks.push(Buffer.from(chunk.subarray(0, read.bytesRead)));
|
|
2588
|
+
}
|
|
2589
|
+
const after = await handle.stat();
|
|
2590
|
+
if (!after.isFile() || qualitativeFileIdentity(after) !== expectedIdentity) {
|
|
2591
|
+
throw newErrorWithCode("ESTALE");
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
finally {
|
|
2595
|
+
await handle.close();
|
|
2596
|
+
}
|
|
2597
|
+
return {
|
|
2598
|
+
content: Buffer.concat(chunks, bytesRead).toString("utf8"),
|
|
2599
|
+
bytesRead
|
|
2600
|
+
};
|
|
2601
|
+
}
|
|
2602
|
+
function recordQualitativeBudgetSkip(scan) {
|
|
2603
|
+
scan.qualitativeCoverage = "partial";
|
|
2604
|
+
scan.qualitativeFilesSkippedForBudget =
|
|
2605
|
+
(scan.qualitativeFilesSkippedForBudget ?? 0) + 1;
|
|
2606
|
+
}
|
|
2607
|
+
function finishQualitativeCoverage(scan, diagnostics) {
|
|
2608
|
+
if (scan.directoryStatus === "unreadable" || scan.unreadableFiles > 0) {
|
|
2609
|
+
scan.qualitativeCoverage = "partial";
|
|
2610
|
+
}
|
|
2611
|
+
// Per-project ledger (BLOCKER-1 semantics, fail closed). An unparsed
|
|
2612
|
+
// eligible file blocks unless the bounded header pass proved it belongs to
|
|
2613
|
+
// a project other than the requested one; a file proven to belong to the
|
|
2614
|
+
// requested project itself keeps blocking until indexed (design section d:
|
|
2615
|
+
// it is the oversized relevant transcript, never an exclusion).
|
|
2616
|
+
const eligible = scan.qualitativeFilesEligible ?? 0;
|
|
2617
|
+
const indexed = Math.min(eligible, scan.qualitativeFilesReadCompletely ?? 0);
|
|
2618
|
+
scan.qualitativeFilesIndexed = indexed;
|
|
2619
|
+
const foreignProven = scan.qualitativeFilesForeignProven ?? 0;
|
|
2620
|
+
scan.qualitativeFilesOwnershipUnknown = Math.max(0, eligible - indexed - foreignProven);
|
|
2621
|
+
// A missing directory is an absent agent, not a failure; only unreadable
|
|
2622
|
+
// state, unreadable files, or index errors force the fail-closed state.
|
|
2623
|
+
const scanFailure = scan.directoryStatus === "unreadable" || scan.unreadableFiles > 0 ||
|
|
2624
|
+
(scan.qualitativeIndexErrors ?? 0) > 0;
|
|
2625
|
+
scan.qualitativeProjectCoverage =
|
|
2626
|
+
!scanFailure && scan.qualitativeFilesOwnershipUnknown === 0
|
|
2627
|
+
? "complete"
|
|
2628
|
+
: "indexing";
|
|
2629
|
+
const indexErrors = scan.qualitativeIndexErrors ?? 0;
|
|
2630
|
+
if (indexErrors > 0) {
|
|
2631
|
+
diagnostics.push({
|
|
2632
|
+
agent: scan.agent,
|
|
2633
|
+
code: "qualitative_index_error",
|
|
2634
|
+
severity: "warning",
|
|
2635
|
+
message: `${agentLabel(scan.agent)} private index had ${indexErrors} read/write failure(s); project coverage is rebuilding. If this persists after several runs, a newer aibill version may own the index.`,
|
|
2636
|
+
count: indexErrors
|
|
2637
|
+
});
|
|
2638
|
+
}
|
|
2639
|
+
const skipped = scan.qualitativeFilesSkippedForBudget ?? 0;
|
|
2640
|
+
if (skipped > 0) {
|
|
2641
|
+
diagnostics.push({
|
|
2642
|
+
agent: scan.agent,
|
|
2643
|
+
code: "qualitative_scan_incomplete",
|
|
2644
|
+
severity: "warning",
|
|
2645
|
+
message: `${agentLabel(scan.agent)} qualitative coverage is partial: ${skipped} eligible transcript file(s) exceeded the configured scan limits. No omitted file contributed an action finding.`,
|
|
2646
|
+
count: skipped
|
|
2647
|
+
});
|
|
2648
|
+
}
|
|
2649
|
+
}
|
|
923
2650
|
function financialSourceScan(descriptor) {
|
|
924
2651
|
const scan = {
|
|
925
2652
|
...emptySourceScan(descriptor.id),
|
|
@@ -1317,6 +3044,30 @@ function newErrorWithCode(code) {
|
|
|
1317
3044
|
error.code = code;
|
|
1318
3045
|
return error;
|
|
1319
3046
|
}
|
|
3047
|
+
function claudeNativeResponseIdentity(message, entry) {
|
|
3048
|
+
const rawMessageId = stringOf(message.id);
|
|
3049
|
+
const rawRequestId = stringOf(entry.requestId);
|
|
3050
|
+
// Never weaken a two-part native identity by silently dropping one
|
|
3051
|
+
// attacker-sized component; without the exact bounded pair there is no
|
|
3052
|
+
// cross-file deduplication proof.
|
|
3053
|
+
if ((rawMessageId && rawMessageId.length > 4_096) ||
|
|
3054
|
+
(rawRequestId && rawRequestId.length > 4_096)) {
|
|
3055
|
+
return undefined;
|
|
3056
|
+
}
|
|
3057
|
+
const messageId = rawMessageId || undefined;
|
|
3058
|
+
const requestId = rawRequestId || undefined;
|
|
3059
|
+
if (!messageId && !requestId)
|
|
3060
|
+
return undefined;
|
|
3061
|
+
const localDedupeKey = JSON.stringify([messageId ?? null, requestId ?? null]);
|
|
3062
|
+
return {
|
|
3063
|
+
localDedupeKey,
|
|
3064
|
+
callId: `callref_${createHash("sha256")
|
|
3065
|
+
.update("claude-native-response-v1")
|
|
3066
|
+
.update("\u0000")
|
|
3067
|
+
.update(localDedupeKey)
|
|
3068
|
+
.digest("hex")}`
|
|
3069
|
+
};
|
|
3070
|
+
}
|
|
1320
3071
|
function parseClaudeFinancialEntry(entry, filePath, sinceMs, seen, onDiagnostic) {
|
|
1321
3072
|
if (entry.type !== "assistant")
|
|
1322
3073
|
return undefined;
|
|
@@ -1324,10 +3075,11 @@ function parseClaudeFinancialEntry(entry, filePath, sinceMs, seen, onDiagnostic)
|
|
|
1324
3075
|
const usage = message && isRecord(message.usage) ? message.usage : undefined;
|
|
1325
3076
|
if (!message || !usage || stringOf(message.model) === "<synthetic>")
|
|
1326
3077
|
return undefined;
|
|
1327
|
-
const
|
|
1328
|
-
if (
|
|
3078
|
+
const nativeResponse = claudeNativeResponseIdentity(message, entry);
|
|
3079
|
+
if (nativeResponse && seen.has(nativeResponse.localDedupeKey))
|
|
1329
3080
|
return undefined;
|
|
1330
|
-
|
|
3081
|
+
if (nativeResponse)
|
|
3082
|
+
seen.add(nativeResponse.localDedupeKey);
|
|
1331
3083
|
const timestamp = toIso(stringOf(entry.timestamp)) ?? new Date(0).toISOString();
|
|
1332
3084
|
if (typeof sinceMs === "number" && Date.parse(timestamp) < sinceMs)
|
|
1333
3085
|
return undefined;
|
|
@@ -1335,11 +3087,13 @@ function parseClaudeFinancialEntry(entry, filePath, sinceMs, seen, onDiagnostic)
|
|
|
1335
3087
|
const workingDirectory = absoluteWorkingDirectory(stringOf(entry.cwd));
|
|
1336
3088
|
return {
|
|
1337
3089
|
agent: "claude-code",
|
|
3090
|
+
...(nativeResponse ? { callId: nativeResponse.callId } : {}),
|
|
1338
3091
|
model: stringOf(message.model) ?? "claude-code",
|
|
1339
3092
|
timestamp,
|
|
1340
3093
|
project: projectFromCwd(workingDirectory) ?? projectFromTranscriptPath(filePath),
|
|
1341
3094
|
workingDirectory,
|
|
1342
3095
|
sessionId: stringOf(entry.sessionId),
|
|
3096
|
+
...(stringOf(entry.version) ? { sourceVersion: stringOf(entry.version) } : {}),
|
|
1343
3097
|
...(parsedUsage.latestTurnUsage
|
|
1344
3098
|
? { latestTurnUsage: parsedUsage.latestTurnUsage }
|
|
1345
3099
|
: {}),
|
|
@@ -1348,6 +3102,9 @@ function parseClaudeFinancialEntry(entry, filePath, sinceMs, seen, onDiagnostic)
|
|
|
1348
3102
|
...(parsedUsage.reportedTotalTokens !== undefined
|
|
1349
3103
|
? { reportedTotalTokens: parsedUsage.reportedTotalTokens }
|
|
1350
3104
|
: {}),
|
|
3105
|
+
...(parsedUsage.tokenComponentEvidence
|
|
3106
|
+
? { tokenComponentEvidence: parsedUsage.tokenComponentEvidence }
|
|
3107
|
+
: {}),
|
|
1351
3108
|
usage: parsedUsage.usage
|
|
1352
3109
|
};
|
|
1353
3110
|
}
|
|
@@ -1363,6 +3120,7 @@ function consumeCodexFinancialEntry(state, entry) {
|
|
|
1363
3120
|
if (entry.type === "session_meta" && payload && !state.rootSessionMetaSeen) {
|
|
1364
3121
|
state.rootSessionMetaSeen = true;
|
|
1365
3122
|
state.sessionId = stringOf(payload.id);
|
|
3123
|
+
state.sourceVersion = stringOf(payload.cli_version);
|
|
1366
3124
|
state.rootCwd = stringOf(payload.cwd);
|
|
1367
3125
|
state.startedAt = toIso(stringOf(payload.timestamp) ?? stringOf(entry.timestamp));
|
|
1368
3126
|
state.rootStartedAtMs = timestampMilliseconds(payload.timestamp ?? entry.timestamp);
|
|
@@ -1431,6 +3189,7 @@ function finishCodexFinancialStream(state, onDiagnostic) {
|
|
|
1431
3189
|
project: projectFromCwd(workingDirectory),
|
|
1432
3190
|
workingDirectory,
|
|
1433
3191
|
sessionId: state.sessionId,
|
|
3192
|
+
...(state.sourceVersion ? { sourceVersion: state.sourceVersion } : {}),
|
|
1434
3193
|
rateLimits: state.lastRateLimits,
|
|
1435
3194
|
...(usageSupport === "complete" && parsedTurn.usage
|
|
1436
3195
|
? { latestTurnUsage: parsedTurn.usage }
|
|
@@ -1440,6 +3199,9 @@ function finishCodexFinancialStream(state, onDiagnostic) {
|
|
|
1440
3199
|
...(parsedUsage.reportedTotalTokens !== undefined
|
|
1441
3200
|
? { reportedTotalTokens: parsedUsage.reportedTotalTokens }
|
|
1442
3201
|
: {}),
|
|
3202
|
+
...(usageSupport === "complete" && parsedUsage.tokenComponentEvidence
|
|
3203
|
+
? { tokenComponentEvidence: parsedUsage.tokenComponentEvidence }
|
|
3204
|
+
: {}),
|
|
1443
3205
|
usage: parsedUsage.usage
|
|
1444
3206
|
};
|
|
1445
3207
|
}
|
|
@@ -1745,6 +3507,35 @@ function projectFromTranscriptPath(filePath) {
|
|
|
1745
3507
|
const tail = parent.split("-").filter(Boolean).pop();
|
|
1746
3508
|
return tail && tail.length > 0 ? tail : undefined;
|
|
1747
3509
|
}
|
|
3510
|
+
/**
|
|
3511
|
+
* Parse the host-framed task-notification a background subagent's completion
|
|
3512
|
+
* writes into the owning transcript. Only the leading host template tags are
|
|
3513
|
+
* read (first match): the trailing `<result>` section is model output and is
|
|
3514
|
+
* never trusted or retained. The same task-id may notify more than once for
|
|
3515
|
+
* a resumed run; callers keep the latest record and the session-vitals join
|
|
3516
|
+
* still requires the record to postdate the run's last observed activity.
|
|
3517
|
+
*/
|
|
3518
|
+
function parseTaskNotification(content) {
|
|
3519
|
+
if (!content || !content.startsWith("<task-notification>"))
|
|
3520
|
+
return undefined;
|
|
3521
|
+
const taskId = /<task-id>([A-Za-z0-9._-]{1,128})<\/task-id>/.exec(content)?.[1];
|
|
3522
|
+
const status = /<status>([a-z_-]{1,64})<\/status>/.exec(content)?.[1];
|
|
3523
|
+
return taskId && status ? { taskId, status } : undefined;
|
|
3524
|
+
}
|
|
3525
|
+
/**
|
|
3526
|
+
* Fallback subagent-run identity from the `subagents/agent-<id>.jsonl` file
|
|
3527
|
+
* name, for subagent transcripts whose lines omit `agentId`. Never derived
|
|
3528
|
+
* for files outside a `subagents` directory, and never a path (one validated
|
|
3529
|
+
* basename segment only).
|
|
3530
|
+
*/
|
|
3531
|
+
function subagentTranscriptFileId(filePath) {
|
|
3532
|
+
const segments = filePath.split(sep);
|
|
3533
|
+
if (!segments.includes("subagents"))
|
|
3534
|
+
return undefined;
|
|
3535
|
+
const base = segments.at(-1) ?? "";
|
|
3536
|
+
const name = base.endsWith(".jsonl") ? base.slice(0, -".jsonl".length) : base;
|
|
3537
|
+
return /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(name) ? name : undefined;
|
|
3538
|
+
}
|
|
1748
3539
|
function localActivityScopeKey(sessionId, workingDirectory, project) {
|
|
1749
3540
|
// The absolute cwd stays inside this ephemeral parser key and is never
|
|
1750
3541
|
// persisted or rendered. It prevents one transcript that changes projects
|
|
@@ -1821,11 +3612,11 @@ function buildLocalAgentActivity(input) {
|
|
|
1821
3612
|
kind,
|
|
1822
3613
|
action,
|
|
1823
3614
|
source,
|
|
1824
|
-
promptCount: prompts.length,
|
|
3615
|
+
promptCount: prompts.length + (input.priorPromptCount ?? 0),
|
|
1825
3616
|
toolCallCount: input.toolCallCount,
|
|
1826
3617
|
files,
|
|
1827
3618
|
isSubagent: input.isSubagent,
|
|
1828
|
-
parentSessionId: input.parentSessionId
|
|
3619
|
+
...(input.parentSessionId ? { parentSessionId: input.parentSessionId } : {})
|
|
1829
3620
|
};
|
|
1830
3621
|
}
|
|
1831
3622
|
function focusTopic(prompts) {
|
|
@@ -1978,11 +3769,19 @@ function promptTokens(value) {
|
|
|
1978
3769
|
.map((token) => token.replace(/^[.-]+|[.-]+$/g, ""))
|
|
1979
3770
|
.filter((token) => token.length >= 2 && token !== "cz" && !/^\d+$/.test(token));
|
|
1980
3771
|
}
|
|
3772
|
+
/**
|
|
3773
|
+
* Strip absolute-path spans (including file:// forms) from free text. Shared
|
|
3774
|
+
* by topic derivation and by checkpoint prompt persistence: persisted prompt
|
|
3775
|
+
* survivors must never carry a raw local path (req 1).
|
|
3776
|
+
*/
|
|
3777
|
+
function stripAbsolutePathSpans(value) {
|
|
3778
|
+
return value.replace(/(^|[\s("'=:])(?:file:\/\/)?\/[^\s)"']+/g, "$1");
|
|
3779
|
+
}
|
|
1981
3780
|
function topicTokens(value) {
|
|
1982
3781
|
// Absolute paths often appear in attached-image metadata and tool-oriented
|
|
1983
3782
|
// prompts. They are machine context, not the user's work topic, and can
|
|
1984
3783
|
// otherwise outrank meaningful words when only one recent prompt exists.
|
|
1985
|
-
const withoutAbsolutePaths = value
|
|
3784
|
+
const withoutAbsolutePaths = stripAbsolutePathSpans(value)
|
|
1986
3785
|
.replace(/\b[^\s/\\]+\.(?:png|jpe?g|gif|webp|heic|svg|pdf|mov|mp4)\b/gi, " ")
|
|
1987
3786
|
.replace(/\b(?:attached|attachment|clipboard|image|images|photo|picture|screenshot|screenshots)\b/gi, " ");
|
|
1988
3787
|
return promptTokens(sanitizeLocalActivityText(withoutAbsolutePaths));
|