@agent-finops/core 0.5.9 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/activitySnapshot.d.ts +676 -0
- package/dist/activitySnapshot.js +1220 -0
- package/dist/activitySnapshotCache.d.ts +54 -0
- package/dist/activitySnapshotCache.js +489 -0
- package/dist/discovery.d.ts +6 -2
- package/dist/discovery.js +36 -14
- package/dist/glance.d.ts +6 -2
- package/dist/glance.js +62 -13
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/localAgentLogs.d.ts +68 -2
- package/dist/localAgentLogs.js +972 -59
- package/dist/modelPricing.js +0 -1
- package/dist/providerConnectors.d.ts +13 -2
- package/dist/providerConnectors.js +708 -95
- package/dist/sampleData.js +4 -3
- package/dist/schema.d.ts +31 -29
- package/dist/schema.js +27 -3
- package/dist/sourceRegistry.d.ts +30 -5
- package/dist/sourceRegistry.js +250 -21
- package/dist/sourceStatus.d.ts +65 -0
- package/dist/sourceStatus.js +147 -0
- package/dist/stateTrust.d.ts +37 -0
- package/dist/stateTrust.js +277 -0
- package/package.json +1 -1
- package/samples/openai-usage.csv +2 -2
package/dist/localAgentLogs.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createReadStream } from "node:fs";
|
|
2
|
+
import { lstat, open, readdir, readFile, stat } from "node:fs/promises";
|
|
2
3
|
import { basename, isAbsolute, join, resolve, sep } from "node:path";
|
|
3
4
|
import { homedir } from "node:os";
|
|
5
|
+
import { createInterface } from "node:readline";
|
|
4
6
|
import { estimateTokenCostUsd } from "./modelPricing.js";
|
|
5
7
|
import { redactSecrets } from "./discovery.js";
|
|
6
8
|
import { createCodexInvocationCollector } from "./toolInvocations.js";
|
|
@@ -52,8 +54,143 @@ function totalUsageTokens(usage) {
|
|
|
52
54
|
(usage.cacheWrite5mTokens ?? 0) +
|
|
53
55
|
(usage.cacheWrite1hTokens ?? 0);
|
|
54
56
|
}
|
|
57
|
+
function parseClaudeFinancialUsage(value, onDiagnostic) {
|
|
58
|
+
const inputTokens = tokenComponentOf(value.input_tokens);
|
|
59
|
+
const outputTokens = tokenComponentOf(value.output_tokens);
|
|
60
|
+
const cacheReadField = optionalTokenComponent(value, "cache_read_input_tokens");
|
|
61
|
+
const writeTotalField = optionalTokenComponent(value, "cache_creation_input_tokens");
|
|
62
|
+
const reportedTotalField = optionalTokenComponent(value, "total_tokens");
|
|
63
|
+
const cacheCreationPresent = Object.prototype.hasOwnProperty.call(value, "cache_creation");
|
|
64
|
+
const cacheCreation = isRecord(value.cache_creation) ? value.cache_creation : undefined;
|
|
65
|
+
const write5mField = cacheCreation
|
|
66
|
+
? optionalTokenComponent(cacheCreation, "ephemeral_5m_input_tokens")
|
|
67
|
+
: { present: false };
|
|
68
|
+
const write1hField = cacheCreation
|
|
69
|
+
? optionalTokenComponent(cacheCreation, "ephemeral_1h_input_tokens")
|
|
70
|
+
: { present: false };
|
|
71
|
+
const componentsSupported = inputTokens !== undefined &&
|
|
72
|
+
outputTokens !== undefined &&
|
|
73
|
+
(!cacheReadField.present || cacheReadField.value !== undefined) &&
|
|
74
|
+
(!writeTotalField.present || writeTotalField.value !== undefined) &&
|
|
75
|
+
(!reportedTotalField.present || reportedTotalField.value !== undefined) &&
|
|
76
|
+
(!cacheCreationPresent || Boolean(cacheCreation)) &&
|
|
77
|
+
(!write5mField.present || write5mField.value !== undefined) &&
|
|
78
|
+
(!write1hField.present || write1hField.value !== undefined);
|
|
79
|
+
const usage = {
|
|
80
|
+
// Retain every valid component for partial evidence, but never let a
|
|
81
|
+
// missing/invalid required field become a priceable zero-dollar call.
|
|
82
|
+
inputTokens: inputTokens ?? 0,
|
|
83
|
+
outputTokens: outputTokens ?? 0,
|
|
84
|
+
cacheReadTokens: cacheReadField.value ?? 0,
|
|
85
|
+
// Prefer the 5m/1h breakdown; fall back to the total as 5m (cheaper bound).
|
|
86
|
+
cacheWrite5mTokens: write5mField.value ?? writeTotalField.value ?? 0,
|
|
87
|
+
cacheWrite1hTokens: write1hField.value ?? 0
|
|
88
|
+
};
|
|
89
|
+
if (componentsSupported) {
|
|
90
|
+
return {
|
|
91
|
+
usage,
|
|
92
|
+
latestTurnUsage: toTurnUsage(usage, "assistant_message_usage")
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
onDiagnostic?.({ code: "unsupported_token_shape", count: 1 });
|
|
96
|
+
const reportedTotalTokens = reportedTotalField.value;
|
|
97
|
+
return {
|
|
98
|
+
usage,
|
|
99
|
+
usageSupport: "unsupported_token_shape",
|
|
100
|
+
...(reportedTotalTokens !== undefined ? { reportedTotalTokens } : {})
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
function optionalTokenComponent(record, key) {
|
|
104
|
+
if (!Object.prototype.hasOwnProperty.call(record, key))
|
|
105
|
+
return { present: false };
|
|
106
|
+
const value = tokenComponentOf(record[key]);
|
|
107
|
+
return value === undefined ? { present: true } : { present: true, value };
|
|
108
|
+
}
|
|
109
|
+
function parseCodexCumulativeUsage(current, baseline) {
|
|
110
|
+
const rawInput = tokenComponentOf(current.input_tokens);
|
|
111
|
+
const rawOutput = tokenComponentOf(current.output_tokens);
|
|
112
|
+
const currentCachedField = optionalTokenComponent(current, "cached_input_tokens");
|
|
113
|
+
const currentTotalField = optionalTokenComponent(current, "total_tokens");
|
|
114
|
+
const rawCached = currentCachedField.value ?? 0;
|
|
115
|
+
const rawReportedTotal = currentTotalField.value;
|
|
116
|
+
const baselineInput = baseline ? tokenComponentOf(baseline.input_tokens) : undefined;
|
|
117
|
+
const baselineOutput = baseline ? tokenComponentOf(baseline.output_tokens) : undefined;
|
|
118
|
+
const baselineCachedField = baseline
|
|
119
|
+
? optionalTokenComponent(baseline, "cached_input_tokens")
|
|
120
|
+
: { present: false };
|
|
121
|
+
const baselineTotalField = baseline
|
|
122
|
+
? optionalTokenComponent(baseline, "total_tokens")
|
|
123
|
+
: { present: false };
|
|
124
|
+
const baselineCached = baselineCachedField.value ?? 0;
|
|
125
|
+
const baselineReportedTotal = baselineTotalField.value;
|
|
126
|
+
const currentSupported = rawInput !== undefined &&
|
|
127
|
+
rawOutput !== undefined &&
|
|
128
|
+
(!currentCachedField.present || currentCachedField.value !== undefined) &&
|
|
129
|
+
(!currentTotalField.present || currentTotalField.value !== undefined) &&
|
|
130
|
+
rawCached <= rawInput &&
|
|
131
|
+
(rawReportedTotal === undefined || rawReportedTotal >= rawInput + rawOutput) &&
|
|
132
|
+
!((rawReportedTotal ?? 0) > 0 && rawInput === 0 && rawOutput === 0);
|
|
133
|
+
const baselineSupported = !baseline || (baselineInput !== undefined &&
|
|
134
|
+
baselineOutput !== undefined &&
|
|
135
|
+
(!baselineCachedField.present || baselineCachedField.value !== undefined) &&
|
|
136
|
+
(!baselineTotalField.present || baselineTotalField.value !== undefined) &&
|
|
137
|
+
baselineCached <= baselineInput &&
|
|
138
|
+
(baselineReportedTotal === undefined ||
|
|
139
|
+
baselineReportedTotal >= baselineInput + baselineOutput) &&
|
|
140
|
+
!((baselineReportedTotal ?? 0) > 0 && baselineInput === 0 && baselineOutput === 0));
|
|
141
|
+
const monotonic = !baseline || (rawInput !== undefined && baselineInput !== undefined && rawInput >= baselineInput &&
|
|
142
|
+
rawOutput !== undefined && baselineOutput !== undefined && rawOutput >= baselineOutput &&
|
|
143
|
+
rawCached >= baselineCached &&
|
|
144
|
+
rawInput - rawCached >= baselineInput - baselineCached &&
|
|
145
|
+
(rawReportedTotal === undefined ||
|
|
146
|
+
baselineReportedTotal === undefined ||
|
|
147
|
+
rawReportedTotal >= baselineReportedTotal));
|
|
148
|
+
const input = Math.max(0, (rawInput ?? 0) - (baselineInput ?? 0));
|
|
149
|
+
const cached = Math.max(0, rawCached - baselineCached);
|
|
150
|
+
const output = Math.max(0, (rawOutput ?? 0) - (baselineOutput ?? 0));
|
|
151
|
+
const reportedTotalTokens = rawReportedTotal === undefined
|
|
152
|
+
? undefined
|
|
153
|
+
: Math.max(0, rawReportedTotal - (baselineReportedTotal ?? 0));
|
|
154
|
+
return {
|
|
155
|
+
usage: {
|
|
156
|
+
// Codex input_tokens includes cached input; expose the non-cached split.
|
|
157
|
+
inputTokens: Math.max(0, input - cached),
|
|
158
|
+
outputTokens: output,
|
|
159
|
+
cacheReadTokens: cached
|
|
160
|
+
},
|
|
161
|
+
supported: currentSupported && baselineSupported && monotonic,
|
|
162
|
+
...(reportedTotalTokens !== undefined ? { reportedTotalTokens } : {})
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
function parseCodexTurnUsage(value) {
|
|
166
|
+
const rawInput = tokenComponentOf(value.input_tokens);
|
|
167
|
+
const rawOutput = tokenComponentOf(value.output_tokens);
|
|
168
|
+
const cachedField = optionalTokenComponent(value, "cached_input_tokens");
|
|
169
|
+
const totalField = optionalTokenComponent(value, "total_tokens");
|
|
170
|
+
const cached = cachedField.value ?? 0;
|
|
171
|
+
const total = totalField.value;
|
|
172
|
+
const supported = rawInput !== undefined &&
|
|
173
|
+
rawOutput !== undefined &&
|
|
174
|
+
(!cachedField.present || cachedField.value !== undefined) &&
|
|
175
|
+
(!totalField.present || totalField.value !== undefined) &&
|
|
176
|
+
cached <= rawInput &&
|
|
177
|
+
(total === undefined || total >= rawInput + rawOutput);
|
|
178
|
+
if (!supported)
|
|
179
|
+
return { supported: false };
|
|
180
|
+
return {
|
|
181
|
+
supported: true,
|
|
182
|
+
usage: {
|
|
183
|
+
inputTokens: rawInput - cached,
|
|
184
|
+
outputTokens: rawOutput,
|
|
185
|
+
cacheReadTokens: cached,
|
|
186
|
+
contextTokens: rawInput,
|
|
187
|
+
totalTokens: total ?? rawInput + rawOutput,
|
|
188
|
+
source: "transcript_last_token_usage"
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
}
|
|
55
192
|
/** Parse one Claude Code transcript (JSONL). Exported for tests. */
|
|
56
|
-
export function parseClaudeCodeTranscript(content, filePath = "", sinceMs) {
|
|
193
|
+
export function parseClaudeCodeTranscript(content, filePath = "", sinceMs, onDiagnostic) {
|
|
57
194
|
const calls = [];
|
|
58
195
|
const seen = new Set();
|
|
59
196
|
const pendingPrompts = [];
|
|
@@ -63,6 +200,7 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs) {
|
|
|
63
200
|
let latestActivityKey;
|
|
64
201
|
let isSubagent = filePath.split(sep).includes("subagents");
|
|
65
202
|
let parentSessionId;
|
|
203
|
+
let malformedLines = 0;
|
|
66
204
|
for (const line of content.split("\n")) {
|
|
67
205
|
if (!line.trim())
|
|
68
206
|
continue;
|
|
@@ -71,6 +209,7 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs) {
|
|
|
71
209
|
entry = JSON.parse(line);
|
|
72
210
|
}
|
|
73
211
|
catch {
|
|
212
|
+
malformedLines += 1;
|
|
74
213
|
continue;
|
|
75
214
|
}
|
|
76
215
|
if (!isRecord(entry))
|
|
@@ -111,18 +250,7 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs) {
|
|
|
111
250
|
pendingPrompts.length = 0;
|
|
112
251
|
continue;
|
|
113
252
|
}
|
|
114
|
-
const
|
|
115
|
-
const write5m = numberOf(cacheCreation?.ephemeral_5m_input_tokens);
|
|
116
|
-
const write1h = numberOf(cacheCreation?.ephemeral_1h_input_tokens);
|
|
117
|
-
const writeTotal = numberOf(usage.cache_creation_input_tokens) ?? 0;
|
|
118
|
-
const parsedUsage = {
|
|
119
|
-
inputTokens: numberOf(usage.input_tokens) ?? 0,
|
|
120
|
-
outputTokens: numberOf(usage.output_tokens) ?? 0,
|
|
121
|
-
cacheReadTokens: numberOf(usage.cache_read_input_tokens) ?? 0,
|
|
122
|
-
// Prefer the 5m/1h breakdown; fall back to the total as 5m (cheaper bound).
|
|
123
|
-
cacheWrite5mTokens: write5m ?? writeTotal,
|
|
124
|
-
cacheWrite1hTokens: write1h ?? 0
|
|
125
|
-
};
|
|
253
|
+
const parsedUsage = parseClaudeFinancialUsage(usage, onDiagnostic);
|
|
126
254
|
const workingDirectory = absoluteWorkingDirectory(stringOf(entry.cwd));
|
|
127
255
|
const project = projectFromCwd(workingDirectory) ?? projectFromTranscriptPath(filePath);
|
|
128
256
|
const sessionId = stringOf(entry.sessionId);
|
|
@@ -133,9 +261,15 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs) {
|
|
|
133
261
|
project,
|
|
134
262
|
workingDirectory,
|
|
135
263
|
sessionId,
|
|
136
|
-
|
|
264
|
+
...(parsedUsage.latestTurnUsage
|
|
265
|
+
? { latestTurnUsage: parsedUsage.latestTurnUsage }
|
|
266
|
+
: {}),
|
|
137
267
|
usageScope: "turn",
|
|
138
|
-
|
|
268
|
+
...(parsedUsage.usageSupport ? { usageSupport: parsedUsage.usageSupport } : {}),
|
|
269
|
+
...(parsedUsage.reportedTotalTokens !== undefined
|
|
270
|
+
? { reportedTotalTokens: parsedUsage.reportedTotalTokens }
|
|
271
|
+
: {}),
|
|
272
|
+
usage: parsedUsage.usage
|
|
139
273
|
};
|
|
140
274
|
calls.push(call);
|
|
141
275
|
const activityKey = localActivityScopeKey(sessionId, workingDirectory, project);
|
|
@@ -179,10 +313,13 @@ export function parseClaudeCodeTranscript(content, filePath = "", sinceMs) {
|
|
|
179
313
|
for (const call of calls) {
|
|
180
314
|
call.activity = activities.get(localActivityScopeKey(call.sessionId, call.workingDirectory, call.project));
|
|
181
315
|
}
|
|
316
|
+
if (malformedLines > 0) {
|
|
317
|
+
onDiagnostic?.({ code: "malformed_jsonl", count: malformedLines });
|
|
318
|
+
}
|
|
182
319
|
return calls;
|
|
183
320
|
}
|
|
184
321
|
/** Parse one Codex rollout file (JSONL event stream). Exported for tests. */
|
|
185
|
-
export function parseCodexRollout(content, onEntry) {
|
|
322
|
+
export function parseCodexRollout(content, onEntry, onDiagnostic) {
|
|
186
323
|
let model;
|
|
187
324
|
let rootCwd;
|
|
188
325
|
const toolWorkdirs = new Map();
|
|
@@ -201,6 +338,7 @@ export function parseCodexRollout(content, onEntry) {
|
|
|
201
338
|
let toolCallCount = 0;
|
|
202
339
|
let isSubagent = false;
|
|
203
340
|
let parentSessionId;
|
|
341
|
+
let malformedLines = 0;
|
|
204
342
|
for (const line of content.split("\n")) {
|
|
205
343
|
if (!line.trim())
|
|
206
344
|
continue;
|
|
@@ -209,6 +347,7 @@ export function parseCodexRollout(content, onEntry) {
|
|
|
209
347
|
entry = JSON.parse(line);
|
|
210
348
|
}
|
|
211
349
|
catch {
|
|
350
|
+
malformedLines += 1;
|
|
212
351
|
continue;
|
|
213
352
|
}
|
|
214
353
|
if (!isRecord(entry))
|
|
@@ -300,17 +439,21 @@ export function parseCodexRollout(content, onEntry) {
|
|
|
300
439
|
// safer than charging the parent cumulative counter again. Likewise, a
|
|
301
440
|
// recognized boundary with no later total_token_usage is not a financial
|
|
302
441
|
// call yet.
|
|
442
|
+
if (malformedLines > 0) {
|
|
443
|
+
onDiagnostic?.({ code: "malformed_jsonl", count: malformedLines });
|
|
444
|
+
}
|
|
303
445
|
if (!lastTotal || isSubagent && !rootTaskStarted)
|
|
304
446
|
return [];
|
|
305
|
-
const
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
const
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
:
|
|
447
|
+
const parsedUsage = parseCodexCumulativeUsage(lastTotal, inheritedUsageBaseline);
|
|
448
|
+
const parsedTurn = lastTurn
|
|
449
|
+
? parseCodexTurnUsage(lastTurn)
|
|
450
|
+
: { supported: true };
|
|
451
|
+
const usageSupport = parsedUsage.supported && parsedTurn.supported
|
|
452
|
+
? "complete"
|
|
453
|
+
: "unsupported_token_shape";
|
|
454
|
+
if (usageSupport === "unsupported_token_shape") {
|
|
455
|
+
onDiagnostic?.({ code: "unsupported_token_shape", count: 1 });
|
|
456
|
+
}
|
|
314
457
|
const workingDirectory = absoluteWorkingDirectory(dominantCodexCwd(rootCwd, toolWorkdirs));
|
|
315
458
|
const project = projectFromCwd(workingDirectory);
|
|
316
459
|
const activity = buildLocalAgentActivity({
|
|
@@ -331,14 +474,15 @@ export function parseCodexRollout(content, onEntry) {
|
|
|
331
474
|
sessionId,
|
|
332
475
|
rateLimits: lastRateLimits,
|
|
333
476
|
activity,
|
|
334
|
-
|
|
477
|
+
...(usageSupport === "complete" && parsedTurn.usage
|
|
478
|
+
? { latestTurnUsage: parsedTurn.usage }
|
|
479
|
+
: {}),
|
|
335
480
|
usageScope: "session_cumulative",
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
}
|
|
481
|
+
usageSupport,
|
|
482
|
+
...(parsedUsage.reportedTotalTokens !== undefined
|
|
483
|
+
? { reportedTotalTokens: parsedUsage.reportedTotalTokens }
|
|
484
|
+
: {}),
|
|
485
|
+
usage: parsedUsage.usage
|
|
342
486
|
}];
|
|
343
487
|
}
|
|
344
488
|
function collectEmbeddedToolWorkdirs(input, toolWorkdirs) {
|
|
@@ -417,24 +561,51 @@ export async function loadLocalAgentUsage(options = {}) {
|
|
|
417
561
|
let filesParsed = 0;
|
|
418
562
|
const since = options.sinceIso ? Date.parse(options.sinceIso) : undefined;
|
|
419
563
|
const sinceMs = typeof since === "number" && Number.isFinite(since) ? since : undefined;
|
|
420
|
-
|
|
421
|
-
|
|
564
|
+
const diagnostics = [];
|
|
565
|
+
const sourceScans = [
|
|
566
|
+
emptySourceScan("claude-code"),
|
|
567
|
+
emptySourceScan("codex")
|
|
568
|
+
];
|
|
569
|
+
const claudeScan = sourceScans[0];
|
|
570
|
+
const codexScan = sourceScans[1];
|
|
571
|
+
for (const file of await listJsonlFiles(claudeDir, claudeScan, diagnostics)) {
|
|
572
|
+
let content;
|
|
573
|
+
try {
|
|
574
|
+
content = await readFile(file, "utf8");
|
|
575
|
+
}
|
|
576
|
+
catch (error) {
|
|
577
|
+
recordUnreadableFile("claude-code", claudeScan, diagnostics, error);
|
|
578
|
+
continue;
|
|
579
|
+
}
|
|
422
580
|
if (!content)
|
|
423
581
|
continue;
|
|
424
582
|
filesParsed += 1;
|
|
425
|
-
|
|
583
|
+
claudeScan.filesParsed += 1;
|
|
584
|
+
calls.push(...parseClaudeCodeTranscript(content, file, sinceMs, (diagnostic) => {
|
|
585
|
+
recordParseDiagnostic("claude-code", claudeScan, diagnostics, diagnostic);
|
|
586
|
+
}));
|
|
426
587
|
}
|
|
427
|
-
for (const file of await listJsonlFiles(codexDir)) {
|
|
588
|
+
for (const file of await listJsonlFiles(codexDir, codexScan, diagnostics)) {
|
|
428
589
|
if (!basename(file).startsWith("rollout-"))
|
|
429
590
|
continue;
|
|
430
|
-
|
|
591
|
+
let content;
|
|
592
|
+
try {
|
|
593
|
+
content = await readFile(file, "utf8");
|
|
594
|
+
}
|
|
595
|
+
catch (error) {
|
|
596
|
+
recordUnreadableFile("codex", codexScan, diagnostics, error);
|
|
597
|
+
continue;
|
|
598
|
+
}
|
|
431
599
|
if (!content)
|
|
432
600
|
continue;
|
|
433
601
|
filesParsed += 1;
|
|
602
|
+
codexScan.filesParsed += 1;
|
|
434
603
|
const collector = codexInvocationFiles
|
|
435
604
|
? createCodexInvocationCollector(sinceMs)
|
|
436
605
|
: undefined;
|
|
437
|
-
calls.push(...parseCodexRollout(content, collector?.consume)
|
|
606
|
+
calls.push(...parseCodexRollout(content, collector?.consume, (diagnostic) => {
|
|
607
|
+
recordParseDiagnostic("codex", codexScan, diagnostics, diagnostic);
|
|
608
|
+
}));
|
|
438
609
|
if (collector)
|
|
439
610
|
codexInvocationFiles.push(collector.finish());
|
|
440
611
|
}
|
|
@@ -447,9 +618,653 @@ export async function loadLocalAgentUsage(options = {}) {
|
|
|
447
618
|
calls: filtered,
|
|
448
619
|
filesParsed,
|
|
449
620
|
agentsDetected: [...new Set(filtered.map((call) => call.agent))],
|
|
621
|
+
sourceScans,
|
|
622
|
+
diagnostics,
|
|
450
623
|
...(codexInvocationFiles ? { codexInvocationFiles } : {})
|
|
451
624
|
};
|
|
452
625
|
}
|
|
626
|
+
/**
|
|
627
|
+
* Stream only the financial evidence required by init/status snapshots.
|
|
628
|
+
*
|
|
629
|
+
* Unlike `loadLocalAgentUsage`, this path never derives prompts, file focus,
|
|
630
|
+
* tool activity, or invocation evidence. It deliberately returns the same
|
|
631
|
+
* result contract so callers can preserve the existing evidence and
|
|
632
|
+
* diagnostics vocabulary without maintaining a second loader schema. Codex
|
|
633
|
+
* project attribution intentionally uses only root metadata; home-launched
|
|
634
|
+
* tool-workdir inference remains exclusive to the full qualitative loader.
|
|
635
|
+
*/
|
|
636
|
+
export async function loadLocalAgentFinancialUsage(options = {}) {
|
|
637
|
+
const home = homedir();
|
|
638
|
+
const claudeDir = options.claudeProjectsDir ?? join(home, ".claude", "projects");
|
|
639
|
+
const codexDir = options.codexSessionsDir ?? join(home, ".codex", "sessions");
|
|
640
|
+
const claudeCalls = [];
|
|
641
|
+
const codexCalls = [];
|
|
642
|
+
const since = options.sinceIso ? Date.parse(options.sinceIso) : undefined;
|
|
643
|
+
const sinceMs = typeof since === "number" && Number.isFinite(since) ? since : undefined;
|
|
644
|
+
const claudeDiagnostics = [];
|
|
645
|
+
const codexDiagnostics = [];
|
|
646
|
+
const sourceScans = [
|
|
647
|
+
{
|
|
648
|
+
...emptySourceScan("claude-code"),
|
|
649
|
+
filesSkippedBeforeWindow: 0,
|
|
650
|
+
jsonlValidationCoverage: "complete"
|
|
651
|
+
},
|
|
652
|
+
{
|
|
653
|
+
...emptySourceScan("codex"),
|
|
654
|
+
filesSkippedBeforeWindow: 0,
|
|
655
|
+
filesReadFinancially: 0,
|
|
656
|
+
bytesSkippedAsNonFinancialHistory: 0,
|
|
657
|
+
nonFinancialLinesPrefiltered: 0,
|
|
658
|
+
nonFinancialBytesPrefiltered: 0,
|
|
659
|
+
jsonlValidationCoverage: "complete"
|
|
660
|
+
}
|
|
661
|
+
];
|
|
662
|
+
const claudeScan = sourceScans[0];
|
|
663
|
+
const codexScan = sourceScans[1];
|
|
664
|
+
const scanClaude = async () => {
|
|
665
|
+
for (const file of await listJsonlFiles(claudeDir, claudeScan, claudeDiagnostics)) {
|
|
666
|
+
if (!await shouldStreamFile(file, sinceMs, "claude-code", claudeScan, claudeDiagnostics)) {
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
const fileCalls = [];
|
|
670
|
+
const fileDiagnostics = [];
|
|
671
|
+
const seen = new Set();
|
|
672
|
+
let streamed;
|
|
673
|
+
try {
|
|
674
|
+
streamed = await streamJsonlRecords(file, (entry) => {
|
|
675
|
+
const call = parseClaudeFinancialEntry(entry, file, sinceMs, seen, (diagnostic) => fileDiagnostics.push(diagnostic));
|
|
676
|
+
if (call)
|
|
677
|
+
fileCalls.push(call);
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
catch (error) {
|
|
681
|
+
recordUnreadableFile("claude-code", claudeScan, claudeDiagnostics, error);
|
|
682
|
+
continue;
|
|
683
|
+
}
|
|
684
|
+
if (!streamed.hadContent)
|
|
685
|
+
continue;
|
|
686
|
+
claudeScan.filesParsed += 1;
|
|
687
|
+
claudeCalls.push(...fileCalls);
|
|
688
|
+
for (const diagnostic of fileDiagnostics) {
|
|
689
|
+
recordParseDiagnostic("claude-code", claudeScan, claudeDiagnostics, diagnostic);
|
|
690
|
+
}
|
|
691
|
+
if (streamed.malformedLines > 0) {
|
|
692
|
+
recordParseDiagnostic("claude-code", claudeScan, claudeDiagnostics, {
|
|
693
|
+
code: "malformed_jsonl",
|
|
694
|
+
count: streamed.malformedLines
|
|
695
|
+
});
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
};
|
|
699
|
+
const scanCodex = async () => {
|
|
700
|
+
for (const file of await listJsonlFiles(codexDir, codexScan, codexDiagnostics)) {
|
|
701
|
+
if (!basename(file).startsWith("rollout-"))
|
|
702
|
+
continue;
|
|
703
|
+
if (!await shouldStreamFile(file, sinceMs, "codex", codexScan, codexDiagnostics)) {
|
|
704
|
+
continue;
|
|
705
|
+
}
|
|
706
|
+
let financialFile;
|
|
707
|
+
try {
|
|
708
|
+
financialFile = await readCodexFinancialFile(file);
|
|
709
|
+
}
|
|
710
|
+
catch (error) {
|
|
711
|
+
recordUnreadableFile("codex", codexScan, codexDiagnostics, error);
|
|
712
|
+
continue;
|
|
713
|
+
}
|
|
714
|
+
if (!financialFile.hadContent)
|
|
715
|
+
continue;
|
|
716
|
+
codexScan.filesParsed += 1;
|
|
717
|
+
codexScan.filesReadFinancially = (codexScan.filesReadFinancially ?? 0) + 1;
|
|
718
|
+
codexScan.bytesSkippedAsNonFinancialHistory =
|
|
719
|
+
(codexScan.bytesSkippedAsNonFinancialHistory ?? 0) + financialFile.bytesSkipped;
|
|
720
|
+
codexScan.nonFinancialLinesPrefiltered =
|
|
721
|
+
(codexScan.nonFinancialLinesPrefiltered ?? 0) + financialFile.prefilteredLines;
|
|
722
|
+
codexScan.nonFinancialBytesPrefiltered =
|
|
723
|
+
(codexScan.nonFinancialBytesPrefiltered ?? 0) + financialFile.prefilteredBytes;
|
|
724
|
+
if (financialFile.bytesSkipped > 0 || financialFile.prefilteredLines > 0) {
|
|
725
|
+
codexScan.jsonlValidationCoverage = "financial_events_only";
|
|
726
|
+
}
|
|
727
|
+
if (financialFile.malformedLines > 0) {
|
|
728
|
+
recordParseDiagnostic("codex", codexScan, codexDiagnostics, {
|
|
729
|
+
code: "malformed_jsonl",
|
|
730
|
+
count: financialFile.malformedLines
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
const state = createCodexFinancialStreamState();
|
|
734
|
+
for (const entry of financialFile.entries) {
|
|
735
|
+
consumeCodexFinancialEntry(state, entry);
|
|
736
|
+
}
|
|
737
|
+
const call = finishCodexFinancialStream(state, (diagnostic) => {
|
|
738
|
+
recordParseDiagnostic("codex", codexScan, codexDiagnostics, diagnostic);
|
|
739
|
+
});
|
|
740
|
+
if (call)
|
|
741
|
+
codexCalls.push(call);
|
|
742
|
+
}
|
|
743
|
+
};
|
|
744
|
+
// The sources are independent and can be traversed concurrently without
|
|
745
|
+
// changing per-file ordering or cumulative-session deduplication.
|
|
746
|
+
await Promise.all([scanClaude(), scanCodex()]);
|
|
747
|
+
const calls = [...claudeCalls, ...codexCalls];
|
|
748
|
+
const normalizedCalls = dedupeCumulativeSessionCalls(calls);
|
|
749
|
+
const filtered = typeof sinceMs === "number"
|
|
750
|
+
? normalizedCalls.filter((call) => Date.parse(call.timestamp) >= sinceMs)
|
|
751
|
+
: normalizedCalls;
|
|
752
|
+
return {
|
|
753
|
+
records: aggregateCalls(filtered),
|
|
754
|
+
calls: filtered,
|
|
755
|
+
filesParsed: claudeScan.filesParsed + codexScan.filesParsed,
|
|
756
|
+
agentsDetected: [...new Set(filtered.map((call) => call.agent))],
|
|
757
|
+
sourceScans,
|
|
758
|
+
diagnostics: [...claudeDiagnostics, ...codexDiagnostics]
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
async function streamJsonlRecords(file, onRecord) {
|
|
762
|
+
const input = createReadStream(file, { encoding: "utf8" });
|
|
763
|
+
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
764
|
+
let hadContent = false;
|
|
765
|
+
let malformedLines = 0;
|
|
766
|
+
try {
|
|
767
|
+
for await (const line of lines) {
|
|
768
|
+
hadContent = true;
|
|
769
|
+
if (!line.trim())
|
|
770
|
+
continue;
|
|
771
|
+
let entry;
|
|
772
|
+
try {
|
|
773
|
+
entry = JSON.parse(line);
|
|
774
|
+
}
|
|
775
|
+
catch {
|
|
776
|
+
malformedLines += 1;
|
|
777
|
+
continue;
|
|
778
|
+
}
|
|
779
|
+
// Parsed event type is checked before any deeper traversal. This keeps
|
|
780
|
+
// qualitative payloads out of the fast path while still validating
|
|
781
|
+
// every JSONL line and reporting malformed coverage honestly.
|
|
782
|
+
if (isRecord(entry))
|
|
783
|
+
onRecord(entry);
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
finally {
|
|
787
|
+
lines.close();
|
|
788
|
+
input.destroy();
|
|
789
|
+
}
|
|
790
|
+
return { hadContent, malformedLines };
|
|
791
|
+
}
|
|
792
|
+
const FINANCIAL_REVERSE_CHUNK_BYTES = 16 * 1024 * 1024;
|
|
793
|
+
/**
|
|
794
|
+
* Codex token counters are cumulative. Reading the root metadata plus the
|
|
795
|
+
* newest complete financial tail is therefore equivalent to replaying copied
|
|
796
|
+
* prompt/tool history, while avoiding multi-gigabyte inherited histories in
|
|
797
|
+
* compaction and subagent rollouts.
|
|
798
|
+
*
|
|
799
|
+
* The reverse scan has proof-based stopping conditions. A normal rollout must
|
|
800
|
+
* expose the newest total, last-turn usage, model, and rate-limit snapshot.
|
|
801
|
+
* Subagent rollouts always continue to byte zero because the full parser's
|
|
802
|
+
* first qualifying task boundary cannot be proved from timestamps alone. If
|
|
803
|
+
* any normal-session proof is missing, it also continues to byte zero; no
|
|
804
|
+
* byte/line cap silently truncates evidence.
|
|
805
|
+
*/
|
|
806
|
+
async function readCodexFinancialFile(file) {
|
|
807
|
+
const handle = await open(file, "r");
|
|
808
|
+
try {
|
|
809
|
+
const fileStat = await handle.stat();
|
|
810
|
+
if (fileStat.size === 0) {
|
|
811
|
+
return {
|
|
812
|
+
hadContent: false,
|
|
813
|
+
malformedLines: 0,
|
|
814
|
+
entries: [],
|
|
815
|
+
bytesSkipped: 0,
|
|
816
|
+
prefilteredLines: 0,
|
|
817
|
+
prefilteredBytes: 0
|
|
818
|
+
};
|
|
819
|
+
}
|
|
820
|
+
const rootEntry = await readFirstJsonlRecord(handle, fileStat.size);
|
|
821
|
+
const rootPayload = rootEntry?.type === "session_meta" && isRecord(rootEntry.payload)
|
|
822
|
+
? rootEntry.payload
|
|
823
|
+
: undefined;
|
|
824
|
+
const isSubagent = Boolean(rootPayload) && (stringOf(rootPayload?.thread_source) === "subagent" ||
|
|
825
|
+
isRecord(rootPayload?.source) && "subagent" in rootPayload.source);
|
|
826
|
+
const entriesReverse = [];
|
|
827
|
+
let malformedLines = 0;
|
|
828
|
+
let prefilteredLines = 0;
|
|
829
|
+
let prefilteredBytes = 0;
|
|
830
|
+
let position = fileStat.size;
|
|
831
|
+
// Chunks for one cross-boundary line are retained newest-first. Appending
|
|
832
|
+
// is O(1); they are ordered only once when the line's start is found.
|
|
833
|
+
let suffixPartsReverse = [];
|
|
834
|
+
let stoppedEarly = false;
|
|
835
|
+
let bytesSkipped = 0;
|
|
836
|
+
const proof = createCodexReverseProof(isSubagent, !rootPayload);
|
|
837
|
+
while (position > 0 && !stoppedEarly) {
|
|
838
|
+
const chunkStart = Math.max(0, position - FINANCIAL_REVERSE_CHUNK_BYTES);
|
|
839
|
+
const length = position - chunkStart;
|
|
840
|
+
const chunk = Buffer.allocUnsafe(length);
|
|
841
|
+
const { bytesRead } = await handle.read(chunk, 0, length, chunkStart);
|
|
842
|
+
if (bytesRead !== length) {
|
|
843
|
+
throw newErrorWithCode("EIO");
|
|
844
|
+
}
|
|
845
|
+
const current = chunk.subarray(0, bytesRead);
|
|
846
|
+
let lineEnd = current.length;
|
|
847
|
+
let newline = current.lastIndexOf(0x0a, lineEnd - 1);
|
|
848
|
+
while (newline >= 0) {
|
|
849
|
+
const leadingPart = current.subarray(newline + 1, lineEnd);
|
|
850
|
+
const parts = suffixPartsReverse.length > 0
|
|
851
|
+
? [leadingPart, ...suffixPartsReverse.slice().reverse()]
|
|
852
|
+
: [leadingPart];
|
|
853
|
+
const observed = parseCodexFinancialLine(parts);
|
|
854
|
+
if (observed.malformed)
|
|
855
|
+
malformedLines += 1;
|
|
856
|
+
if (observed.prefilteredBytes > 0) {
|
|
857
|
+
prefilteredLines += 1;
|
|
858
|
+
prefilteredBytes += observed.prefilteredBytes;
|
|
859
|
+
}
|
|
860
|
+
if (observed.entry) {
|
|
861
|
+
entriesReverse.push(observed.entry);
|
|
862
|
+
if (observeCodexReverseProof(proof, observed.entry)) {
|
|
863
|
+
stoppedEarly = true;
|
|
864
|
+
bytesSkipped = chunkStart + newline;
|
|
865
|
+
break;
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
suffixPartsReverse = [];
|
|
869
|
+
lineEnd = newline;
|
|
870
|
+
newline = current.lastIndexOf(0x0a, lineEnd - 1);
|
|
871
|
+
}
|
|
872
|
+
if (!stoppedEarly) {
|
|
873
|
+
const prefix = current.subarray(0, lineEnd);
|
|
874
|
+
if (prefix.length > 0)
|
|
875
|
+
suffixPartsReverse.push(prefix);
|
|
876
|
+
position = chunkStart;
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
if (!stoppedEarly && suffixPartsReverse.length > 0) {
|
|
880
|
+
const observed = parseCodexFinancialLine(suffixPartsReverse.slice().reverse());
|
|
881
|
+
if (observed.malformed)
|
|
882
|
+
malformedLines += 1;
|
|
883
|
+
if (observed.prefilteredBytes > 0) {
|
|
884
|
+
prefilteredLines += 1;
|
|
885
|
+
prefilteredBytes += observed.prefilteredBytes;
|
|
886
|
+
}
|
|
887
|
+
if (observed.entry)
|
|
888
|
+
entriesReverse.push(observed.entry);
|
|
889
|
+
}
|
|
890
|
+
const entries = entriesReverse.reverse();
|
|
891
|
+
// A proof-complete tail does not contain byte-zero root metadata. Inject
|
|
892
|
+
// the separately parsed first record so session identity/start/cwd remain
|
|
893
|
+
// identical to the full parser. A full scan already contains that record.
|
|
894
|
+
if (stoppedEarly && rootEntry)
|
|
895
|
+
entries.unshift(rootEntry);
|
|
896
|
+
return {
|
|
897
|
+
hadContent: true,
|
|
898
|
+
malformedLines,
|
|
899
|
+
entries,
|
|
900
|
+
bytesSkipped: Math.max(0, bytesSkipped),
|
|
901
|
+
prefilteredLines,
|
|
902
|
+
prefilteredBytes
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
finally {
|
|
906
|
+
await handle.close();
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
async function readFirstJsonlRecord(handle, fileSize) {
|
|
910
|
+
const chunks = [];
|
|
911
|
+
let position = 0;
|
|
912
|
+
while (position < fileSize) {
|
|
913
|
+
const length = Math.min(FINANCIAL_REVERSE_CHUNK_BYTES, fileSize - position);
|
|
914
|
+
const chunk = Buffer.allocUnsafe(length);
|
|
915
|
+
const { bytesRead } = await handle.read(chunk, 0, length, position);
|
|
916
|
+
if (bytesRead <= 0)
|
|
917
|
+
break;
|
|
918
|
+
const value = chunk.subarray(0, bytesRead);
|
|
919
|
+
const newline = value.indexOf(0x0a);
|
|
920
|
+
if (newline >= 0) {
|
|
921
|
+
chunks.push(value.subarray(0, newline));
|
|
922
|
+
break;
|
|
923
|
+
}
|
|
924
|
+
chunks.push(value);
|
|
925
|
+
position += bytesRead;
|
|
926
|
+
}
|
|
927
|
+
if (chunks.length === 0)
|
|
928
|
+
return undefined;
|
|
929
|
+
try {
|
|
930
|
+
const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
931
|
+
return isRecord(parsed) ? parsed : undefined;
|
|
932
|
+
}
|
|
933
|
+
catch {
|
|
934
|
+
return undefined;
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
const CODEX_ENVELOPE_PREFIX_BYTES = 16 * 1024;
|
|
938
|
+
function parseCodexFinancialLine(parts) {
|
|
939
|
+
const byteLength = parts.reduce((total, part) => total + part.length, 0);
|
|
940
|
+
if (byteLength === 0)
|
|
941
|
+
return { malformed: false, prefilteredBytes: 0 };
|
|
942
|
+
const prefix = bufferPartsPrefix(parts, CODEX_ENVELOPE_PREFIX_BYTES).toString("utf8");
|
|
943
|
+
const classification = classifyCodexFinancialEnvelope(prefix);
|
|
944
|
+
if (classification === "nonfinancial") {
|
|
945
|
+
return { malformed: false, prefilteredBytes: byteLength };
|
|
946
|
+
}
|
|
947
|
+
const text = Buffer.concat(parts, byteLength).toString("utf8").trim();
|
|
948
|
+
if (!text)
|
|
949
|
+
return { malformed: false, prefilteredBytes: 0 };
|
|
950
|
+
let parsed;
|
|
951
|
+
try {
|
|
952
|
+
parsed = JSON.parse(text);
|
|
953
|
+
}
|
|
954
|
+
catch {
|
|
955
|
+
return { malformed: true, prefilteredBytes: 0 };
|
|
956
|
+
}
|
|
957
|
+
if (!isRecord(parsed))
|
|
958
|
+
return { malformed: false, prefilteredBytes: 0 };
|
|
959
|
+
const payload = isRecord(parsed.payload) ? parsed.payload : undefined;
|
|
960
|
+
const financial = parsed.type === "session_meta" ||
|
|
961
|
+
parsed.type === "turn_context" ||
|
|
962
|
+
parsed.type === "event_msg" && (payload?.type === "token_count" || payload?.type === "task_started");
|
|
963
|
+
return financial
|
|
964
|
+
? { entry: parsed, malformed: false, prefilteredBytes: 0 }
|
|
965
|
+
: { malformed: false, prefilteredBytes: byteLength };
|
|
966
|
+
}
|
|
967
|
+
function bufferPartsPrefix(parts, limit) {
|
|
968
|
+
const selected = [];
|
|
969
|
+
let remaining = limit;
|
|
970
|
+
for (const part of parts) {
|
|
971
|
+
if (remaining <= 0)
|
|
972
|
+
break;
|
|
973
|
+
const value = part.subarray(0, Math.min(part.length, remaining));
|
|
974
|
+
if (value.length > 0)
|
|
975
|
+
selected.push(value);
|
|
976
|
+
remaining -= value.length;
|
|
977
|
+
}
|
|
978
|
+
return selected.length === 1
|
|
979
|
+
? selected[0]
|
|
980
|
+
: Buffer.concat(selected, limit - remaining);
|
|
981
|
+
}
|
|
982
|
+
function classifyCodexFinancialEnvelope(prefix) {
|
|
983
|
+
// JSON property order is not semantic. Scan string/object depth so a nested
|
|
984
|
+
// payload type can never be mistaken for the top-level event type. If the
|
|
985
|
+
// bounded prefix does not prove the envelope, return unknown and let the
|
|
986
|
+
// complete line go through JSON.parse rather than silently prefiltering it.
|
|
987
|
+
const { topLevelType, payloadType } = scanCodexEnvelopeTypes(prefix);
|
|
988
|
+
if (!topLevelType)
|
|
989
|
+
return "unknown";
|
|
990
|
+
if (topLevelType === "session_meta" || topLevelType === "turn_context") {
|
|
991
|
+
return "financial";
|
|
992
|
+
}
|
|
993
|
+
if (topLevelType !== "event_msg")
|
|
994
|
+
return "nonfinancial";
|
|
995
|
+
if (!payloadType)
|
|
996
|
+
return "unknown";
|
|
997
|
+
return payloadType === "token_count" || payloadType === "task_started"
|
|
998
|
+
? "financial"
|
|
999
|
+
: "nonfinancial";
|
|
1000
|
+
}
|
|
1001
|
+
function scanCodexEnvelopeTypes(prefix) {
|
|
1002
|
+
let objectDepth = 0;
|
|
1003
|
+
let payloadDepth;
|
|
1004
|
+
let topLevelType;
|
|
1005
|
+
let payloadType;
|
|
1006
|
+
for (let index = 0; index < prefix.length; index += 1) {
|
|
1007
|
+
const character = prefix[index];
|
|
1008
|
+
if (character === "{") {
|
|
1009
|
+
objectDepth += 1;
|
|
1010
|
+
continue;
|
|
1011
|
+
}
|
|
1012
|
+
if (character === "}") {
|
|
1013
|
+
if (payloadDepth === objectDepth)
|
|
1014
|
+
payloadDepth = undefined;
|
|
1015
|
+
objectDepth = Math.max(0, objectDepth - 1);
|
|
1016
|
+
continue;
|
|
1017
|
+
}
|
|
1018
|
+
if (character !== '"')
|
|
1019
|
+
continue;
|
|
1020
|
+
const token = readJsonStringToken(prefix, index);
|
|
1021
|
+
if (!token)
|
|
1022
|
+
break;
|
|
1023
|
+
index = token.end;
|
|
1024
|
+
let cursor = skipJsonWhitespace(prefix, token.end + 1);
|
|
1025
|
+
if (prefix[cursor] !== ":")
|
|
1026
|
+
continue;
|
|
1027
|
+
cursor = skipJsonWhitespace(prefix, cursor + 1);
|
|
1028
|
+
if (token.value === "payload" && objectDepth === 1 && prefix[cursor] === "{") {
|
|
1029
|
+
payloadDepth = objectDepth + 1;
|
|
1030
|
+
continue;
|
|
1031
|
+
}
|
|
1032
|
+
if (token.value !== "type" || prefix[cursor] !== '"')
|
|
1033
|
+
continue;
|
|
1034
|
+
const value = readJsonStringToken(prefix, cursor);
|
|
1035
|
+
if (!value)
|
|
1036
|
+
break;
|
|
1037
|
+
if (objectDepth === 1)
|
|
1038
|
+
topLevelType = value.value;
|
|
1039
|
+
if (payloadDepth === objectDepth)
|
|
1040
|
+
payloadType = value.value;
|
|
1041
|
+
index = value.end;
|
|
1042
|
+
}
|
|
1043
|
+
return { topLevelType, payloadType };
|
|
1044
|
+
}
|
|
1045
|
+
function readJsonStringToken(input, start) {
|
|
1046
|
+
if (input[start] !== '"')
|
|
1047
|
+
return undefined;
|
|
1048
|
+
let escaped = false;
|
|
1049
|
+
for (let index = start + 1; index < input.length; index += 1) {
|
|
1050
|
+
const character = input[index];
|
|
1051
|
+
if (escaped) {
|
|
1052
|
+
escaped = false;
|
|
1053
|
+
continue;
|
|
1054
|
+
}
|
|
1055
|
+
if (character === "\\") {
|
|
1056
|
+
escaped = true;
|
|
1057
|
+
continue;
|
|
1058
|
+
}
|
|
1059
|
+
if (character !== '"')
|
|
1060
|
+
continue;
|
|
1061
|
+
try {
|
|
1062
|
+
const value = JSON.parse(input.slice(start, index + 1));
|
|
1063
|
+
return typeof value === "string" ? { value, end: index } : undefined;
|
|
1064
|
+
}
|
|
1065
|
+
catch {
|
|
1066
|
+
return undefined;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
return undefined;
|
|
1070
|
+
}
|
|
1071
|
+
function skipJsonWhitespace(input, start) {
|
|
1072
|
+
let index = start;
|
|
1073
|
+
while (index < input.length && /\s/.test(input[index]))
|
|
1074
|
+
index += 1;
|
|
1075
|
+
return index;
|
|
1076
|
+
}
|
|
1077
|
+
function createCodexReverseProof(isSubagent, forceFullScan) {
|
|
1078
|
+
return {
|
|
1079
|
+
isSubagent,
|
|
1080
|
+
forceFullScan,
|
|
1081
|
+
totalSeen: false,
|
|
1082
|
+
turnSeen: false,
|
|
1083
|
+
rateLimitsSeen: false,
|
|
1084
|
+
modelSeen: false
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
1087
|
+
function observeCodexReverseProof(proof, entry) {
|
|
1088
|
+
if (proof.forceFullScan || proof.isSubagent)
|
|
1089
|
+
return false;
|
|
1090
|
+
const payload = isRecord(entry.payload) ? entry.payload : undefined;
|
|
1091
|
+
const info = payload?.type === "token_count" && isRecord(payload.info)
|
|
1092
|
+
? payload.info
|
|
1093
|
+
: undefined;
|
|
1094
|
+
const hasTotal = Boolean(info && isRecord(info.total_token_usage));
|
|
1095
|
+
if (hasTotal)
|
|
1096
|
+
proof.totalSeen = true;
|
|
1097
|
+
if (info && isRecord(info.last_token_usage))
|
|
1098
|
+
proof.turnSeen = true;
|
|
1099
|
+
if (parseCodexRateLimits(payload?.rate_limits, stringOf(entry.timestamp))) {
|
|
1100
|
+
proof.rateLimitsSeen = true;
|
|
1101
|
+
}
|
|
1102
|
+
if (entry.type === "turn_context" && stringOf(payload?.model)) {
|
|
1103
|
+
proof.modelSeen = true;
|
|
1104
|
+
}
|
|
1105
|
+
return !proof.isSubagent &&
|
|
1106
|
+
proof.totalSeen &&
|
|
1107
|
+
proof.turnSeen &&
|
|
1108
|
+
proof.rateLimitsSeen &&
|
|
1109
|
+
proof.modelSeen;
|
|
1110
|
+
}
|
|
1111
|
+
async function shouldStreamFile(file, sinceMs, agent, scan, diagnostics) {
|
|
1112
|
+
let fileStat;
|
|
1113
|
+
try {
|
|
1114
|
+
fileStat = await lstat(file);
|
|
1115
|
+
}
|
|
1116
|
+
catch (error) {
|
|
1117
|
+
recordUnreadableFile(agent, scan, diagnostics, error);
|
|
1118
|
+
return false;
|
|
1119
|
+
}
|
|
1120
|
+
// Refuse a path swapped to a symlink/non-file after directory discovery.
|
|
1121
|
+
if (!fileStat.isFile()) {
|
|
1122
|
+
recordUnreadableFile(agent, scan, diagnostics, newErrorWithCode("EINVAL"));
|
|
1123
|
+
return false;
|
|
1124
|
+
}
|
|
1125
|
+
if (typeof sinceMs !== "number")
|
|
1126
|
+
return true;
|
|
1127
|
+
// ctime catches a file whose mtime was restored after recent replacement;
|
|
1128
|
+
// birthtime catches a newly copied file with a preserved old mtime. Skip
|
|
1129
|
+
// only when all available metadata proves the file predates the window.
|
|
1130
|
+
const newestFileEvidence = Math.max(fileStat.mtimeMs, fileStat.ctimeMs, fileStat.birthtimeMs);
|
|
1131
|
+
if (!Number.isFinite(newestFileEvidence) || newestFileEvidence >= sinceMs) {
|
|
1132
|
+
return true;
|
|
1133
|
+
}
|
|
1134
|
+
scan.filesSkippedBeforeWindow = (scan.filesSkippedBeforeWindow ?? 0) + 1;
|
|
1135
|
+
return false;
|
|
1136
|
+
}
|
|
1137
|
+
function newErrorWithCode(code) {
|
|
1138
|
+
const error = new Error(code);
|
|
1139
|
+
error.code = code;
|
|
1140
|
+
return error;
|
|
1141
|
+
}
|
|
1142
|
+
function parseClaudeFinancialEntry(entry, filePath, sinceMs, seen, onDiagnostic) {
|
|
1143
|
+
if (entry.type !== "assistant")
|
|
1144
|
+
return undefined;
|
|
1145
|
+
const message = isRecord(entry.message) ? entry.message : undefined;
|
|
1146
|
+
const usage = message && isRecord(message.usage) ? message.usage : undefined;
|
|
1147
|
+
if (!message || !usage || stringOf(message.model) === "<synthetic>")
|
|
1148
|
+
return undefined;
|
|
1149
|
+
const dedupeKey = `${stringOf(message.id) ?? ""}:${stringOf(entry.requestId) ?? ""}`;
|
|
1150
|
+
if (dedupeKey !== ":" && seen.has(dedupeKey))
|
|
1151
|
+
return undefined;
|
|
1152
|
+
seen.add(dedupeKey);
|
|
1153
|
+
const timestamp = toIso(stringOf(entry.timestamp)) ?? new Date(0).toISOString();
|
|
1154
|
+
if (typeof sinceMs === "number" && Date.parse(timestamp) < sinceMs)
|
|
1155
|
+
return undefined;
|
|
1156
|
+
const parsedUsage = parseClaudeFinancialUsage(usage, onDiagnostic);
|
|
1157
|
+
const workingDirectory = absoluteWorkingDirectory(stringOf(entry.cwd));
|
|
1158
|
+
return {
|
|
1159
|
+
agent: "claude-code",
|
|
1160
|
+
model: stringOf(message.model) ?? "claude-code",
|
|
1161
|
+
timestamp,
|
|
1162
|
+
project: projectFromCwd(workingDirectory) ?? projectFromTranscriptPath(filePath),
|
|
1163
|
+
workingDirectory,
|
|
1164
|
+
sessionId: stringOf(entry.sessionId),
|
|
1165
|
+
...(parsedUsage.latestTurnUsage
|
|
1166
|
+
? { latestTurnUsage: parsedUsage.latestTurnUsage }
|
|
1167
|
+
: {}),
|
|
1168
|
+
usageScope: "turn",
|
|
1169
|
+
...(parsedUsage.usageSupport ? { usageSupport: parsedUsage.usageSupport } : {}),
|
|
1170
|
+
...(parsedUsage.reportedTotalTokens !== undefined
|
|
1171
|
+
? { reportedTotalTokens: parsedUsage.reportedTotalTokens }
|
|
1172
|
+
: {}),
|
|
1173
|
+
usage: parsedUsage.usage
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
1176
|
+
function createCodexFinancialStreamState() {
|
|
1177
|
+
return {
|
|
1178
|
+
rootSessionMetaSeen: false,
|
|
1179
|
+
rootTaskStarted: false,
|
|
1180
|
+
isSubagent: false
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
function consumeCodexFinancialEntry(state, entry) {
|
|
1184
|
+
const payload = isRecord(entry.payload) ? entry.payload : undefined;
|
|
1185
|
+
if (entry.type === "session_meta" && payload && !state.rootSessionMetaSeen) {
|
|
1186
|
+
state.rootSessionMetaSeen = true;
|
|
1187
|
+
state.sessionId = stringOf(payload.id);
|
|
1188
|
+
state.rootCwd = stringOf(payload.cwd);
|
|
1189
|
+
state.startedAt = toIso(stringOf(payload.timestamp) ?? stringOf(entry.timestamp));
|
|
1190
|
+
state.rootStartedAtMs = timestampMilliseconds(payload.timestamp ?? entry.timestamp);
|
|
1191
|
+
state.isSubagent = stringOf(payload.thread_source) === "subagent" ||
|
|
1192
|
+
isRecord(payload.source) && "subagent" in payload.source;
|
|
1193
|
+
}
|
|
1194
|
+
if (entry.type === "turn_context" && payload) {
|
|
1195
|
+
state.model = stringOf(payload.model) ?? state.model;
|
|
1196
|
+
state.rootCwd ??= stringOf(payload.cwd);
|
|
1197
|
+
}
|
|
1198
|
+
if (state.isSubagent &&
|
|
1199
|
+
!state.rootTaskStarted &&
|
|
1200
|
+
payload?.type === "task_started" &&
|
|
1201
|
+
isRootSpecificTaskStart(payload.started_at, state.rootStartedAtMs)) {
|
|
1202
|
+
state.inheritedUsageBaseline = state.lastTotal;
|
|
1203
|
+
state.lastTotal = undefined;
|
|
1204
|
+
state.rootTaskStarted = true;
|
|
1205
|
+
state.model = undefined;
|
|
1206
|
+
state.lastTurn = undefined;
|
|
1207
|
+
state.lastRateLimits = undefined;
|
|
1208
|
+
state.lastActivityAt = toIso(stringOf(entry.timestamp)) ?? state.startedAt;
|
|
1209
|
+
}
|
|
1210
|
+
if (entry.type !== "event_msg" || payload?.type !== "token_count")
|
|
1211
|
+
return;
|
|
1212
|
+
const eventTimestamp = toIso(stringOf(entry.timestamp)) ??
|
|
1213
|
+
state.lastActivityAt ??
|
|
1214
|
+
state.startedAt;
|
|
1215
|
+
const info = isRecord(payload.info) ? payload.info : undefined;
|
|
1216
|
+
const total = info && isRecord(info.total_token_usage)
|
|
1217
|
+
? info.total_token_usage
|
|
1218
|
+
: undefined;
|
|
1219
|
+
const turn = info && isRecord(info.last_token_usage)
|
|
1220
|
+
? info.last_token_usage
|
|
1221
|
+
: undefined;
|
|
1222
|
+
if (total) {
|
|
1223
|
+
state.lastTotal = total;
|
|
1224
|
+
state.lastActivityAt = eventTimestamp;
|
|
1225
|
+
}
|
|
1226
|
+
if (turn) {
|
|
1227
|
+
state.lastTurn = turn;
|
|
1228
|
+
state.lastActivityAt = eventTimestamp;
|
|
1229
|
+
}
|
|
1230
|
+
const rateLimits = parseCodexRateLimits(payload.rate_limits, eventTimestamp);
|
|
1231
|
+
if (rateLimits)
|
|
1232
|
+
state.lastRateLimits = rateLimits;
|
|
1233
|
+
}
|
|
1234
|
+
function finishCodexFinancialStream(state, onDiagnostic) {
|
|
1235
|
+
if (!state.lastTotal || state.isSubagent && !state.rootTaskStarted)
|
|
1236
|
+
return undefined;
|
|
1237
|
+
const parsedUsage = parseCodexCumulativeUsage(state.lastTotal, state.inheritedUsageBaseline);
|
|
1238
|
+
const parsedTurn = state.lastTurn
|
|
1239
|
+
? parseCodexTurnUsage(state.lastTurn)
|
|
1240
|
+
: { supported: true };
|
|
1241
|
+
const usageSupport = parsedUsage.supported && parsedTurn.supported
|
|
1242
|
+
? "complete"
|
|
1243
|
+
: "unsupported_token_shape";
|
|
1244
|
+
if (usageSupport === "unsupported_token_shape") {
|
|
1245
|
+
onDiagnostic?.({ code: "unsupported_token_shape", count: 1 });
|
|
1246
|
+
}
|
|
1247
|
+
const workingDirectory = absoluteWorkingDirectory(state.rootCwd);
|
|
1248
|
+
return {
|
|
1249
|
+
agent: "codex",
|
|
1250
|
+
model: state.model ?? "codex",
|
|
1251
|
+
timestamp: state.lastActivityAt ?? state.startedAt ?? new Date(0).toISOString(),
|
|
1252
|
+
startedAt: state.startedAt,
|
|
1253
|
+
project: projectFromCwd(workingDirectory),
|
|
1254
|
+
workingDirectory,
|
|
1255
|
+
sessionId: state.sessionId,
|
|
1256
|
+
rateLimits: state.lastRateLimits,
|
|
1257
|
+
...(usageSupport === "complete" && parsedTurn.usage
|
|
1258
|
+
? { latestTurnUsage: parsedTurn.usage }
|
|
1259
|
+
: {}),
|
|
1260
|
+
usageScope: "session_cumulative",
|
|
1261
|
+
usageSupport,
|
|
1262
|
+
...(parsedUsage.reportedTotalTokens !== undefined
|
|
1263
|
+
? { reportedTotalTokens: parsedUsage.reportedTotalTokens }
|
|
1264
|
+
: {}),
|
|
1265
|
+
usage: parsedUsage.usage
|
|
1266
|
+
};
|
|
1267
|
+
}
|
|
453
1268
|
/** Aggregate per-call usage into one UsageRecord per day+agent+model+project. */
|
|
454
1269
|
export function aggregateCalls(calls) {
|
|
455
1270
|
const groups = new Map();
|
|
@@ -468,8 +1283,9 @@ export function aggregateCalls(calls) {
|
|
|
468
1283
|
cacheWrite5mTokens: sum(groupCalls, (c) => c.usage.cacheWrite5mTokens ?? 0),
|
|
469
1284
|
cacheWrite1hTokens: sum(groupCalls, (c) => c.usage.cacheWrite1hTokens ?? 0)
|
|
470
1285
|
};
|
|
471
|
-
const
|
|
472
|
-
const
|
|
1286
|
+
const usageSupported = groupCalls.every((call) => call.usageSupport !== "unsupported_token_shape");
|
|
1287
|
+
const amountUsd = usageSupported ? estimateTokenCostUsd(model, usage) : undefined;
|
|
1288
|
+
const priced = usageSupported && typeof amountUsd === "number";
|
|
473
1289
|
records.push({
|
|
474
1290
|
id: slug(["local", agent, day, model, project].join("-")),
|
|
475
1291
|
timestamp: new Date(`${day}T00:00:00Z`).toISOString(),
|
|
@@ -498,25 +1314,131 @@ export function aggregateCalls(calls) {
|
|
|
498
1314
|
}
|
|
499
1315
|
return records.sort((left, right) => left.id.localeCompare(right.id));
|
|
500
1316
|
}
|
|
501
|
-
async function listJsonlFiles(root) {
|
|
502
|
-
|
|
503
|
-
|
|
1317
|
+
async function listJsonlFiles(root, scan, diagnostics) {
|
|
1318
|
+
let rootStat;
|
|
1319
|
+
try {
|
|
1320
|
+
rootStat = await stat(root);
|
|
1321
|
+
}
|
|
1322
|
+
catch (error) {
|
|
1323
|
+
if (isNodeError(error, "ENOENT")) {
|
|
1324
|
+
scan.directoryStatus = "missing";
|
|
1325
|
+
diagnostics.push({
|
|
1326
|
+
agent: scan.agent,
|
|
1327
|
+
code: "directory_missing",
|
|
1328
|
+
severity: "info",
|
|
1329
|
+
message: `${agentLabel(scan.agent)} transcript directory was not found.`,
|
|
1330
|
+
count: 1
|
|
1331
|
+
});
|
|
1332
|
+
}
|
|
1333
|
+
else {
|
|
1334
|
+
scan.directoryStatus = "unreadable";
|
|
1335
|
+
diagnostics.push({
|
|
1336
|
+
agent: scan.agent,
|
|
1337
|
+
code: "directory_unreadable",
|
|
1338
|
+
severity: "error",
|
|
1339
|
+
message: `${agentLabel(scan.agent)} transcript directory could not be read${errorCodeSuffix(error)}.`,
|
|
1340
|
+
count: 1
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
504
1343
|
return [];
|
|
1344
|
+
}
|
|
1345
|
+
if (!rootStat.isDirectory()) {
|
|
1346
|
+
scan.directoryStatus = "unreadable";
|
|
1347
|
+
diagnostics.push({
|
|
1348
|
+
agent: scan.agent,
|
|
1349
|
+
code: "directory_unreadable",
|
|
1350
|
+
severity: "error",
|
|
1351
|
+
message: `${agentLabel(scan.agent)} transcript path is not a readable directory.`,
|
|
1352
|
+
count: 1
|
|
1353
|
+
});
|
|
1354
|
+
return [];
|
|
1355
|
+
}
|
|
1356
|
+
scan.directoryStatus = "readable";
|
|
505
1357
|
const out = [];
|
|
506
1358
|
const queue = [root];
|
|
507
1359
|
while (queue.length > 0) {
|
|
508
1360
|
const dir = queue.pop();
|
|
509
|
-
|
|
1361
|
+
let entries;
|
|
1362
|
+
try {
|
|
1363
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
1364
|
+
}
|
|
1365
|
+
catch (error) {
|
|
1366
|
+
scan.directoryStatus = "unreadable";
|
|
1367
|
+
diagnostics.push({
|
|
1368
|
+
agent: scan.agent,
|
|
1369
|
+
code: "directory_unreadable",
|
|
1370
|
+
severity: "error",
|
|
1371
|
+
message: `${agentLabel(scan.agent)} transcript directory could not be read${errorCodeSuffix(error)}.`,
|
|
1372
|
+
count: 1
|
|
1373
|
+
});
|
|
1374
|
+
continue;
|
|
1375
|
+
}
|
|
510
1376
|
for (const entry of entries) {
|
|
511
1377
|
const path = join(dir, entry.name);
|
|
512
1378
|
if (entry.isDirectory())
|
|
513
1379
|
queue.push(path);
|
|
514
|
-
else if (entry.isFile() && entry.name.endsWith(".jsonl"))
|
|
1380
|
+
else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
515
1381
|
out.push(path);
|
|
1382
|
+
scan.filesDiscovered += 1;
|
|
1383
|
+
}
|
|
516
1384
|
}
|
|
517
1385
|
}
|
|
518
1386
|
return out;
|
|
519
1387
|
}
|
|
1388
|
+
function emptySourceScan(agent) {
|
|
1389
|
+
return {
|
|
1390
|
+
agent,
|
|
1391
|
+
directoryStatus: "readable",
|
|
1392
|
+
filesDiscovered: 0,
|
|
1393
|
+
filesParsed: 0,
|
|
1394
|
+
malformedLines: 0,
|
|
1395
|
+
unreadableFiles: 0,
|
|
1396
|
+
unsupportedUsageSnapshots: 0
|
|
1397
|
+
};
|
|
1398
|
+
}
|
|
1399
|
+
function recordUnreadableFile(agent, scan, diagnostics, error) {
|
|
1400
|
+
scan.unreadableFiles += 1;
|
|
1401
|
+
diagnostics.push({
|
|
1402
|
+
agent,
|
|
1403
|
+
code: "file_unreadable",
|
|
1404
|
+
severity: "error",
|
|
1405
|
+
message: `${agentLabel(agent)} transcript file could not be read${errorCodeSuffix(error)}.`,
|
|
1406
|
+
count: 1
|
|
1407
|
+
});
|
|
1408
|
+
}
|
|
1409
|
+
function recordParseDiagnostic(agent, scan, diagnostics, diagnostic) {
|
|
1410
|
+
if (diagnostic.code === "malformed_jsonl") {
|
|
1411
|
+
scan.malformedLines += diagnostic.count;
|
|
1412
|
+
diagnostics.push({
|
|
1413
|
+
agent,
|
|
1414
|
+
code: diagnostic.code,
|
|
1415
|
+
severity: "warning",
|
|
1416
|
+
message: `${diagnostic.count} malformed JSONL line(s) were skipped in ${agentLabel(agent)} transcripts.`,
|
|
1417
|
+
count: diagnostic.count
|
|
1418
|
+
});
|
|
1419
|
+
return;
|
|
1420
|
+
}
|
|
1421
|
+
scan.unsupportedUsageSnapshots += diagnostic.count;
|
|
1422
|
+
diagnostics.push({
|
|
1423
|
+
agent,
|
|
1424
|
+
code: diagnostic.code,
|
|
1425
|
+
severity: "warning",
|
|
1426
|
+
message: `${diagnostic.count} ${agentLabel(agent)} token snapshot(s) lacked the input/output components required for pricing.`,
|
|
1427
|
+
count: diagnostic.count
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
function agentLabel(agent) {
|
|
1431
|
+
return agent === "claude-code" ? "Claude Code" : "Codex";
|
|
1432
|
+
}
|
|
1433
|
+
function errorCodeSuffix(error) {
|
|
1434
|
+
const code = error instanceof Error
|
|
1435
|
+
? error.code
|
|
1436
|
+
: undefined;
|
|
1437
|
+
return code && /^[A-Z0-9_]+$/.test(code) ? ` (${code})` : "";
|
|
1438
|
+
}
|
|
1439
|
+
function isNodeError(error, code) {
|
|
1440
|
+
return error instanceof Error && error.code === code;
|
|
1441
|
+
}
|
|
520
1442
|
function projectFromCwd(cwd) {
|
|
521
1443
|
if (!cwd)
|
|
522
1444
|
return undefined;
|
|
@@ -805,19 +1727,6 @@ function topicTokens(value) {
|
|
|
805
1727
|
.replace(/\b(?:attached|attachment|clipboard|image|images|photo|picture|screenshot|screenshots)\b/gi, " ");
|
|
806
1728
|
return promptTokens(sanitizeLocalActivityText(withoutAbsolutePaths));
|
|
807
1729
|
}
|
|
808
|
-
function codexTurnUsage(value) {
|
|
809
|
-
const rawInput = numberOf(value.input_tokens) ?? 0;
|
|
810
|
-
const cached = Math.min(rawInput, numberOf(value.cached_input_tokens) ?? 0);
|
|
811
|
-
const output = numberOf(value.output_tokens) ?? 0;
|
|
812
|
-
return {
|
|
813
|
-
inputTokens: Math.max(0, rawInput - cached),
|
|
814
|
-
outputTokens: output,
|
|
815
|
-
cacheReadTokens: cached,
|
|
816
|
-
contextTokens: rawInput,
|
|
817
|
-
totalTokens: numberOf(value.total_tokens) ?? rawInput + output,
|
|
818
|
-
source: "transcript_last_token_usage"
|
|
819
|
-
};
|
|
820
|
-
}
|
|
821
1730
|
function toTurnUsage(usage, source) {
|
|
822
1731
|
const contextTokens = usage.inputTokens +
|
|
823
1732
|
(usage.cacheReadTokens ?? 0) +
|
|
@@ -932,6 +1841,10 @@ function sum(calls, pick) {
|
|
|
932
1841
|
function numberOf(value) {
|
|
933
1842
|
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
934
1843
|
}
|
|
1844
|
+
function tokenComponentOf(value) {
|
|
1845
|
+
const parsed = numberOf(value);
|
|
1846
|
+
return parsed !== undefined && parsed >= 0 ? parsed : undefined;
|
|
1847
|
+
}
|
|
935
1848
|
function stringOf(value) {
|
|
936
1849
|
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
937
1850
|
}
|