@dzhechkov/harness-core 0.3.146 → 0.3.148
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +66 -18
- package/README.md +2 -0
- package/dist/cost-ledger.d.ts +318 -0
- package/dist/cost-ledger.d.ts.map +1 -0
- package/dist/cost-ledger.js +871 -0
- package/dist/cost-ledger.js.map +1 -0
- package/dist/cost-scoring.d.ts +9 -0
- package/dist/cost-scoring.d.ts.map +1 -1
- package/dist/cost-scoring.js +18 -0
- package/dist/cost-scoring.js.map +1 -1
- package/dist/feature-adr-checkpoints.d.ts +113 -0
- package/dist/feature-adr-checkpoints.d.ts.map +1 -0
- package/dist/feature-adr-checkpoints.js +184 -0
- package/dist/feature-adr-checkpoints.js.map +1 -0
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/usage.d.ts +32 -0
- package/dist/usage.d.ts.map +1 -1
- package/dist/usage.js +59 -10
- package/dist/usage.js.map +1 -1
- package/package.json +4 -4
- package/sbom.json +137 -17
- package/src/cost-ledger.ts +1105 -0
- package/src/cost-scoring.ts +17 -0
- package/src/feature-adr-checkpoints.ts +221 -0
- package/src/index.ts +55 -0
- package/src/usage.ts +74 -12
|
@@ -0,0 +1,1105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-stage cost ledger with a reconciliation invariant for feature-adr runs
|
|
3
|
+
* (feature `cost-ledger`, ADR-001/ADR-002/ADR-003).
|
|
4
|
+
*
|
|
5
|
+
* A feature-adr run reports ONE number. The recorded run `wf_0576bd7d-797` has
|
|
6
|
+
* `totalTokens: 623290` — the "623k subagent tokens" figure in project memory. That number cannot
|
|
7
|
+
* be attributed to a stage, so "where the budget burns" is a feeling. feature-adr ALREADY labels
|
|
8
|
+
* every stage via `stageLabel()` and the harness ALREADY persists those labels next to per-agent
|
|
9
|
+
* transcripts; nothing joined labels to spend. This module is that join.
|
|
10
|
+
*
|
|
11
|
+
* ## What this is
|
|
12
|
+
*
|
|
13
|
+
* A POST-HOC DERIVER (ADR-001). It reads what is already on disk —
|
|
14
|
+
* `<session>/workflows/wf_<runId>.json` for the stage labels and
|
|
15
|
+
* `<session>/subagents/workflows/<runId>/agent-<agentId>.jsonl` for the spend — and never edits
|
|
16
|
+
* `.claude/workflows/feature-adr.js`. A killed run is still derivable, which a stage-boundary
|
|
17
|
+
* writer could not manage; 5 of 29 recorded runs on this machine are killed.
|
|
18
|
+
*
|
|
19
|
+
* ## The invariant (ADR-002 — the load-bearing half)
|
|
20
|
+
*
|
|
21
|
+
* The obvious run total, the record's own `totalTokens`, is EXACTLY `Σ workflowProgress[].tokens`
|
|
22
|
+
* in 29 of 29 recorded runs. Reconciling against it can never fail: a vacuous gate that would print
|
|
23
|
+
* BALANCED forever and be believed. So the right-hand side comes from the run's transcript
|
|
24
|
+
* DIRECTORY LISTING — a source independent of the record — and both sides run the SAME estimator
|
|
25
|
+
* (`weightedTokensOf`, shared with `dz usage`):
|
|
26
|
+
*
|
|
27
|
+
* ```
|
|
28
|
+
* accountedTokens + unaccountedTokens === runTotalTokens
|
|
29
|
+
* accountedTokens + doubleAttributedTokens === stageTokensSum
|
|
30
|
+
* ```
|
|
31
|
+
*
|
|
32
|
+
* Raw integer equality, no epsilon: rounding happens exactly once, per sample, at extraction.
|
|
33
|
+
* {@link verifyCostLedgerReport} re-derives both identities from the emitted report — the writer
|
|
34
|
+
* clamps, the verifier enforces raw equality (the `event-chain.ts` house pattern). A mismatch is a
|
|
35
|
+
* NAMED defect from {@link COST_LEDGER_DEFECT_KINDS}, never a rounding remainder.
|
|
36
|
+
*
|
|
37
|
+
* ## What this is NOT — read {@link COST_LEDGER_SCOPE} before describing it to anyone
|
|
38
|
+
*
|
|
39
|
+
* The totals are LOCAL TRANSCRIPT ESTIMATES. No billing API is consulted. The invariant therefore
|
|
40
|
+
* catches ATTRIBUTION errors — a double-counted stage, a stage missing from the ledger — and says
|
|
41
|
+
* NOTHING about whether the prices are right. The USD column is a secondary figure derived from a
|
|
42
|
+
* static table that has no `claude-fable` entry, so it falls back to sonnet-class pricing for the
|
|
43
|
+
* default model of every recorded run; the fallback is REPORTED, per ADR-003, not hidden.
|
|
44
|
+
*
|
|
45
|
+
* The ADR-158 reference implementation this feature is grounded in quotes a ~50.5% figure. That
|
|
46
|
+
* number is SYNTHETIC, belongs to their document, and is never a measurement of this repo.
|
|
47
|
+
*
|
|
48
|
+
* @packageDocumentation
|
|
49
|
+
*/
|
|
50
|
+
|
|
51
|
+
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
52
|
+
import { dirname, join } from 'node:path';
|
|
53
|
+
|
|
54
|
+
import { hasKnownPricing, usageCost } from './cost-scoring.js';
|
|
55
|
+
import { claudeProjectsRoot, rawTokenMixOf, weightedTokensOf } from './usage.js';
|
|
56
|
+
|
|
57
|
+
// ── Scope + vocabulary ──────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
/** The one sentence that states what the ledger is and is not. Printed by EVERY surface (ADR-003). */
|
|
60
|
+
export const COST_LEDGER_SCOPE =
|
|
61
|
+
'local transcript ESTIMATES, not billed amounts — the reconciliation invariant catches ATTRIBUTION ' +
|
|
62
|
+
'errors (double-counted or missing stages), NOT pricing errors';
|
|
63
|
+
|
|
64
|
+
export type CostLedgerDefectKind =
|
|
65
|
+
/** Run spend attributed to no stage — an agent transcript with no `workflowProgress[]` entry. */
|
|
66
|
+
| 'Unaccounted'
|
|
67
|
+
/** One usage sample claimed by more than one stage. */
|
|
68
|
+
| 'DoubleAttributed'
|
|
69
|
+
/** A stage claims a sample absent from the run's universe — the join went outside the run. */
|
|
70
|
+
| 'ForeignSample'
|
|
71
|
+
/** A stage present in the run record has no transcript, or a transcript with no usage samples. */
|
|
72
|
+
| 'MissingStageTranscript'
|
|
73
|
+
/** A record or sample whose fields are missing, mistyped or non-finite — never silently ignored. */
|
|
74
|
+
| 'MalformedRecord'
|
|
75
|
+
/** The transcript listing hit the file cap — the run total is INCOMPLETE, so no verdict may be
|
|
76
|
+
* built on it (Codex QE HIGH: a silent cap could emit BALANCED from a partial directory). */
|
|
77
|
+
| 'TruncatedListing';
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* The defect vocabulary, as data. Deliberately absent: any name implying these are BILLED amounts —
|
|
81
|
+
* that name would assert exactly the promise {@link COST_LEDGER_SCOPE} refuses. A test pins this
|
|
82
|
+
* list so the vocabulary cannot quietly grow such a name.
|
|
83
|
+
*/
|
|
84
|
+
export const COST_LEDGER_DEFECT_KINDS: readonly CostLedgerDefectKind[] = [
|
|
85
|
+
'Unaccounted',
|
|
86
|
+
'DoubleAttributed',
|
|
87
|
+
'ForeignSample',
|
|
88
|
+
'MissingStageTranscript',
|
|
89
|
+
'MalformedRecord',
|
|
90
|
+
'TruncatedListing',
|
|
91
|
+
];
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Three values, not two. `INSUFFICIENT_DATA` is NOT success: a caller must not read
|
|
95
|
+
* `verdict !== 'DEFECT'` as "reconciled" (ADR-003).
|
|
96
|
+
*/
|
|
97
|
+
export type CostLedgerVerdict = 'BALANCED' | 'DEFECT' | 'INSUFFICIENT_DATA';
|
|
98
|
+
|
|
99
|
+
export const COST_LEDGER_VERDICTS: readonly CostLedgerVerdict[] = [
|
|
100
|
+
'BALANCED',
|
|
101
|
+
'DEFECT',
|
|
102
|
+
'INSUFFICIENT_DATA',
|
|
103
|
+
];
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Default reconciliation tolerance, as a FRACTION of the run total. Zero, because the arithmetic is
|
|
107
|
+
* exact integer — there is no rounding remainder for a tolerance to absorb, so any remainder is a
|
|
108
|
+
* defect. A caller may raise it to tolerate small orphans; its value is always printed.
|
|
109
|
+
*/
|
|
110
|
+
export const DEFAULT_COST_LEDGER_EPSILON = 0;
|
|
111
|
+
|
|
112
|
+
/** Guard against a pathological run directory degrading into a hang. */
|
|
113
|
+
const MAX_RUN_TRANSCRIPT_FILES = 2_000;
|
|
114
|
+
|
|
115
|
+
/** `--run` / `--slug` become path segments; only these shapes are ever joined onto a root. */
|
|
116
|
+
const RUN_ID_PATTERN = /^[A-Za-z0-9_.-]{1,128}$/;
|
|
117
|
+
const SLUG_PATTERN = /^[A-Za-z0-9_.-]{1,128}$/;
|
|
118
|
+
|
|
119
|
+
// ── Types ───────────────────────────────────────────────────
|
|
120
|
+
|
|
121
|
+
/** One deduped usage sample extracted from a transcript. */
|
|
122
|
+
export interface CostLedgerSample {
|
|
123
|
+
/** Dedup key: `message.id + ':' + requestId`, or a content key when both are absent. */
|
|
124
|
+
readonly key: string;
|
|
125
|
+
/** Epoch ms, or `null` when the record carried no parseable timestamp. */
|
|
126
|
+
readonly ts: number | null;
|
|
127
|
+
/** Cost-weighted input-equivalent tokens, ROUNDED — the single rounding point of the feature. */
|
|
128
|
+
readonly weighted: number;
|
|
129
|
+
readonly input: number;
|
|
130
|
+
readonly cacheWrite: number;
|
|
131
|
+
readonly cacheRead: number;
|
|
132
|
+
readonly output: number;
|
|
133
|
+
readonly model: string | null;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** One `type: "workflow_agent"` entry of a run record, after clamping. */
|
|
137
|
+
export interface WorkflowStageEntry {
|
|
138
|
+
/** `stageLabel()` output, VERBATIM — the ledger invents no taxonomy (FR-2). */
|
|
139
|
+
readonly label: string;
|
|
140
|
+
readonly agentId: string;
|
|
141
|
+
readonly model: string;
|
|
142
|
+
readonly phase: string | null;
|
|
143
|
+
readonly startedAtMs: number | null;
|
|
144
|
+
readonly durationMs: number | null;
|
|
145
|
+
readonly state: string | null;
|
|
146
|
+
/** The record's own per-agent token count. Reported for traceability; NOT the invariant's input. */
|
|
147
|
+
readonly recordTokens: number | null;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** A parsed `wf_<runId>.json` workflow run record. */
|
|
151
|
+
export interface WorkflowRunRecord {
|
|
152
|
+
readonly runId: string;
|
|
153
|
+
readonly workflowName: string | null;
|
|
154
|
+
readonly slug: string | null;
|
|
155
|
+
readonly status: string | null;
|
|
156
|
+
readonly startedAtMs: number | null;
|
|
157
|
+
readonly durationMs: number | null;
|
|
158
|
+
/** The record's cached `Σ workflowProgress[].tokens` — RAW, unweighted, and NOT the run total. */
|
|
159
|
+
readonly recordTotalTokens: number | null;
|
|
160
|
+
readonly stages: readonly WorkflowStageEntry[];
|
|
161
|
+
/** Non-fatal problems found while parsing — surfaced as `MalformedRecord` defects. */
|
|
162
|
+
readonly malformed: readonly string[];
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** One ledger row: a stage's aggregated spend (FR-1). */
|
|
166
|
+
export interface CostLedgerRow {
|
|
167
|
+
readonly runId: string;
|
|
168
|
+
readonly slug: string | null;
|
|
169
|
+
/** `stageLabel()` output, verbatim. */
|
|
170
|
+
readonly stage: string;
|
|
171
|
+
readonly phase: string | null;
|
|
172
|
+
/** The stage's model id, or `'mixed'` when several agents share a label with different models. */
|
|
173
|
+
readonly model: string;
|
|
174
|
+
readonly agentIds: readonly string[];
|
|
175
|
+
readonly tokensIn: number;
|
|
176
|
+
readonly tokensCacheWrite: number;
|
|
177
|
+
readonly tokensCacheRead: number;
|
|
178
|
+
readonly tokensOut: number;
|
|
179
|
+
/** Cost-weighted input-equivalent tokens — the PRIMARY number of the row. */
|
|
180
|
+
readonly weightedTokens: number;
|
|
181
|
+
/** Secondary, derived estimate. `pricingKnown === false` ⇒ sonnet-class fallback pricing. */
|
|
182
|
+
readonly costUsd: number;
|
|
183
|
+
readonly pricingKnown: boolean;
|
|
184
|
+
/** ISO, from the run record's stage boundaries (ADR-001) — `null` when the record lacked them. */
|
|
185
|
+
readonly startedTs: string | null;
|
|
186
|
+
readonly endedTs: string | null;
|
|
187
|
+
/** Number of deduped usage samples (billed calls) attributed to this stage. */
|
|
188
|
+
readonly calls: number;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export interface CostLedgerDefect {
|
|
192
|
+
readonly kind: CostLedgerDefectKind;
|
|
193
|
+
readonly detail: string;
|
|
194
|
+
/** Weighted tokens implicated, when the defect is quantitative. */
|
|
195
|
+
readonly tokens?: number;
|
|
196
|
+
/** Stage labels or agent ids implicated, when the defect is locatable. */
|
|
197
|
+
readonly subjects?: readonly string[];
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export interface CostLedgerReconciliation {
|
|
201
|
+
/** RIGHT side — dedup-union over the run's transcript DIRECTORY (independent of the record). */
|
|
202
|
+
readonly runTotalTokens: number;
|
|
203
|
+
/** Dedup-union of samples claimed by at least one stage. */
|
|
204
|
+
readonly accountedTokens: number;
|
|
205
|
+
/** Σ of the per-stage sums. Exceeds `accountedTokens` exactly when a sample is double-claimed. */
|
|
206
|
+
readonly stageTokensSum: number;
|
|
207
|
+
/** `runTotalTokens - accountedTokens`. */
|
|
208
|
+
readonly unaccountedTokens: number;
|
|
209
|
+
/** `stageTokensSum - accountedTokens`. */
|
|
210
|
+
readonly doubleAttributedTokens: number;
|
|
211
|
+
/** Tolerance as a FRACTION of the run total; `0` by default (ADR-002). */
|
|
212
|
+
readonly epsilon: number;
|
|
213
|
+
/** Both raw integer identities held when the report was built. */
|
|
214
|
+
readonly identityHolds: boolean;
|
|
215
|
+
readonly verdict: CostLedgerVerdict;
|
|
216
|
+
readonly defects: readonly CostLedgerDefect[];
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export interface CostLedgerReport {
|
|
220
|
+
readonly runId: string;
|
|
221
|
+
readonly slug: string | null;
|
|
222
|
+
readonly workflowName: string | null;
|
|
223
|
+
readonly status: string | null;
|
|
224
|
+
readonly startedTs: string | null;
|
|
225
|
+
readonly rows: readonly CostLedgerRow[];
|
|
226
|
+
readonly reconciliation: CostLedgerReconciliation;
|
|
227
|
+
/** The record's cached raw sum — reported, never the invariant's right-hand side (ADR-002). */
|
|
228
|
+
readonly recordTotalTokens: number | null;
|
|
229
|
+
readonly totalCostUsd: number;
|
|
230
|
+
/** Model ids whose USD figures used sonnet-class fallback pricing (ADR-003). */
|
|
231
|
+
readonly pricingFallbackModels: readonly string[];
|
|
232
|
+
/** ALWAYS `true` — a local aggregation, not an official API (mirrors `dz usage`). */
|
|
233
|
+
readonly estimated: true;
|
|
234
|
+
/** ALWAYS {@link COST_LEDGER_SCOPE}. */
|
|
235
|
+
readonly scope: string;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** FR-8 feed-forward aggregate. WIRING INTO auto-cost ROUTING IS OUT OF SCOPE — this is a reader. */
|
|
239
|
+
export interface StageCostAggregate {
|
|
240
|
+
readonly stage: string;
|
|
241
|
+
readonly model: string;
|
|
242
|
+
readonly avgTokens: number;
|
|
243
|
+
readonly runs: number;
|
|
244
|
+
readonly totalTokens: number;
|
|
245
|
+
readonly avgCostUsd: number;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// ── Small clamped helpers ───────────────────────────────────
|
|
249
|
+
|
|
250
|
+
function finiteNonNegative(v: unknown): number | null {
|
|
251
|
+
return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : null;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function nonEmptyString(v: unknown): string | null {
|
|
255
|
+
return typeof v === 'string' && v.length > 0 ? v : null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
259
|
+
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function isoOrNull(ms: number | null): string | null {
|
|
263
|
+
if (ms === null || !Number.isFinite(ms) || Math.abs(ms) > 8.64e15) return null;
|
|
264
|
+
try {
|
|
265
|
+
return new Date(ms).toISOString();
|
|
266
|
+
} catch {
|
|
267
|
+
return null;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ── PURE: transcript sample extraction ──────────────────────
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Extract deduped, weighted usage samples from ONE transcript's text. Pure and never-throw — a
|
|
275
|
+
* corrupt line is skipped, exactly as `computeUsage` does.
|
|
276
|
+
*
|
|
277
|
+
* `weighted` is `Math.round(weightedTokensOf(...))`: the SINGLE rounding point of the feature, so
|
|
278
|
+
* every sum downstream is exact integer arithmetic and the reconciliation identity is raw equality
|
|
279
|
+
* rather than a float comparison (ADR-002).
|
|
280
|
+
*/
|
|
281
|
+
export function extractCostSamples(text: string): CostLedgerSample[] {
|
|
282
|
+
const out: CostLedgerSample[] = [];
|
|
283
|
+
if (typeof text !== 'string' || text.length === 0) return out;
|
|
284
|
+
const seen = new Set<string>();
|
|
285
|
+
for (const line of text.split('\n')) {
|
|
286
|
+
if (line.length === 0) continue;
|
|
287
|
+
if (line.indexOf('usage') === -1) continue; // cheap pre-filter before the parse
|
|
288
|
+
let rec: unknown;
|
|
289
|
+
try {
|
|
290
|
+
rec = JSON.parse(line);
|
|
291
|
+
} catch {
|
|
292
|
+
continue; // corrupt line — skip, never throw
|
|
293
|
+
}
|
|
294
|
+
if (!isRecord(rec)) continue;
|
|
295
|
+
const message: Record<string, unknown> = isRecord(rec['message']) ? rec['message'] : {};
|
|
296
|
+
const usage = isRecord(message['usage']) ? message['usage'] : null;
|
|
297
|
+
if (usage === null) continue;
|
|
298
|
+
|
|
299
|
+
const weighted = Math.round(weightedTokensOf(usage));
|
|
300
|
+
if (!Number.isFinite(weighted) || weighted <= 0) continue;
|
|
301
|
+
const mix = rawTokenMixOf(usage);
|
|
302
|
+
|
|
303
|
+
const tsRaw = rec['timestamp'];
|
|
304
|
+
let ts: number | null = null;
|
|
305
|
+
if (typeof tsRaw === 'number' && Number.isFinite(tsRaw)) ts = tsRaw;
|
|
306
|
+
else if (typeof tsRaw === 'string') {
|
|
307
|
+
const parsed = Date.parse(tsRaw);
|
|
308
|
+
ts = Number.isFinite(parsed) ? parsed : null;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
const model = nonEmptyString(message['model']) ?? nonEmptyString(rec['model']);
|
|
312
|
+
const id = nonEmptyString(message['id']) ?? '';
|
|
313
|
+
const reqId = nonEmptyString(rec['requestId']) ?? '';
|
|
314
|
+
// With no ids, fall back to a CONTENT key including the raw vector + model: `{input:50}` and
|
|
315
|
+
// `{output:10}` both weigh 50, so a total-only key would silently merge distinct records.
|
|
316
|
+
// The WEIGHTED value is part of the anon key (Codex QE MED): two calls with identical raw
|
|
317
|
+
// totals but different cache-TTL classes weigh differently (125 vs 200) — a key blind to the
|
|
318
|
+
// weight would merge them and the ledger could stay BALANCED with a call missing.
|
|
319
|
+
const key =
|
|
320
|
+
id !== '' || reqId !== ''
|
|
321
|
+
? id + ':' + reqId
|
|
322
|
+
: `anon:${String(ts)}:${mix.input}:${mix.cacheWrite}:${mix.cacheRead}:${mix.output}:${model ?? ''}:${weighted}`;
|
|
323
|
+
if (seen.has(key)) continue;
|
|
324
|
+
seen.add(key);
|
|
325
|
+
out.push({ key, ts, weighted, ...mix, model });
|
|
326
|
+
}
|
|
327
|
+
return out;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// ── PURE: run-record parsing ────────────────────────────────
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Parse a `wf_<runId>.json` object into a {@link WorkflowRunRecord}. Pure and never-throw; every
|
|
334
|
+
* number is clamped and every unusable field is RECORDED in `malformed` rather than dropped, so it
|
|
335
|
+
* can surface as a `MalformedRecord` defect (the vocabulary refuses silent ignores).
|
|
336
|
+
*
|
|
337
|
+
* `args` is stored as a JSON STRING in the recorded runs on this machine and as an object in
|
|
338
|
+
* others; both shapes are accepted.
|
|
339
|
+
*/
|
|
340
|
+
export function parseWorkflowRunRecord(raw: unknown): WorkflowRunRecord | null {
|
|
341
|
+
if (!isRecord(raw)) return null;
|
|
342
|
+
const runId = nonEmptyString(raw['runId']);
|
|
343
|
+
if (runId === null) return null;
|
|
344
|
+
const malformed: string[] = [];
|
|
345
|
+
|
|
346
|
+
let slug: string | null = null;
|
|
347
|
+
const args = raw['args'];
|
|
348
|
+
if (isRecord(args)) {
|
|
349
|
+
slug = nonEmptyString(args['slug']);
|
|
350
|
+
} else if (typeof args === 'string' && args.length > 0) {
|
|
351
|
+
try {
|
|
352
|
+
const parsed: unknown = JSON.parse(args);
|
|
353
|
+
if (isRecord(parsed)) slug = nonEmptyString(parsed['slug']);
|
|
354
|
+
} catch {
|
|
355
|
+
const m = /"slug"\s*:\s*"([^"]+)"/.exec(args);
|
|
356
|
+
slug = m ? (m[1] ?? null) : null;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const stages: WorkflowStageEntry[] = [];
|
|
361
|
+
const progress = raw['workflowProgress'];
|
|
362
|
+
if (progress !== undefined && !Array.isArray(progress)) {
|
|
363
|
+
malformed.push('workflowProgress is not an array');
|
|
364
|
+
}
|
|
365
|
+
if (Array.isArray(progress)) {
|
|
366
|
+
for (let i = 0; i < progress.length; i += 1) {
|
|
367
|
+
const e: unknown = progress[i];
|
|
368
|
+
if (!isRecord(e) || e['type'] !== 'workflow_agent') continue;
|
|
369
|
+
const label = nonEmptyString(e['label']);
|
|
370
|
+
const agentId = nonEmptyString(e['agentId']);
|
|
371
|
+
if (label === null || agentId === null) {
|
|
372
|
+
malformed.push(`workflowProgress[${i}]: workflow_agent without ${label === null ? 'label' : 'agentId'}`);
|
|
373
|
+
continue;
|
|
374
|
+
}
|
|
375
|
+
stages.push({
|
|
376
|
+
label,
|
|
377
|
+
agentId,
|
|
378
|
+
model: nonEmptyString(e['model']) ?? 'unknown',
|
|
379
|
+
phase: nonEmptyString(e['phaseTitle']),
|
|
380
|
+
startedAtMs: finiteNonNegative(e['startedAt']),
|
|
381
|
+
durationMs: finiteNonNegative(e['durationMs']),
|
|
382
|
+
state: nonEmptyString(e['state']),
|
|
383
|
+
recordTokens: finiteNonNegative(e['tokens']),
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
return {
|
|
389
|
+
runId,
|
|
390
|
+
workflowName: nonEmptyString(raw['workflowName']),
|
|
391
|
+
slug,
|
|
392
|
+
status: nonEmptyString(raw['status']),
|
|
393
|
+
startedAtMs: finiteNonNegative(raw['startTime']),
|
|
394
|
+
durationMs: finiteNonNegative(raw['durationMs']),
|
|
395
|
+
recordTotalTokens: finiteNonNegative(raw['totalTokens']),
|
|
396
|
+
stages,
|
|
397
|
+
malformed,
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// ── PURE: the ledger + the invariant ────────────────────────
|
|
402
|
+
|
|
403
|
+
/** Per-stage sample sets, keyed by the `stageLabel()` string. */
|
|
404
|
+
export interface StageSampleSet {
|
|
405
|
+
readonly stage: string;
|
|
406
|
+
readonly samples: readonly CostLedgerSample[];
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
export interface BuildCostLedgerInput {
|
|
410
|
+
readonly record: WorkflowRunRecord;
|
|
411
|
+
/** LEFT side — one entry per `agentId` that had a transcript. */
|
|
412
|
+
readonly stageSamples: readonly { readonly agentId: string; readonly samples: readonly CostLedgerSample[] }[];
|
|
413
|
+
/** RIGHT side — the dedup-union over the run's transcript DIRECTORY (ADR-002). */
|
|
414
|
+
readonly runSamples: readonly CostLedgerSample[];
|
|
415
|
+
/** Agent transcripts present in the run directory with no `workflowProgress[]` entry. */
|
|
416
|
+
readonly orphanAgentIds?: readonly string[];
|
|
417
|
+
/** Fraction of the run total tolerated as unaccounted. Default {@link DEFAULT_COST_LEDGER_EPSILON}. */
|
|
418
|
+
readonly epsilon?: number;
|
|
419
|
+
/** True when the transcript listing hit the file cap — the run total is incomplete (Codex QE HIGH). */
|
|
420
|
+
readonly transcriptListingTruncated?: boolean;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Build the report and evaluate the invariant. PURE — no filesystem, no clock. Every number that
|
|
425
|
+
* enters is clamped here (the writer clamps; {@link verifyCostLedgerReport} enforces raw equality).
|
|
426
|
+
*/
|
|
427
|
+
export function buildCostLedger(input: BuildCostLedgerInput): CostLedgerReport {
|
|
428
|
+
const { record } = input;
|
|
429
|
+
const epsilonRaw = input.epsilon;
|
|
430
|
+
const epsilon =
|
|
431
|
+
typeof epsilonRaw === 'number' && Number.isFinite(epsilonRaw) && epsilonRaw >= 0 && epsilonRaw <= 1
|
|
432
|
+
? epsilonRaw
|
|
433
|
+
: DEFAULT_COST_LEDGER_EPSILON;
|
|
434
|
+
|
|
435
|
+
const defects: CostLedgerDefect[] = [];
|
|
436
|
+
for (const m of record.malformed) defects.push({ kind: 'MalformedRecord', detail: m });
|
|
437
|
+
|
|
438
|
+
// A capped listing means the right-hand side is PARTIAL — BALANCED must be impossible on it.
|
|
439
|
+
if (input.transcriptListingTruncated === true) {
|
|
440
|
+
defects.push({
|
|
441
|
+
kind: 'TruncatedListing',
|
|
442
|
+
detail: `transcript listing hit the ${MAX_RUN_TRANSCRIPT_FILES}-file cap — the run total is incomplete, no verdict may rest on it`,
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// Every sample number is CLAMPED here (Codex QE MED): the contract says the writer clamps, and a
|
|
447
|
+
// negative/non-finite `weighted` sliding through would make negative totals read BALANCED.
|
|
448
|
+
const clampSample = (s: CostLedgerSample): CostLedgerSample => {
|
|
449
|
+
const n = (v: number): number => (Number.isFinite(v) && v >= 0 ? Math.floor(v) : 0);
|
|
450
|
+
return { ...s, weighted: n(s.weighted), input: n(s.input), cacheWrite: n(s.cacheWrite), cacheRead: n(s.cacheRead), output: n(s.output) };
|
|
451
|
+
};
|
|
452
|
+
input = {
|
|
453
|
+
...input,
|
|
454
|
+
runSamples: input.runSamples.map(clampSample),
|
|
455
|
+
stageSamples: input.stageSamples.map((e) => ({ agentId: e.agentId, samples: e.samples.map(clampSample) })),
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
// RIGHT — the run's universe, deduped by sample key.
|
|
459
|
+
const universe = new Map<string, CostLedgerSample>();
|
|
460
|
+
for (const s of input.runSamples) if (!universe.has(s.key)) universe.set(s.key, s);
|
|
461
|
+
let runTotalTokens = 0;
|
|
462
|
+
for (const s of universe.values()) runTotalTokens += s.weighted;
|
|
463
|
+
|
|
464
|
+
// LEFT — per stage, joined agentId → label. Several agents may share one label.
|
|
465
|
+
const byAgent = new Map<string, readonly CostLedgerSample[]>();
|
|
466
|
+
for (const e of input.stageSamples) if (!byAgent.has(e.agentId)) byAgent.set(e.agentId, e.samples);
|
|
467
|
+
|
|
468
|
+
interface Bucket {
|
|
469
|
+
stage: string;
|
|
470
|
+
phase: string | null;
|
|
471
|
+
models: Set<string>;
|
|
472
|
+
agentIds: string[];
|
|
473
|
+
/** Every CLAIM this bucket made, in claim order — NOT deduped. */
|
|
474
|
+
claims: CostLedgerSample[];
|
|
475
|
+
sum: number;
|
|
476
|
+
startedAtMs: number | null;
|
|
477
|
+
endedAtMs: number | null;
|
|
478
|
+
costUsd: number;
|
|
479
|
+
pricingKnown: boolean;
|
|
480
|
+
}
|
|
481
|
+
const buckets = new Map<string, Bucket>();
|
|
482
|
+
const keyOwners = new Map<string, Set<string>>();
|
|
483
|
+
const foreign: string[] = [];
|
|
484
|
+
const conflicting: string[] = [];
|
|
485
|
+
const missingTranscript: string[] = [];
|
|
486
|
+
let stageTokensSum = 0;
|
|
487
|
+
|
|
488
|
+
for (const stage of record.stages) {
|
|
489
|
+
const samples = byAgent.get(stage.agentId) ?? [];
|
|
490
|
+
if (samples.length === 0) missingTranscript.push(`${stage.label} (${stage.agentId})`);
|
|
491
|
+
|
|
492
|
+
let b = buckets.get(stage.label);
|
|
493
|
+
if (b === undefined) {
|
|
494
|
+
b = {
|
|
495
|
+
stage: stage.label,
|
|
496
|
+
phase: stage.phase,
|
|
497
|
+
models: new Set<string>(),
|
|
498
|
+
agentIds: [],
|
|
499
|
+
claims: [],
|
|
500
|
+
sum: 0,
|
|
501
|
+
startedAtMs: null,
|
|
502
|
+
endedAtMs: null,
|
|
503
|
+
costUsd: 0,
|
|
504
|
+
pricingKnown: true,
|
|
505
|
+
};
|
|
506
|
+
buckets.set(stage.label, b);
|
|
507
|
+
}
|
|
508
|
+
b.models.add(stage.model);
|
|
509
|
+
b.agentIds.push(stage.agentId);
|
|
510
|
+
if (stage.startedAtMs !== null) {
|
|
511
|
+
b.startedAtMs = b.startedAtMs === null ? stage.startedAtMs : Math.min(b.startedAtMs, stage.startedAtMs);
|
|
512
|
+
const end = stage.durationMs === null ? stage.startedAtMs : stage.startedAtMs + stage.durationMs;
|
|
513
|
+
b.endedAtMs = b.endedAtMs === null ? end : Math.max(b.endedAtMs, end);
|
|
514
|
+
}
|
|
515
|
+
if (!hasKnownPricing(stage.model)) b.pricingKnown = false;
|
|
516
|
+
|
|
517
|
+
// Price per AGENT, using that agent's own model, then aggregate — a `mixed` label must not be
|
|
518
|
+
// priced at one arbitrary model's rate.
|
|
519
|
+
let mix = { promptTokens: 0, cachedInputTokens: 0, cacheCreationTokens: 0, completionTokens: 0 };
|
|
520
|
+
for (const s of samples) {
|
|
521
|
+
if (!universe.has(s.key)) {
|
|
522
|
+
foreign.push(s.key);
|
|
523
|
+
continue; // NEVER add a sample outside the run's universe — it would break the identity
|
|
524
|
+
}
|
|
525
|
+
const canonical = universe.get(s.key);
|
|
526
|
+
if (canonical !== undefined && canonical.weighted !== s.weighted) conflicting.push(s.key);
|
|
527
|
+
stageTokensSum += s.weighted;
|
|
528
|
+
b.sum += s.weighted;
|
|
529
|
+
b.claims.push(s);
|
|
530
|
+
let owners = keyOwners.get(s.key);
|
|
531
|
+
if (owners === undefined) {
|
|
532
|
+
owners = new Set<string>();
|
|
533
|
+
keyOwners.set(s.key, owners);
|
|
534
|
+
}
|
|
535
|
+
owners.add(stage.label);
|
|
536
|
+
mix = {
|
|
537
|
+
promptTokens: mix.promptTokens + s.input,
|
|
538
|
+
cachedInputTokens: mix.cachedInputTokens + s.cacheRead,
|
|
539
|
+
cacheCreationTokens: mix.cacheCreationTokens + s.cacheWrite,
|
|
540
|
+
completionTokens: mix.completionTokens + s.output,
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
const cost = usageCost(mix, stage.model);
|
|
544
|
+
b.costUsd += Number.isFinite(cost) && cost > 0 ? cost : 0;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// accountedTokens — the DEDUPED union of stage-claimed samples, so a double-claim inflates
|
|
548
|
+
// `stageTokensSum` without inflating this. That difference IS `doubleAttributedTokens`.
|
|
549
|
+
let accountedTokens = 0;
|
|
550
|
+
for (const key of keyOwners.keys()) {
|
|
551
|
+
const s = universe.get(key);
|
|
552
|
+
if (s !== undefined) accountedTokens += s.weighted;
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
const unaccountedTokens = runTotalTokens - accountedTokens;
|
|
556
|
+
const doubleAttributedTokens = stageTokensSum - accountedTokens;
|
|
557
|
+
|
|
558
|
+
const rows: CostLedgerRow[] = [];
|
|
559
|
+
for (const b of buckets.values()) {
|
|
560
|
+
let tokensIn = 0;
|
|
561
|
+
let tokensCacheWrite = 0;
|
|
562
|
+
let tokensCacheRead = 0;
|
|
563
|
+
let tokensOut = 0;
|
|
564
|
+
// Sum over CLAIMS, not over deduped keys. `weightedTokens` must equal this bucket's
|
|
565
|
+
// contribution to `stageTokensSum` (the verifier asserts Σ rows === stageTokensSum), so the raw
|
|
566
|
+
// columns and `calls` have to count the same way — otherwise a double-attributed run shows a
|
|
567
|
+
// weighted total its own in/out columns contradict.
|
|
568
|
+
for (const s of b.claims) {
|
|
569
|
+
tokensIn += s.input;
|
|
570
|
+
tokensCacheWrite += s.cacheWrite;
|
|
571
|
+
tokensCacheRead += s.cacheRead;
|
|
572
|
+
tokensOut += s.output;
|
|
573
|
+
}
|
|
574
|
+
const models = [...b.models].sort();
|
|
575
|
+
rows.push({
|
|
576
|
+
runId: record.runId,
|
|
577
|
+
slug: record.slug,
|
|
578
|
+
stage: b.stage,
|
|
579
|
+
phase: b.phase,
|
|
580
|
+
model: models.length === 1 ? (models[0] ?? 'unknown') : 'mixed',
|
|
581
|
+
agentIds: b.agentIds,
|
|
582
|
+
tokensIn,
|
|
583
|
+
tokensCacheWrite,
|
|
584
|
+
tokensCacheRead,
|
|
585
|
+
tokensOut,
|
|
586
|
+
weightedTokens: b.sum,
|
|
587
|
+
costUsd: b.costUsd,
|
|
588
|
+
pricingKnown: b.pricingKnown,
|
|
589
|
+
startedTs: isoOrNull(b.startedAtMs),
|
|
590
|
+
endedTs: isoOrNull(b.endedAtMs),
|
|
591
|
+
calls: b.claims.length,
|
|
592
|
+
});
|
|
593
|
+
}
|
|
594
|
+
rows.sort((a, z) => z.weightedTokens - a.weightedTokens || a.stage.localeCompare(z.stage));
|
|
595
|
+
|
|
596
|
+
// ── named defects ──
|
|
597
|
+
const orphans = (input.orphanAgentIds ?? []).filter((x) => typeof x === 'string' && x.length > 0);
|
|
598
|
+
if (unaccountedTokens > Math.floor(epsilon * runTotalTokens)) {
|
|
599
|
+
defects.push({
|
|
600
|
+
kind: 'Unaccounted',
|
|
601
|
+
detail:
|
|
602
|
+
orphans.length > 0
|
|
603
|
+
? `${orphans.length} agent transcript(s) in the run directory have no workflowProgress entry`
|
|
604
|
+
: 'run spend is attributed to no stage',
|
|
605
|
+
tokens: unaccountedTokens,
|
|
606
|
+
...(orphans.length > 0 ? { subjects: orphans } : {}),
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
const doubleClaimed = [...keyOwners.entries()].filter(([, owners]) => owners.size > 1);
|
|
610
|
+
if (doubleAttributedTokens > 0 || doubleClaimed.length > 0) {
|
|
611
|
+
const stagesInvolved = new Set<string>();
|
|
612
|
+
for (const [, owners] of doubleClaimed) for (const o of owners) stagesInvolved.add(o);
|
|
613
|
+
defects.push({
|
|
614
|
+
kind: 'DoubleAttributed',
|
|
615
|
+
detail: `${doubleClaimed.length} usage sample(s) claimed by more than one stage`,
|
|
616
|
+
tokens: doubleAttributedTokens,
|
|
617
|
+
subjects: [...stagesInvolved].sort(),
|
|
618
|
+
});
|
|
619
|
+
}
|
|
620
|
+
if (foreign.length > 0) {
|
|
621
|
+
defects.push({
|
|
622
|
+
kind: 'ForeignSample',
|
|
623
|
+
detail: `${foreign.length} stage sample(s) absent from the run's transcript directory`,
|
|
624
|
+
subjects: foreign.slice(0, 10),
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
if (conflicting.length > 0) {
|
|
628
|
+
defects.push({
|
|
629
|
+
kind: 'MalformedRecord',
|
|
630
|
+
detail: `${conflicting.length} sample(s) extracted to different token values in two files — the extractor is not deterministic`,
|
|
631
|
+
subjects: conflicting.slice(0, 10),
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
if (missingTranscript.length > 0) {
|
|
635
|
+
defects.push({
|
|
636
|
+
kind: 'MissingStageTranscript',
|
|
637
|
+
detail: `${missingTranscript.length} stage(s) in the run record have no usage samples`,
|
|
638
|
+
subjects: missingTranscript,
|
|
639
|
+
});
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
const identityHolds =
|
|
643
|
+
accountedTokens + unaccountedTokens === runTotalTokens &&
|
|
644
|
+
accountedTokens + doubleAttributedTokens === stageTokensSum;
|
|
645
|
+
if (!identityHolds) {
|
|
646
|
+
defects.push({
|
|
647
|
+
kind: 'MalformedRecord',
|
|
648
|
+
detail:
|
|
649
|
+
`reconciliation identity broken: accounted ${accountedTokens} + unaccounted ${unaccountedTokens} ` +
|
|
650
|
+
`!= total ${runTotalTokens}, or + double ${doubleAttributedTokens} != stageSum ${stageTokensSum}`,
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
// INSUFFICIENT_DATA is NOT success (ADR-003): no samples means nothing was measured, and a
|
|
655
|
+
// "0 === 0, so it balances" shortcut would let an absent transcript store read as a clean run.
|
|
656
|
+
const verdict: CostLedgerVerdict =
|
|
657
|
+
runTotalTokens === 0 && stageTokensSum === 0
|
|
658
|
+
? 'INSUFFICIENT_DATA'
|
|
659
|
+
: defects.length > 0
|
|
660
|
+
? 'DEFECT'
|
|
661
|
+
: 'BALANCED';
|
|
662
|
+
|
|
663
|
+
let totalCostUsd = 0;
|
|
664
|
+
for (const r of rows) totalCostUsd += r.costUsd;
|
|
665
|
+
const fallbackModels = [...new Set(record.stages.filter((s) => !hasKnownPricing(s.model)).map((s) => s.model))].sort();
|
|
666
|
+
|
|
667
|
+
return {
|
|
668
|
+
runId: record.runId,
|
|
669
|
+
slug: record.slug,
|
|
670
|
+
workflowName: record.workflowName,
|
|
671
|
+
status: record.status,
|
|
672
|
+
startedTs: isoOrNull(record.startedAtMs),
|
|
673
|
+
rows,
|
|
674
|
+
reconciliation: {
|
|
675
|
+
runTotalTokens,
|
|
676
|
+
accountedTokens,
|
|
677
|
+
stageTokensSum,
|
|
678
|
+
unaccountedTokens,
|
|
679
|
+
doubleAttributedTokens,
|
|
680
|
+
epsilon,
|
|
681
|
+
identityHolds,
|
|
682
|
+
verdict,
|
|
683
|
+
defects,
|
|
684
|
+
},
|
|
685
|
+
recordTotalTokens: record.recordTotalTokens,
|
|
686
|
+
totalCostUsd,
|
|
687
|
+
pricingFallbackModels: fallbackModels,
|
|
688
|
+
estimated: true,
|
|
689
|
+
scope: COST_LEDGER_SCOPE,
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/**
|
|
694
|
+
* Re-derive both identities from an EMITTED report — the verifier half of the house pattern. It
|
|
695
|
+
* trusts nothing the builder computed except the numbers it printed, so a future writer bug shows
|
|
696
|
+
* up as a `MalformedRecord` finding instead of a plausible table.
|
|
697
|
+
*/
|
|
698
|
+
export function verifyCostLedgerReport(report: CostLedgerReport): readonly CostLedgerDefect[] {
|
|
699
|
+
const out: CostLedgerDefect[] = [];
|
|
700
|
+
const r = report.reconciliation;
|
|
701
|
+
const nums = [r.runTotalTokens, r.accountedTokens, r.stageTokensSum, r.unaccountedTokens, r.doubleAttributedTokens];
|
|
702
|
+
if (nums.some((n) => !Number.isFinite(n))) {
|
|
703
|
+
out.push({ kind: 'MalformedRecord', detail: 'reconciliation carries a non-finite number' });
|
|
704
|
+
return out;
|
|
705
|
+
}
|
|
706
|
+
if (r.accountedTokens + r.unaccountedTokens !== r.runTotalTokens) {
|
|
707
|
+
out.push({
|
|
708
|
+
kind: 'MalformedRecord',
|
|
709
|
+
detail: `accounted ${r.accountedTokens} + unaccounted ${r.unaccountedTokens} !== runTotal ${r.runTotalTokens}`,
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
if (r.accountedTokens + r.doubleAttributedTokens !== r.stageTokensSum) {
|
|
713
|
+
out.push({
|
|
714
|
+
kind: 'MalformedRecord',
|
|
715
|
+
detail: `accounted ${r.accountedTokens} + double ${r.doubleAttributedTokens} !== stageSum ${r.stageTokensSum}`,
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
let rowSum = 0;
|
|
719
|
+
for (const row of report.rows) rowSum += row.weightedTokens;
|
|
720
|
+
if (rowSum !== r.stageTokensSum) {
|
|
721
|
+
out.push({ kind: 'MalformedRecord', detail: `Σ rows ${rowSum} !== stageSum ${r.stageTokensSum}` });
|
|
722
|
+
}
|
|
723
|
+
if (!(COST_LEDGER_VERDICTS as readonly string[]).includes(r.verdict)) {
|
|
724
|
+
out.push({ kind: 'MalformedRecord', detail: `unknown verdict ${String(r.verdict)}` });
|
|
725
|
+
}
|
|
726
|
+
return out;
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
// ── PURE: FR-8 feed-forward reader ──────────────────────────
|
|
730
|
+
|
|
731
|
+
/**
|
|
732
|
+
* Aggregate per-stage cost across runs, for a future auto-cost router that today chooses models
|
|
733
|
+
* from a STATIC assumptions table.
|
|
734
|
+
*
|
|
735
|
+
* **WIRING INTO ROUTING IS OUT OF SCOPE for this feature** — this returns data and nothing consumes
|
|
736
|
+
* it yet. That is deliberate: an ESTIMATED number must not drive an expensive routing decision
|
|
737
|
+
* until it has been calibrated. Rows from runs whose verdict is not `BALANCED` are EXCLUDED, so a
|
|
738
|
+
* run with a known attribution defect can never quietly become a routing input.
|
|
739
|
+
*/
|
|
740
|
+
export function stageCostAggregates(reports: readonly CostLedgerReport[]): StageCostAggregate[] {
|
|
741
|
+
const acc = new Map<string, { stage: string; model: string; total: number; cost: number; runs: Set<string> }>();
|
|
742
|
+
for (const report of reports) {
|
|
743
|
+
if (report.reconciliation.verdict !== 'BALANCED') continue;
|
|
744
|
+
for (const row of report.rows) {
|
|
745
|
+
// JSON-tuple key (Codex QE LOW): NUL is a LEGAL JSON-string character, so even a NUL join
|
|
746
|
+
// can collide when labels themselves contain NUL — the same delimiter-ambiguity class the
|
|
747
|
+
// guard-promotion digest fixed. Unambiguous serialization beats a cleverer separator.
|
|
748
|
+
const key = JSON.stringify([row.stage, row.model]);
|
|
749
|
+
let a = acc.get(key);
|
|
750
|
+
if (a === undefined) {
|
|
751
|
+
a = { stage: row.stage, model: row.model, total: 0, cost: 0, runs: new Set<string>() };
|
|
752
|
+
acc.set(key, a);
|
|
753
|
+
}
|
|
754
|
+
a.total += row.weightedTokens;
|
|
755
|
+
a.cost += row.costUsd;
|
|
756
|
+
a.runs.add(row.runId);
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
const out: StageCostAggregate[] = [];
|
|
760
|
+
for (const a of acc.values()) {
|
|
761
|
+
const runs = a.runs.size;
|
|
762
|
+
out.push({
|
|
763
|
+
stage: a.stage,
|
|
764
|
+
model: a.model,
|
|
765
|
+
avgTokens: runs > 0 ? Math.round(a.total / runs) : 0,
|
|
766
|
+
runs,
|
|
767
|
+
totalTokens: a.total,
|
|
768
|
+
avgCostUsd: runs > 0 ? a.cost / runs : 0,
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
out.sort((x, y) => y.avgTokens - x.avgTokens || x.stage.localeCompare(y.stage));
|
|
772
|
+
return out;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
// ── PURE: rendering + serialization ─────────────────────────
|
|
776
|
+
|
|
777
|
+
function fmt(n: number): string {
|
|
778
|
+
if (!Number.isFinite(n)) return '?';
|
|
779
|
+
return Math.round(n).toLocaleString('en-US');
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
function usd(n: number): string {
|
|
783
|
+
if (!Number.isFinite(n) || n <= 0) return '$0.00';
|
|
784
|
+
return '$' + n.toFixed(n < 1 ? 4 : 2);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
function pad(s: string, width: number): string {
|
|
788
|
+
return s.length >= width ? s : s + ' '.repeat(width - s.length);
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
function padLeft(s: string, width: number): string {
|
|
792
|
+
return s.length >= width ? s : ' '.repeat(width - s.length) + s;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/** Human table + reconciliation line + verdict + the honest-scope note (ADR-003). */
|
|
796
|
+
export function renderCostLedger(report: CostLedgerReport): string {
|
|
797
|
+
const lines: string[] = [];
|
|
798
|
+
const head = [
|
|
799
|
+
`run ${report.runId}`,
|
|
800
|
+
report.slug !== null ? `slug ${report.slug}` : null,
|
|
801
|
+
report.workflowName !== null ? report.workflowName : null,
|
|
802
|
+
report.status !== null ? report.status : null,
|
|
803
|
+
report.startedTs !== null ? report.startedTs : null,
|
|
804
|
+
]
|
|
805
|
+
.filter((x): x is string => x !== null)
|
|
806
|
+
.join(' · ');
|
|
807
|
+
lines.push(`usage --by-stage: ${head}`);
|
|
808
|
+
|
|
809
|
+
const r = report.reconciliation;
|
|
810
|
+
if (report.rows.length === 0) {
|
|
811
|
+
lines.push('usage --by-stage: no stage rows — nothing was measured for this run');
|
|
812
|
+
} else {
|
|
813
|
+
const stageW = Math.max(5, ...report.rows.map((x) => x.stage.length));
|
|
814
|
+
const modelW = Math.max(5, ...report.rows.map((x) => x.model.length));
|
|
815
|
+
lines.push(
|
|
816
|
+
` ${pad('stage', stageW)} ${pad('model', modelW)} ${padLeft('weighted', 12)} ${padLeft('in', 9)} ${padLeft('out', 9)} ${padLeft('calls', 5)} ${padLeft('~USD', 9)}`,
|
|
817
|
+
);
|
|
818
|
+
for (const row of report.rows) {
|
|
819
|
+
lines.push(
|
|
820
|
+
` ${pad(row.stage, stageW)} ${pad(row.model, modelW)} ${padLeft(fmt(row.weightedTokens), 12)} ${padLeft(fmt(row.tokensIn), 9)} ${padLeft(fmt(row.tokensOut), 9)} ${padLeft(String(row.calls), 5)} ${padLeft(usd(row.costUsd) + (row.pricingKnown ? '' : '*'), 9)}`,
|
|
821
|
+
);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
const pctUn = r.runTotalTokens > 0 ? (100 * r.unaccountedTokens) / r.runTotalTokens : 0;
|
|
826
|
+
lines.push(
|
|
827
|
+
` reconciliation: accounted ${fmt(r.accountedTokens)} + unaccounted ${fmt(r.unaccountedTokens)} = run total ${fmt(r.runTotalTokens)}` +
|
|
828
|
+
` (epsilon ${(r.epsilon * 100).toFixed(2)}%, unaccounted ${pctUn.toFixed(1)}%)`,
|
|
829
|
+
);
|
|
830
|
+
if (r.doubleAttributedTokens !== 0) {
|
|
831
|
+
lines.push(
|
|
832
|
+
` reconciliation: accounted ${fmt(r.accountedTokens)} + double-attributed ${fmt(r.doubleAttributedTokens)} = Σ stages ${fmt(r.stageTokensSum)}`,
|
|
833
|
+
);
|
|
834
|
+
}
|
|
835
|
+
lines.push(` identity: ${r.identityHolds ? 'holds (raw integer equality)' : 'BROKEN'}`);
|
|
836
|
+
lines.push(` verdict: ${r.verdict}`);
|
|
837
|
+
for (const d of r.defects) {
|
|
838
|
+
const tok = d.tokens === undefined ? '' : ` (${fmt(d.tokens)} weighted tokens)`;
|
|
839
|
+
const subj = d.subjects === undefined || d.subjects.length === 0 ? '' : ` [${d.subjects.slice(0, 6).join(', ')}${d.subjects.length > 6 ? ', …' : ''}]`;
|
|
840
|
+
lines.push(` ${d.kind}: ${d.detail}${tok}${subj}`);
|
|
841
|
+
}
|
|
842
|
+
if (report.recordTotalTokens !== null) {
|
|
843
|
+
lines.push(
|
|
844
|
+
` note: the run record's own totalTokens is ${fmt(report.recordTotalTokens)} — a RAW unweighted cached sum of the same per-agent list, reported for traceability, NOT the invariant's right-hand side`,
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
if (report.pricingFallbackModels.length > 0) {
|
|
848
|
+
lines.push(` note: ~USD marked * uses sonnet-class FALLBACK pricing for: ${report.pricingFallbackModels.join(', ')}`);
|
|
849
|
+
}
|
|
850
|
+
lines.push(` scope: ${COST_LEDGER_SCOPE}`);
|
|
851
|
+
return lines.join('\n');
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
/**
|
|
855
|
+
* FR-7 serialization: one JSON object per line. The first line is a `kind: "cost-ledger-scope"`
|
|
856
|
+
* header carrying {@link COST_LEDGER_SCOPE}, so the honest scope travels with the file; the last is
|
|
857
|
+
* the reconciliation. This is a REGENERABLE REPORT, never a read-back source of truth (ADR-001).
|
|
858
|
+
*/
|
|
859
|
+
export function costLedgerJsonl(report: CostLedgerReport): string {
|
|
860
|
+
const lines: string[] = [];
|
|
861
|
+
lines.push(
|
|
862
|
+
JSON.stringify({
|
|
863
|
+
kind: 'cost-ledger-scope',
|
|
864
|
+
runId: report.runId,
|
|
865
|
+
slug: report.slug,
|
|
866
|
+
estimated: true,
|
|
867
|
+
derived: true,
|
|
868
|
+
scope: COST_LEDGER_SCOPE,
|
|
869
|
+
}),
|
|
870
|
+
);
|
|
871
|
+
for (const row of report.rows) lines.push(JSON.stringify({ kind: 'cost-ledger-row', ...row }));
|
|
872
|
+
lines.push(JSON.stringify({ kind: 'cost-ledger-reconciliation', ...report.reconciliation }));
|
|
873
|
+
return lines.join('\n') + '\n';
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
// ── IO: discovery + derivation (never-throw, readonly) ──────
|
|
877
|
+
|
|
878
|
+
export interface CostLedgerIoOptions {
|
|
879
|
+
/** Override the `~/.claude/projects` root. Defaults to {@link claudeProjectsRoot}. */
|
|
880
|
+
readonly projectsRoot?: string;
|
|
881
|
+
/** Munged project directory name (e.g. `-home-user-repo`). Absent ⇒ scan every project. */
|
|
882
|
+
readonly projectDir?: string;
|
|
883
|
+
readonly epsilon?: number;
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
export interface CostLedgerRunRef {
|
|
887
|
+
readonly runId: string;
|
|
888
|
+
readonly slug: string | null;
|
|
889
|
+
readonly workflowName: string | null;
|
|
890
|
+
readonly status: string | null;
|
|
891
|
+
readonly startedAtMs: number | null;
|
|
892
|
+
/** Absolute path of the `wf_*.json` record. */
|
|
893
|
+
readonly recordPath: string;
|
|
894
|
+
/** Absolute path of `<session>/subagents/workflows/<runId>` — may not exist. */
|
|
895
|
+
readonly transcriptDir: string;
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
function safeReadJson(path: string): unknown {
|
|
899
|
+
try {
|
|
900
|
+
const st = lstatSync(path);
|
|
901
|
+
if (!st.isFile()) return null;
|
|
902
|
+
return JSON.parse(readFileSync(path, 'utf-8')) as unknown;
|
|
903
|
+
} catch {
|
|
904
|
+
return null;
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
function safeReadText(path: string): string {
|
|
909
|
+
try {
|
|
910
|
+
const st = lstatSync(path);
|
|
911
|
+
if (!st.isFile()) return '';
|
|
912
|
+
return readFileSync(path, 'utf-8');
|
|
913
|
+
} catch {
|
|
914
|
+
return '';
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
function safeListDir(path: string): string[] {
|
|
919
|
+
try {
|
|
920
|
+
const st = lstatSync(path);
|
|
921
|
+
if (!st.isDirectory()) return [];
|
|
922
|
+
return readdirSync(path);
|
|
923
|
+
} catch {
|
|
924
|
+
return [];
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
/**
|
|
929
|
+
* List a PROJECT directory, following a symlink at that ONE level.
|
|
930
|
+
*
|
|
931
|
+
* The asymmetry against {@link safeListDir} is deliberate and load-bearing. `usage.ts` refuses to
|
|
932
|
+
* follow symlinked project directories, and rightly — an account-wide scan that follows links can
|
|
933
|
+
* be pointed at an unbounded tree. But this repo ROAMS its own transcript store: the entry
|
|
934
|
+
* `~/.claude/projects/-home-dz-projects-2026-dz-harness-hub` is a symlink to
|
|
935
|
+
* `<repo>/roam/claude-state` (MEASURED — reproducer: `readlink` on that path). With a plain `lstat`
|
|
936
|
+
* gate the ledger found 0 of this project's 29 run records: the feature was blind to exactly the
|
|
937
|
+
* project it exists to measure.
|
|
938
|
+
*
|
|
939
|
+
* So: the project level follows one link; EVERY level below still uses `lstat` and never follows.
|
|
940
|
+
* That keeps the hazards `usage.ts` guards against — a symlinked session directory, a FIFO or a
|
|
941
|
+
* link to a huge file where a transcript should be — while making the roaming layout readable. The
|
|
942
|
+
* ledger is also per-RUN, not account-wide, so the unbounded-walk concern does not apply.
|
|
943
|
+
*/
|
|
944
|
+
function safeListProjectDir(path: string): string[] {
|
|
945
|
+
try {
|
|
946
|
+
if (!statSync(path).isDirectory()) return [];
|
|
947
|
+
return readdirSync(path);
|
|
948
|
+
} catch {
|
|
949
|
+
return [];
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
/**
|
|
954
|
+
* Enumerate workflow run records, newest first. NEVER throws — an unreadable tree yields `[]`.
|
|
955
|
+
* READONLY. `lstat` everywhere, so a symlinked session or run directory is never walked.
|
|
956
|
+
*/
|
|
957
|
+
export function listCostLedgerRuns(opts: CostLedgerIoOptions = {}): CostLedgerRunRef[] {
|
|
958
|
+
const root = opts.projectsRoot ?? claudeProjectsRoot();
|
|
959
|
+
const out: CostLedgerRunRef[] = [];
|
|
960
|
+
if (!root || !existsSync(root)) return out;
|
|
961
|
+
const projectDirs =
|
|
962
|
+
opts.projectDir !== undefined && opts.projectDir.length > 0 ? [opts.projectDir] : safeListDir(root);
|
|
963
|
+
// Two project-dir ALIASES to one transcript tree (~/.claude/projects entries are symlinks on this
|
|
964
|
+
// machine) would double-discover every run: FR-8 then derives the same run twice and halves into a
|
|
965
|
+
// 2x average (Codex QE MED). Canonicalize and visit each real tree once; runIds dedupe as a belt.
|
|
966
|
+
const seenRealProj = new Set<string>();
|
|
967
|
+
const seenRunIds = new Set<string>();
|
|
968
|
+
for (const proj of projectDirs) {
|
|
969
|
+
// A project dir name is data from the filesystem, but `opts.projectDir` is caller-supplied.
|
|
970
|
+
if (proj.includes('/') || proj.includes('\\') || proj === '.' || proj === '..') continue;
|
|
971
|
+
const projPath = join(root, proj);
|
|
972
|
+
let realProj = projPath;
|
|
973
|
+
try { realProj = realpathSync(projPath); } catch { /* keep the lexical path */ }
|
|
974
|
+
if (seenRealProj.has(realProj)) continue;
|
|
975
|
+
seenRealProj.add(realProj);
|
|
976
|
+
for (const sess of safeListProjectDir(projPath)) {
|
|
977
|
+
if (sess.endsWith('.jsonl')) continue;
|
|
978
|
+
const wfDir = join(projPath, sess, 'workflows');
|
|
979
|
+
for (const f of safeListDir(wfDir)) {
|
|
980
|
+
if (!f.endsWith('.json')) continue;
|
|
981
|
+
const recordPath = join(wfDir, f);
|
|
982
|
+
const parsed = parseWorkflowRunRecord(safeReadJson(recordPath));
|
|
983
|
+
if (parsed === null) continue;
|
|
984
|
+
if (!RUN_ID_PATTERN.test(parsed.runId)) continue; // runId becomes a path segment
|
|
985
|
+
if (seenRunIds.has(parsed.runId)) continue; // belt to the realpath braces
|
|
986
|
+
seenRunIds.add(parsed.runId);
|
|
987
|
+
out.push({
|
|
988
|
+
runId: parsed.runId,
|
|
989
|
+
slug: parsed.slug,
|
|
990
|
+
workflowName: parsed.workflowName,
|
|
991
|
+
status: parsed.status,
|
|
992
|
+
startedAtMs: parsed.startedAtMs,
|
|
993
|
+
recordPath,
|
|
994
|
+
transcriptDir: join(projPath, sess, 'subagents', 'workflows', parsed.runId),
|
|
995
|
+
});
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
out.sort((a, b) => (b.startedAtMs ?? 0) - (a.startedAtMs ?? 0) || b.runId.localeCompare(a.runId));
|
|
1000
|
+
return out;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
export interface DeriveCostLedgerOptions extends CostLedgerIoOptions {
|
|
1004
|
+
/** Exact run id. Must match `[A-Za-z0-9_.-]{1,128}` — it becomes a path segment. */
|
|
1005
|
+
readonly runId?: string;
|
|
1006
|
+
/** Most recent run with this `args.slug`. Same pattern restriction. */
|
|
1007
|
+
readonly slug?: string;
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
/**
|
|
1011
|
+
* Derive the ledger for ONE run. Returns `null` when no run matches — an ABSENT run is never a
|
|
1012
|
+
* BALANCED empty report (ADR-003). NEVER throws; READONLY.
|
|
1013
|
+
*/
|
|
1014
|
+
export function deriveCostLedger(opts: DeriveCostLedgerOptions = {}): CostLedgerReport | null {
|
|
1015
|
+
try {
|
|
1016
|
+
if (opts.runId !== undefined && !RUN_ID_PATTERN.test(opts.runId)) return null;
|
|
1017
|
+
if (opts.slug !== undefined && !SLUG_PATTERN.test(opts.slug)) return null;
|
|
1018
|
+
const runs = listCostLedgerRuns(opts);
|
|
1019
|
+
const ref =
|
|
1020
|
+
opts.runId !== undefined
|
|
1021
|
+
? runs.find((r) => r.runId === opts.runId)
|
|
1022
|
+
: opts.slug !== undefined
|
|
1023
|
+
? runs.find((r) => r.slug === opts.slug)
|
|
1024
|
+
: runs[0];
|
|
1025
|
+
if (ref === undefined) return null;
|
|
1026
|
+
|
|
1027
|
+
const record = parseWorkflowRunRecord(safeReadJson(ref.recordPath));
|
|
1028
|
+
if (record === null) return null;
|
|
1029
|
+
|
|
1030
|
+
const stageAgentIds = new Set(record.stages.map((s) => s.agentId));
|
|
1031
|
+
const allFiles = safeListDir(ref.transcriptDir).filter((f) => f.endsWith('.jsonl'));
|
|
1032
|
+
// A capped listing means the run total is built from a PARTIAL directory — BALANCED on partial
|
|
1033
|
+
// evidence is the false green this feature exists to refuse (Codex QE HIGH). The cap stays (a
|
|
1034
|
+
// pathological dir must not hang us) but it becomes a NAMED defect, never a silent slice.
|
|
1035
|
+
const listingTruncated = allFiles.length > MAX_RUN_TRANSCRIPT_FILES;
|
|
1036
|
+
const files = allFiles.slice(0, MAX_RUN_TRANSCRIPT_FILES);
|
|
1037
|
+
|
|
1038
|
+
const runSamples: CostLedgerSample[] = [];
|
|
1039
|
+
const perAgent = new Map<string, CostLedgerSample[]>();
|
|
1040
|
+
const orphanAgentIds: string[] = [];
|
|
1041
|
+
for (const f of files) {
|
|
1042
|
+
const samples = extractCostSamples(safeReadText(join(ref.transcriptDir, f)));
|
|
1043
|
+
runSamples.push(...samples);
|
|
1044
|
+
const m = /^agent-(.+)\.jsonl$/.exec(f);
|
|
1045
|
+
if (m === null) continue;
|
|
1046
|
+
const agentId = m[1] ?? '';
|
|
1047
|
+
if (stageAgentIds.has(agentId)) perAgent.set(agentId, samples);
|
|
1048
|
+
else if (samples.length > 0) orphanAgentIds.push(agentId);
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
return buildCostLedger({
|
|
1052
|
+
record,
|
|
1053
|
+
stageSamples: [...perAgent.entries()].map(([agentId, samples]) => ({ agentId, samples })),
|
|
1054
|
+
runSamples,
|
|
1055
|
+
orphanAgentIds,
|
|
1056
|
+
...(listingTruncated ? { transcriptListingTruncated: true } : {}),
|
|
1057
|
+
...(opts.epsilon !== undefined ? { epsilon: opts.epsilon } : {}),
|
|
1058
|
+
});
|
|
1059
|
+
} catch {
|
|
1060
|
+
return null; // never-throw contract
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
/**
|
|
1065
|
+
* FR-8 IO wrapper: derive every run and aggregate. Runs that do not reconcile are excluded by
|
|
1066
|
+
* {@link stageCostAggregates}. NEVER throws; READONLY. Still NOT wired into routing.
|
|
1067
|
+
*/
|
|
1068
|
+
export function deriveStageCostAggregates(opts: CostLedgerIoOptions & { readonly maxRuns?: number } = {}): StageCostAggregate[] {
|
|
1069
|
+
try {
|
|
1070
|
+
const maxRuns =
|
|
1071
|
+
typeof opts.maxRuns === 'number' && Number.isFinite(opts.maxRuns) && opts.maxRuns > 0
|
|
1072
|
+
? Math.floor(opts.maxRuns)
|
|
1073
|
+
: 200;
|
|
1074
|
+
const reports: CostLedgerReport[] = [];
|
|
1075
|
+
for (const ref of listCostLedgerRuns(opts).slice(0, maxRuns)) {
|
|
1076
|
+
const rep = deriveCostLedger({ ...opts, runId: ref.runId });
|
|
1077
|
+
if (rep !== null) reports.push(rep);
|
|
1078
|
+
}
|
|
1079
|
+
return stageCostAggregates(reports);
|
|
1080
|
+
} catch {
|
|
1081
|
+
return [];
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
/**
|
|
1086
|
+
* FR-7 opt-in materialization. Atomic: writes a sibling `.tmp` then `renameSync`s over the target,
|
|
1087
|
+
* and removes the temp file if the rename fails, so a crash can never leave a half-written ledger.
|
|
1088
|
+
* Returns `true` on success; never throws.
|
|
1089
|
+
*/
|
|
1090
|
+
export function writeCostLedgerJsonl(path: string, report: CostLedgerReport): boolean {
|
|
1091
|
+
const tmp = path + '.tmp';
|
|
1092
|
+
try {
|
|
1093
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1094
|
+
writeFileSync(tmp, costLedgerJsonl(report), 'utf-8');
|
|
1095
|
+
renameSync(tmp, path);
|
|
1096
|
+
return true;
|
|
1097
|
+
} catch {
|
|
1098
|
+
try {
|
|
1099
|
+
unlinkSync(tmp);
|
|
1100
|
+
} catch {
|
|
1101
|
+
/* nothing to clean up */
|
|
1102
|
+
}
|
|
1103
|
+
return false;
|
|
1104
|
+
}
|
|
1105
|
+
}
|