dsh-token-usage-ledger 0.0.37 → 0.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/README.md +16 -10
- package/lib/client.js +1731 -568
- package/lib/host/aggregate.js +566 -0
- package/lib/host/backfill.d.ts +14 -0
- package/lib/host/backfill.js +30 -2
- package/lib/host/collector.d.ts +54 -3
- package/lib/host/collector.js +289 -10
- package/lib/host/ledger-io.d.ts +28 -1
- package/lib/host/ledger-io.js +54 -4
- package/lib/host/records.d.ts +18 -0
- package/lib/host/records.js +143 -1
- package/lib/host/schema.js +167 -2
- package/lib/index.d.ts +1 -1
- package/lib/index.js +12 -7
- package/lib/shared/format.d.ts +19 -1
- package/lib/shared/format.js +70 -1
- package/lib/shared/protocol.d.ts +277 -3
- package/lib/shared/protocol.js +9 -2
- package/package.json +1 -1
package/lib/host/backfill.js
CHANGED
|
@@ -13,7 +13,7 @@ import { readFile, readdir } from 'node:fs/promises';
|
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
import { TurnUsageCollector } from './collector.js';
|
|
15
15
|
import { fingerprintOfParts, readLedgerKeys } from './ledger-io.js';
|
|
16
|
-
import { buildRecord } from './records.js';
|
|
16
|
+
import { buildRecord, mergeRecordExtras } from './records.js';
|
|
17
17
|
import { decodeZstdFrames } from './zstd.js';
|
|
18
18
|
/** Session-log filenames this build understands, newest first. */
|
|
19
19
|
export const SESSION_LOG_NAMES = ['session.v4.jsonl.zstd', 'session.v3.jsonl.zstd'];
|
|
@@ -167,6 +167,7 @@ export async function backfillLedger(options) {
|
|
|
167
167
|
const lead = options.lead ?? '';
|
|
168
168
|
const limit = options.limit ?? 0;
|
|
169
169
|
const noPrefilter = options.noPrefilter === true;
|
|
170
|
+
const enrich = options.enrich !== false;
|
|
170
171
|
const existing = await readLedgerKeys(ledgerPath);
|
|
171
172
|
const { logs, unknown } = await findSessionLogs(home, lead, limit);
|
|
172
173
|
const report = {
|
|
@@ -181,6 +182,9 @@ export async function backfillLedger(options) {
|
|
|
181
182
|
turnsUndecidable: 0,
|
|
182
183
|
turnsFilteredOut: 0,
|
|
183
184
|
turnsBackfillable: 0,
|
|
185
|
+
turnsEnrichable: 0,
|
|
186
|
+
turnsEnriched: 0,
|
|
187
|
+
turnsEnrichSkipped: 0,
|
|
184
188
|
firstTurn: '',
|
|
185
189
|
lastTurn: '',
|
|
186
190
|
};
|
|
@@ -202,6 +206,8 @@ export async function backfillLedger(options) {
|
|
|
202
206
|
}
|
|
203
207
|
ordered.sort((left, right) => left.createdAt - right.createdAt);
|
|
204
208
|
const records = [];
|
|
209
|
+
const upgrades = [];
|
|
210
|
+
const upgradedKeys = new Set();
|
|
205
211
|
for (const { log } of ordered) {
|
|
206
212
|
let parsed;
|
|
207
213
|
try {
|
|
@@ -237,6 +243,27 @@ export async function backfillLedger(options) {
|
|
|
237
243
|
const key = `${sessionId}\u0000${String(outcome.turn)}`;
|
|
238
244
|
if (existing.keys.has(key)) {
|
|
239
245
|
report.turnsAlreadyRecorded += 1;
|
|
246
|
+
if (!enrich)
|
|
247
|
+
continue;
|
|
248
|
+
// Try to fill v2 timing/activity fields into the existing line. The
|
|
249
|
+
// merge is additive and alignment-checked; when the log has nothing new
|
|
250
|
+
// (or cannot be aligned) the original record is left untouched.
|
|
251
|
+
const previous = existing.recordsByKey.get(key);
|
|
252
|
+
if (previous !== undefined && !upgradedKeys.has(key)) {
|
|
253
|
+
const upgraded = mergeRecordExtras(previous, outcome);
|
|
254
|
+
if (upgraded === undefined) {
|
|
255
|
+
report.turnsEnrichSkipped += 1;
|
|
256
|
+
}
|
|
257
|
+
else {
|
|
258
|
+
report.turnsEnrichable += 1;
|
|
259
|
+
upgradedKeys.add(key);
|
|
260
|
+
upgrades.push(upgraded);
|
|
261
|
+
report.turnsEnriched += 1;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
else {
|
|
265
|
+
report.turnsEnrichSkipped += 1;
|
|
266
|
+
}
|
|
240
267
|
continue;
|
|
241
268
|
}
|
|
242
269
|
const fingerprint = fingerprintOfParts(outcome.turn, outcome.startedAt, outcome.endedAt, outcome.attempts.map((attempt) => attempt.at), outcome.total);
|
|
@@ -260,6 +287,7 @@ export async function backfillLedger(options) {
|
|
|
260
287
|
}
|
|
261
288
|
}
|
|
262
289
|
records.sort((left, right) => left.ts - right.ts);
|
|
263
|
-
|
|
290
|
+
upgrades.sort((left, right) => left.ts - right.ts);
|
|
291
|
+
return { report, records, upgrades };
|
|
264
292
|
}
|
|
265
293
|
//# sourceMappingURL=backfill.js.map
|
package/lib/host/collector.d.ts
CHANGED
|
@@ -19,9 +19,10 @@
|
|
|
19
19
|
* @module dsh-token-usage-ledger/host/collector
|
|
20
20
|
*/
|
|
21
21
|
import type { SessionEvent } from '@deepseek-ai/dsh-session';
|
|
22
|
-
import type { TokenBuckets } from '../shared/protocol.js';
|
|
22
|
+
import type { TokenBuckets, ToolUsage } from '../shared/protocol.js';
|
|
23
23
|
/**
|
|
24
|
-
* One billed attempt as the collector observes it — usage and
|
|
24
|
+
* One billed attempt as the collector observes it — usage, route, and (v2) the
|
|
25
|
+
* per-call timing/stream facts that dsh 0.2.2 already records.
|
|
25
26
|
*
|
|
26
27
|
* Pricing is applied later, when the record is built, so this fold stays pure
|
|
27
28
|
* token accounting exactly as the upstream derivation is.
|
|
@@ -41,6 +42,28 @@ export interface CollectedAttempt {
|
|
|
41
42
|
readonly tokens: TokenBuckets;
|
|
42
43
|
/** `totalTokens` reported by the provider for this attempt. */
|
|
43
44
|
readonly total: number;
|
|
45
|
+
/** Step number this attempt belongs to. */
|
|
46
|
+
readonly step: number;
|
|
47
|
+
/** Local epoch ms when this attempt started waiting for the model. */
|
|
48
|
+
readonly startedAt: number;
|
|
49
|
+
/** Local epoch ms of this attempt's first output token, when the stream has one. */
|
|
50
|
+
readonly firstTokenAt?: number;
|
|
51
|
+
/** `firstTokenAt - startedAt` clamped to `>= 0`. */
|
|
52
|
+
readonly ttftMs?: number;
|
|
53
|
+
/** `at - firstTokenAt` clamped to `>= 0`. */
|
|
54
|
+
readonly decodeMs?: number;
|
|
55
|
+
/** `at - startedAt` clamped to `>= 0`. */
|
|
56
|
+
readonly durationMs?: number;
|
|
57
|
+
/** Output speed of this call in tokens/second. */
|
|
58
|
+
readonly tokensPerSecond?: number;
|
|
59
|
+
/** Which event closed this attempt. */
|
|
60
|
+
readonly settledBy: 'message' | 'attempt' | 'retry' | 'step-end';
|
|
61
|
+
/** True when the settling assistant message was interrupted. */
|
|
62
|
+
readonly interrupted?: boolean;
|
|
63
|
+
/** Whether the provider explicitly reported each optional bucket. */
|
|
64
|
+
readonly reportedCacheRead: boolean;
|
|
65
|
+
readonly reportedCacheWrite: boolean;
|
|
66
|
+
readonly reportedReasoning: boolean;
|
|
44
67
|
}
|
|
45
68
|
/** Outcome of feeding one event to the collector. */
|
|
46
69
|
export type PushResult = {
|
|
@@ -67,6 +90,14 @@ export interface TurnOutcome {
|
|
|
67
90
|
readonly tokens: TokenBuckets;
|
|
68
91
|
/** Prompt + output across attempts. */
|
|
69
92
|
readonly total: number;
|
|
93
|
+
/** Per-tool-name activity (v2). */
|
|
94
|
+
readonly tools: readonly ToolUsage[];
|
|
95
|
+
/** Number of `llm/retry` events in this Turn. */
|
|
96
|
+
readonly retries: number;
|
|
97
|
+
/** Stable failure codes observed on those retries, de-duplicated. */
|
|
98
|
+
readonly retryCodes: readonly string[];
|
|
99
|
+
/** True when any settled assistant message was interrupted. */
|
|
100
|
+
readonly interrupted: boolean;
|
|
70
101
|
}
|
|
71
102
|
/**
|
|
72
103
|
* Fold one Session's live event feed into exact per-Turn accounting.
|
|
@@ -87,6 +118,22 @@ export declare class TurnUsageCollector {
|
|
|
87
118
|
private reason;
|
|
88
119
|
/** Event time of the event currently being consumed; stamps settled attempts. */
|
|
89
120
|
private eventTime;
|
|
121
|
+
/** Current step number and its opening time. */
|
|
122
|
+
private step;
|
|
123
|
+
private stepStartedAt;
|
|
124
|
+
/** Start of the current attempt (step/start or llm/retry-started). */
|
|
125
|
+
private attemptStartedAt;
|
|
126
|
+
/** Calls awaiting their `tool/result`, keyed by call id. */
|
|
127
|
+
private toolCalls;
|
|
128
|
+
/** Calls whose result was already counted (protects against duplicates). */
|
|
129
|
+
private toolResults;
|
|
130
|
+
/** Per-tool-name activity for this Turn. */
|
|
131
|
+
private toolStats;
|
|
132
|
+
/** Provider-routed retries scheduled in this Turn. */
|
|
133
|
+
private retries;
|
|
134
|
+
private retryCodes;
|
|
135
|
+
/** Any settled assistant message marked interrupted. */
|
|
136
|
+
private interrupted;
|
|
90
137
|
/**
|
|
91
138
|
* Whether a Turn is currently being collected.
|
|
92
139
|
* @returns true while an observed Turn is open.
|
|
@@ -124,10 +171,14 @@ export declare class TurnUsageCollector {
|
|
|
124
171
|
*
|
|
125
172
|
* The sample is normalized HERE rather than on arrival so a raw sample can be
|
|
126
173
|
* carried across the `assistant/attempt` → `assistant/message` pair exactly
|
|
127
|
-
* as upstream does.
|
|
174
|
+
* as upstream does. Timing facts are read from the event that closes the
|
|
175
|
+
* attempt and never retained as a stream.
|
|
128
176
|
*
|
|
129
177
|
* @param provider - route provider for the settled attempt.
|
|
130
178
|
* @param model - route model for the settled attempt.
|
|
179
|
+
* @param settledBy - which event closed the attempt.
|
|
180
|
+
* @param firstTokenAt - first token time of this attempt's stream, if any.
|
|
181
|
+
* @param interrupted - whether the settling assistant message was interrupted.
|
|
131
182
|
* @returns true when the attempt was recorded.
|
|
132
183
|
*/
|
|
133
184
|
private closeOpen;
|
package/lib/host/collector.js
CHANGED
|
@@ -70,6 +70,138 @@ function streamUsage(stream) {
|
|
|
70
70
|
}
|
|
71
71
|
return undefined;
|
|
72
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Whether one stream chunk is the first model output token for latency purposes.
|
|
75
|
+
*
|
|
76
|
+
* This is a faithful copy of `@deepseek-ai/dsh-llm`'s `isTokenDelta`: text and
|
|
77
|
+
* reasoning count only with a non-empty fragment, and a tool-call fragment
|
|
78
|
+
* counts when it carries arguments or a name. Block/usage/finish chunks never
|
|
79
|
+
* count. Re-implemented rather than imported for the same reason the token fold
|
|
80
|
+
* is: no runtime dependency on an internal package.
|
|
81
|
+
*
|
|
82
|
+
* @param chunk - raw stream chunk.
|
|
83
|
+
* @returns true when the chunk carries an output token.
|
|
84
|
+
*/
|
|
85
|
+
function isTokenDeltaChunk(chunk) {
|
|
86
|
+
if (typeof chunk !== 'object' || chunk === null)
|
|
87
|
+
return false;
|
|
88
|
+
const typed = chunk;
|
|
89
|
+
if (typed.type === 'text-delta' || typed.type === 'reasoning-delta') {
|
|
90
|
+
return typeof typed.text === 'string' && typed.text !== '';
|
|
91
|
+
}
|
|
92
|
+
if (typed.type === 'tool-call-delta') {
|
|
93
|
+
return (typeof typed.argumentsDelta === 'string' && typed.argumentsDelta !== '') ||
|
|
94
|
+
typed.name !== undefined;
|
|
95
|
+
}
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Time of the first token inside one packed delta run.
|
|
100
|
+
*
|
|
101
|
+
* `time0` is the first member, then each `dt[i]` is the delay before member
|
|
102
|
+
* `i + 1`; a name-bearing tool-call run starts at `time0` exactly as upstream's
|
|
103
|
+
* `runFirstTokenTime` does.
|
|
104
|
+
*
|
|
105
|
+
* @param run - compact delta run.
|
|
106
|
+
* @returns epoch ms of the first token, or undefined.
|
|
107
|
+
*/
|
|
108
|
+
function runFirstTokenTime(run) {
|
|
109
|
+
if (typeof run.time0 !== 'number' || !Number.isFinite(run.time0))
|
|
110
|
+
return undefined;
|
|
111
|
+
const fragments = run.type === 'tool-call-chunks'
|
|
112
|
+
? run.args
|
|
113
|
+
: run.type === 'text-chunks' || run.type === 'reasoning-chunks'
|
|
114
|
+
? run.texts
|
|
115
|
+
: undefined;
|
|
116
|
+
if (run.type === 'tool-call-chunks' && run.name !== undefined)
|
|
117
|
+
return run.time0;
|
|
118
|
+
if (!Array.isArray(fragments))
|
|
119
|
+
return undefined;
|
|
120
|
+
const dt = Array.isArray(run.dt) ? run.dt : [];
|
|
121
|
+
let time = run.time0;
|
|
122
|
+
for (let index = 0; index < fragments.length; index += 1) {
|
|
123
|
+
if (index > 0) {
|
|
124
|
+
const delta = dt[index - 1];
|
|
125
|
+
if (typeof delta !== 'number' || !Number.isFinite(delta))
|
|
126
|
+
return undefined;
|
|
127
|
+
time += delta;
|
|
128
|
+
if (!Number.isFinite(time))
|
|
129
|
+
return undefined;
|
|
130
|
+
}
|
|
131
|
+
if (fragments[index] !== '')
|
|
132
|
+
return time;
|
|
133
|
+
}
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Time of the first output token in one durable compact stream.
|
|
138
|
+
*
|
|
139
|
+
* Mirrors `assistantStreamFirstTokenTime` from `@deepseek-ai/dsh-llm`; a legacy
|
|
140
|
+
* stream that contains only the usage chunk has no token and therefore no
|
|
141
|
+
* latency — returning `undefined` is the honest answer there.
|
|
142
|
+
*
|
|
143
|
+
* @param stream - compact assistant stream records.
|
|
144
|
+
* @returns epoch ms of the first token, or undefined.
|
|
145
|
+
*/
|
|
146
|
+
function firstTokenTimeOf(stream) {
|
|
147
|
+
if (!Array.isArray(stream))
|
|
148
|
+
return undefined;
|
|
149
|
+
for (const record of stream) {
|
|
150
|
+
if (typeof record !== 'object' || record === null)
|
|
151
|
+
continue;
|
|
152
|
+
const typed = record;
|
|
153
|
+
if (typed.type === 'chunk') {
|
|
154
|
+
if (typeof typed.time === 'number' && Number.isFinite(typed.time) && isTokenDeltaChunk(typed.chunk)) {
|
|
155
|
+
return typed.time;
|
|
156
|
+
}
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const run = record;
|
|
160
|
+
const time = runFirstTokenTime(run);
|
|
161
|
+
if (time !== undefined)
|
|
162
|
+
return time;
|
|
163
|
+
}
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Non-negative elapsed time, or undefined when either endpoint is unusable.
|
|
168
|
+
* @param from - earlier epoch ms.
|
|
169
|
+
* @param to - later epoch ms.
|
|
170
|
+
* @returns the duration, clamped to `>= 0`.
|
|
171
|
+
*/
|
|
172
|
+
function elapsed(from, to) {
|
|
173
|
+
if (typeof from !== 'number' || typeof to !== 'number')
|
|
174
|
+
return undefined;
|
|
175
|
+
if (!Number.isFinite(from) || !Number.isFinite(to))
|
|
176
|
+
return undefined;
|
|
177
|
+
return Math.max(0, to - from);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Output speed in tokens/second.
|
|
181
|
+
* @param outputTokens - provider-reported output tokens.
|
|
182
|
+
* @param decodeMs - generation duration.
|
|
183
|
+
* @returns the rate, or undefined when the denominator is unusable/zero.
|
|
184
|
+
*/
|
|
185
|
+
function tokensPerSecondOf(outputTokens, decodeMs) {
|
|
186
|
+
if (!Number.isFinite(outputTokens) || outputTokens < 0)
|
|
187
|
+
return undefined;
|
|
188
|
+
if (decodeMs === undefined || decodeMs <= 0)
|
|
189
|
+
return undefined;
|
|
190
|
+
const rate = (outputTokens / decodeMs) * 1000;
|
|
191
|
+
return Number.isFinite(rate) ? rate : undefined;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Hard bounds for activity tracking inside one Turn.
|
|
195
|
+
*
|
|
196
|
+
* The collector runs on the conversation's event path. A buggy or hostile tool
|
|
197
|
+
* loop must not be able to grow plugin-only maps without limit; tools beyond the
|
|
198
|
+
* cap are folded into one `(其它)` row (and pending calls beyond the pairing cap
|
|
199
|
+
* simply stop being paired). Token accounting is unaffected.
|
|
200
|
+
*/
|
|
201
|
+
const MAX_TRACKED_TOOL_NAMES = 199;
|
|
202
|
+
const MAX_PENDING_TOOL_CALLS = 2_000;
|
|
203
|
+
const OTHER_TOOL_NAME = '(其它)';
|
|
204
|
+
const MAX_RETRY_CODES = 20;
|
|
73
205
|
/**
|
|
74
206
|
* Validate and normalize one raw usage sample, mirroring upstream's
|
|
75
207
|
* `normalizeUsage`. Anything the provider left inconsistent — a missing
|
|
@@ -166,6 +298,22 @@ export class TurnUsageCollector {
|
|
|
166
298
|
reason = '';
|
|
167
299
|
/** Event time of the event currently being consumed; stamps settled attempts. */
|
|
168
300
|
eventTime = 0;
|
|
301
|
+
/** Current step number and its opening time. */
|
|
302
|
+
step = 0;
|
|
303
|
+
stepStartedAt = 0;
|
|
304
|
+
/** Start of the current attempt (step/start or llm/retry-started). */
|
|
305
|
+
attemptStartedAt = 0;
|
|
306
|
+
/** Calls awaiting their `tool/result`, keyed by call id. */
|
|
307
|
+
toolCalls = new Map();
|
|
308
|
+
/** Calls whose result was already counted (protects against duplicates). */
|
|
309
|
+
toolResults = new Set();
|
|
310
|
+
/** Per-tool-name activity for this Turn. */
|
|
311
|
+
toolStats = new Map();
|
|
312
|
+
/** Provider-routed retries scheduled in this Turn. */
|
|
313
|
+
retries = 0;
|
|
314
|
+
retryCodes = new Set();
|
|
315
|
+
/** Any settled assistant message marked interrupted. */
|
|
316
|
+
interrupted = false;
|
|
169
317
|
/**
|
|
170
318
|
* Whether a Turn is currently being collected.
|
|
171
319
|
* @returns true while an observed Turn is open.
|
|
@@ -207,6 +355,15 @@ export class TurnUsageCollector {
|
|
|
207
355
|
this.endedAt = 0;
|
|
208
356
|
this.reason = '';
|
|
209
357
|
this.eventTime = 0;
|
|
358
|
+
this.step = 0;
|
|
359
|
+
this.stepStartedAt = 0;
|
|
360
|
+
this.attemptStartedAt = 0;
|
|
361
|
+
this.toolCalls = new Map();
|
|
362
|
+
this.toolResults = new Set();
|
|
363
|
+
this.toolStats = new Map();
|
|
364
|
+
this.retries = 0;
|
|
365
|
+
this.retryCodes = new Set();
|
|
366
|
+
this.interrupted = false;
|
|
210
367
|
}
|
|
211
368
|
/**
|
|
212
369
|
* The state machine itself; kept separate so `push` can armor it.
|
|
@@ -250,10 +407,15 @@ export class TurnUsageCollector {
|
|
|
250
407
|
switch (type) {
|
|
251
408
|
case 'step/start': {
|
|
252
409
|
const data = dataOf(event);
|
|
253
|
-
if (data.turn !== this.turn || this.state.kind !== 'idle')
|
|
410
|
+
if (data.turn !== this.turn || this.state.kind !== 'idle') {
|
|
254
411
|
this.invalid = true;
|
|
255
|
-
|
|
412
|
+
}
|
|
413
|
+
else {
|
|
414
|
+
this.step = data.step;
|
|
415
|
+
this.stepStartedAt = event.time;
|
|
416
|
+
this.attemptStartedAt = event.time;
|
|
256
417
|
this.state = { kind: 'open', step: data.step, sample: undefined };
|
|
418
|
+
}
|
|
257
419
|
return { kind: 'collecting' };
|
|
258
420
|
}
|
|
259
421
|
case 'llm/retry-started': {
|
|
@@ -265,6 +427,7 @@ export class TurnUsageCollector {
|
|
|
265
427
|
this.invalid = true;
|
|
266
428
|
}
|
|
267
429
|
else {
|
|
430
|
+
this.attemptStartedAt = event.time;
|
|
268
431
|
this.state = { kind: 'open', step: data.step, sample: undefined };
|
|
269
432
|
}
|
|
270
433
|
return { kind: 'collecting' };
|
|
@@ -278,7 +441,8 @@ export class TurnUsageCollector {
|
|
|
278
441
|
const state = this.state;
|
|
279
442
|
const sample = streamUsage(data.stream) ?? state.sample;
|
|
280
443
|
this.state = { kind: 'open', step: data.step, sample };
|
|
281
|
-
|
|
444
|
+
const firstTokenAt = firstTokenTimeOf(data.stream);
|
|
445
|
+
if (!this.closeOpen('', '', 'attempt', firstTokenAt, false))
|
|
282
446
|
this.invalid = true;
|
|
283
447
|
else
|
|
284
448
|
this.state = { kind: 'finishClosed', step: data.step };
|
|
@@ -295,10 +459,16 @@ export class TurnUsageCollector {
|
|
|
295
459
|
this.state =
|
|
296
460
|
sample === undefined ? state : { kind: 'open', step: data.step, sample };
|
|
297
461
|
const route = messageRoute(data.message);
|
|
298
|
-
|
|
462
|
+
const firstTokenAt = firstTokenTimeOf(data.stream);
|
|
463
|
+
const interrupted = data.interrupted === true;
|
|
464
|
+
if (interrupted)
|
|
465
|
+
this.interrupted = true;
|
|
466
|
+
if (!this.closeOpen(route.provider, route.model, 'message', firstTokenAt, interrupted)) {
|
|
299
467
|
this.invalid = true;
|
|
300
|
-
|
|
468
|
+
}
|
|
469
|
+
else {
|
|
301
470
|
this.state = { kind: 'settled', step: data.step, by: 'message' };
|
|
471
|
+
}
|
|
302
472
|
return { kind: 'collecting' };
|
|
303
473
|
}
|
|
304
474
|
case 'llm/retry': {
|
|
@@ -307,8 +477,16 @@ export class TurnUsageCollector {
|
|
|
307
477
|
this.invalid = true;
|
|
308
478
|
return { kind: 'collecting' };
|
|
309
479
|
}
|
|
480
|
+
this.retries += 1;
|
|
481
|
+
const code = data.failure?.code;
|
|
482
|
+
if (typeof code === 'string') {
|
|
483
|
+
const trimmed = code.trim();
|
|
484
|
+
if (trimmed.length > 0 && this.retryCodes.size < MAX_RETRY_CODES) {
|
|
485
|
+
this.retryCodes.add(trimmed.slice(0, 40));
|
|
486
|
+
}
|
|
487
|
+
}
|
|
310
488
|
if (this.state.kind === 'settled' ||
|
|
311
|
-
(this.state.kind === 'open' && !this.closeOpen('', ''))) {
|
|
489
|
+
(this.state.kind === 'open' && !this.closeOpen('', '', 'retry', undefined, false))) {
|
|
312
490
|
this.invalid = true;
|
|
313
491
|
}
|
|
314
492
|
if (!this.invalid)
|
|
@@ -321,12 +499,67 @@ export class TurnUsageCollector {
|
|
|
321
499
|
this.invalid = true;
|
|
322
500
|
return { kind: 'collecting' };
|
|
323
501
|
}
|
|
324
|
-
if (this.state.kind === 'open' && !this.closeOpen('', ''))
|
|
502
|
+
if (this.state.kind === 'open' && !this.closeOpen('', '', 'step-end', undefined, false)) {
|
|
325
503
|
this.invalid = true;
|
|
504
|
+
}
|
|
326
505
|
if (!this.invalid)
|
|
327
506
|
this.state = { kind: 'idle' };
|
|
328
507
|
return { kind: 'collecting' };
|
|
329
508
|
}
|
|
509
|
+
case 'tool/call': {
|
|
510
|
+
const data = dataOf(event);
|
|
511
|
+
if (data.turn === this.turn && typeof data.callId === 'string' && typeof data.name === 'string') {
|
|
512
|
+
const rawName = data.name.trim().slice(0, 100);
|
|
513
|
+
if (data.callId.length > 0 && rawName.length > 0) {
|
|
514
|
+
const name = !this.toolStats.has(rawName) && this.toolStats.size >= MAX_TRACKED_TOOL_NAMES
|
|
515
|
+
? OTHER_TOOL_NAME
|
|
516
|
+
: rawName;
|
|
517
|
+
if (this.toolCalls.size < MAX_PENDING_TOOL_CALLS) {
|
|
518
|
+
this.toolCalls.set(data.callId, { name, time: this.eventTime });
|
|
519
|
+
}
|
|
520
|
+
const stat = this.toolStats.get(name) ?? {
|
|
521
|
+
name,
|
|
522
|
+
calls: 0,
|
|
523
|
+
results: 0,
|
|
524
|
+
errors: 0,
|
|
525
|
+
durationMs: 0,
|
|
526
|
+
};
|
|
527
|
+
stat.calls += 1;
|
|
528
|
+
this.toolStats.set(name, stat);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
return { kind: 'collecting' };
|
|
532
|
+
}
|
|
533
|
+
case 'tool/result': {
|
|
534
|
+
const data = dataOf(event);
|
|
535
|
+
if (data.turn !== this.turn)
|
|
536
|
+
return { kind: 'collecting' };
|
|
537
|
+
const callId = data.message?.toolCallId;
|
|
538
|
+
if (typeof callId !== 'string' || callId.length === 0)
|
|
539
|
+
return { kind: 'collecting' };
|
|
540
|
+
if (this.toolResults.has(callId))
|
|
541
|
+
return { kind: 'collecting' };
|
|
542
|
+
const call = this.toolCalls.get(callId);
|
|
543
|
+
if (call === undefined)
|
|
544
|
+
return { kind: 'collecting' };
|
|
545
|
+
this.toolResults.add(callId);
|
|
546
|
+
this.toolCalls.delete(callId);
|
|
547
|
+
const stat = this.toolStats.get(call.name) ?? {
|
|
548
|
+
name: call.name,
|
|
549
|
+
calls: 0,
|
|
550
|
+
results: 0,
|
|
551
|
+
errors: 0,
|
|
552
|
+
durationMs: 0,
|
|
553
|
+
};
|
|
554
|
+
stat.results += 1;
|
|
555
|
+
if (data.message?.isError === true)
|
|
556
|
+
stat.errors += 1;
|
|
557
|
+
const duration = elapsed(call.time, this.eventTime);
|
|
558
|
+
if (duration !== undefined)
|
|
559
|
+
stat.durationMs += duration;
|
|
560
|
+
this.toolStats.set(call.name, stat);
|
|
561
|
+
return { kind: 'collecting' };
|
|
562
|
+
}
|
|
330
563
|
default:
|
|
331
564
|
// Every other event type is off the usage path and leaves the fold
|
|
332
565
|
// untouched, exactly as upstream's loop does.
|
|
@@ -347,13 +580,17 @@ export class TurnUsageCollector {
|
|
|
347
580
|
*
|
|
348
581
|
* The sample is normalized HERE rather than on arrival so a raw sample can be
|
|
349
582
|
* carried across the `assistant/attempt` → `assistant/message` pair exactly
|
|
350
|
-
* as upstream does.
|
|
583
|
+
* as upstream does. Timing facts are read from the event that closes the
|
|
584
|
+
* attempt and never retained as a stream.
|
|
351
585
|
*
|
|
352
586
|
* @param provider - route provider for the settled attempt.
|
|
353
587
|
* @param model - route model for the settled attempt.
|
|
588
|
+
* @param settledBy - which event closed the attempt.
|
|
589
|
+
* @param firstTokenAt - first token time of this attempt's stream, if any.
|
|
590
|
+
* @param interrupted - whether the settling assistant message was interrupted.
|
|
354
591
|
* @returns true when the attempt was recorded.
|
|
355
592
|
*/
|
|
356
|
-
closeOpen(provider, model) {
|
|
593
|
+
closeOpen(provider, model, settledBy, firstTokenAt, interrupted) {
|
|
357
594
|
if (this.state.kind !== 'open')
|
|
358
595
|
return false;
|
|
359
596
|
const sample = this.state.sample;
|
|
@@ -362,7 +599,27 @@ export class TurnUsageCollector {
|
|
|
362
599
|
const normalized = normalizeUsage(sample);
|
|
363
600
|
if (normalized === undefined)
|
|
364
601
|
return false;
|
|
365
|
-
|
|
602
|
+
const at = this.eventTime;
|
|
603
|
+
const startedAt = this.attemptStartedAt > 0 ? this.attemptStartedAt : this.stepStartedAt > 0 ? this.stepStartedAt : at;
|
|
604
|
+
const ttftMs = firstTokenAt === undefined ? undefined : elapsed(startedAt, firstTokenAt);
|
|
605
|
+
const decodeMs = firstTokenAt === undefined ? undefined : elapsed(firstTokenAt, at);
|
|
606
|
+
const durationMs = elapsed(startedAt, at);
|
|
607
|
+
const rate = tokensPerSecondOf(normalized.buckets.output, decodeMs);
|
|
608
|
+
this.attempts.push({
|
|
609
|
+
...normalized,
|
|
610
|
+
at,
|
|
611
|
+
provider,
|
|
612
|
+
model,
|
|
613
|
+
step: this.step,
|
|
614
|
+
startedAt,
|
|
615
|
+
...(firstTokenAt === undefined ? {} : { firstTokenAt }),
|
|
616
|
+
...(ttftMs === undefined ? {} : { ttftMs }),
|
|
617
|
+
...(decodeMs === undefined ? {} : { decodeMs }),
|
|
618
|
+
...(durationMs === undefined ? {} : { durationMs }),
|
|
619
|
+
...(rate === undefined ? {} : { tokensPerSecond: rate }),
|
|
620
|
+
settledBy,
|
|
621
|
+
...(interrupted ? { interrupted: true } : {}),
|
|
622
|
+
});
|
|
366
623
|
return true;
|
|
367
624
|
}
|
|
368
625
|
/**
|
|
@@ -401,7 +658,25 @@ export class TurnUsageCollector {
|
|
|
401
658
|
at: attempt.at,
|
|
402
659
|
tokens: attempt.buckets,
|
|
403
660
|
total: attempt.total,
|
|
661
|
+
step: attempt.step,
|
|
662
|
+
startedAt: attempt.startedAt,
|
|
663
|
+
settledBy: attempt.settledBy,
|
|
664
|
+
reportedCacheRead: attempt.reportedCacheRead,
|
|
665
|
+
reportedCacheWrite: attempt.reportedCacheWrite,
|
|
666
|
+
reportedReasoning: attempt.reportedReasoning,
|
|
667
|
+
...(attempt.firstTokenAt === undefined ? {} : { firstTokenAt: attempt.firstTokenAt }),
|
|
668
|
+
...(attempt.ttftMs === undefined ? {} : { ttftMs: attempt.ttftMs }),
|
|
669
|
+
...(attempt.decodeMs === undefined ? {} : { decodeMs: attempt.decodeMs }),
|
|
670
|
+
...(attempt.durationMs === undefined ? {} : { durationMs: attempt.durationMs }),
|
|
671
|
+
...(attempt.tokensPerSecond === undefined
|
|
672
|
+
? {}
|
|
673
|
+
: { tokensPerSecond: attempt.tokensPerSecond }),
|
|
674
|
+
...(attempt.interrupted === true ? { interrupted: true } : {}),
|
|
404
675
|
}));
|
|
676
|
+
const tools = [...this.toolStats.values()]
|
|
677
|
+
.filter((tool) => tool.calls > 0 || tool.results > 0)
|
|
678
|
+
.sort((left, right) => right.calls - left.calls || left.name.localeCompare(right.name))
|
|
679
|
+
.slice(0, 200);
|
|
405
680
|
return {
|
|
406
681
|
turn,
|
|
407
682
|
reason: this.reason,
|
|
@@ -416,6 +691,10 @@ export class TurnUsageCollector {
|
|
|
416
691
|
reasoning: reasoning ?? 0,
|
|
417
692
|
},
|
|
418
693
|
total: totals,
|
|
694
|
+
tools,
|
|
695
|
+
retries: this.retries,
|
|
696
|
+
retryCodes: [...this.retryCodes].sort(),
|
|
697
|
+
interrupted: this.interrupted,
|
|
419
698
|
};
|
|
420
699
|
}
|
|
421
700
|
}
|
package/lib/host/ledger-io.d.ts
CHANGED
|
@@ -27,11 +27,31 @@ export interface LedgerKeys {
|
|
|
27
27
|
keys: Set<string>;
|
|
28
28
|
/** Fork fingerprint of every record. */
|
|
29
29
|
fingerprints: Set<string>;
|
|
30
|
+
/**
|
|
31
|
+
* Canonical record per key, newest line wins (the same rule the reader uses).
|
|
32
|
+
*
|
|
33
|
+
* Backfill needs the existing record to decide whether a v1 line can be
|
|
34
|
+
* upgraded with v2 timing/activity fields; it never rewrites the tokens/cost.
|
|
35
|
+
*/
|
|
36
|
+
recordsByKey: Map<string, UsageRecord>;
|
|
30
37
|
/** Parse tallies. */
|
|
31
38
|
malformedLines: number;
|
|
32
39
|
/** Unsupported versions. */
|
|
33
40
|
unsupportedRecords: number;
|
|
34
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Collapse repeated `(sessionId, turn)` lines, keeping the newest one.
|
|
44
|
+
*
|
|
45
|
+
* The ledger is append-only, so an enriched v1 record is followed by its v2
|
|
46
|
+
* upgrade. The dashboard reader already applies newest-wins; a backup must use
|
|
47
|
+
* the same rule, otherwise a restore would keep the older line and silently
|
|
48
|
+
* drop the v2 fields. The position of the first occurrence is preserved so the
|
|
49
|
+
* exported order still reads chronologically.
|
|
50
|
+
*
|
|
51
|
+
* @param records - records in file order (oldest first).
|
|
52
|
+
* @returns unique records, newest content per key.
|
|
53
|
+
*/
|
|
54
|
+
export declare function dedupeRecordsNewest(records: readonly UsageRecord[]): UsageRecord[];
|
|
35
55
|
/** Validation outcome for one backup payload. */
|
|
36
56
|
export interface ParsedBackup {
|
|
37
57
|
/** Canonical records accepted by the schema reader. */
|
|
@@ -107,7 +127,14 @@ export declare function createBackup(path: string, pluginVersion: string): Promi
|
|
|
107
127
|
export declare function parseBackup(value: unknown): ParsedBackup;
|
|
108
128
|
/**
|
|
109
129
|
* Select records that are not already present (and de-duplicate the file itself).
|
|
110
|
-
*
|
|
130
|
+
*
|
|
131
|
+
* A backup produced by an older build (or a hand-edited file) may still contain
|
|
132
|
+
* two lines for one Turn. The import keeps the LAST one, mirroring the ledger
|
|
133
|
+
* reader's newest-wins rule, so a v2 upgrade is never displaced by its older v1
|
|
134
|
+
* source. Internal duplicates are counted in `duplicates`, like duplicates
|
|
135
|
+
* already present in the target ledger.
|
|
136
|
+
*
|
|
137
|
+
* @param records - validated records from the backup, in file order.
|
|
111
138
|
* @param existing - duplicate-tracking view of the current ledger.
|
|
112
139
|
* @returns new records and the duplicate count.
|
|
113
140
|
*/
|