dsh-token-usage-ledger 0.0.0-stage → 0.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1067 -0
- package/LICENSE +21 -0
- package/README.md +237 -2
- package/cordis.patch.yml +30 -0
- package/lib/client.js +4469 -0
- package/lib/host/aggregate.d.ts +132 -0
- package/lib/host/aggregate.js +840 -0
- package/lib/host/api.d.ts +118 -0
- package/lib/host/api.js +346 -0
- package/lib/host/backfill.d.ts +54 -0
- package/lib/host/backfill.js +265 -0
- package/lib/host/collector.d.ts +139 -0
- package/lib/host/collector.js +441 -0
- package/lib/host/ledger-io.d.ts +125 -0
- package/lib/host/ledger-io.js +211 -0
- package/lib/host/operation-error.d.ts +22 -0
- package/lib/host/operation-error.js +28 -0
- package/lib/host/paths.d.ts +35 -0
- package/lib/host/paths.js +63 -0
- package/lib/host/pricing-module.d.ts +112 -0
- package/lib/host/pricing-module.js +63 -0
- package/lib/host/pricing-modules/deepseek-2026-09-10.d.ts +31 -0
- package/lib/host/pricing-modules/deepseek-2026-09-10.js +67 -0
- package/lib/host/pricing-modules/index.d.ts +27 -0
- package/lib/host/pricing-modules/index.js +30 -0
- package/lib/host/pricing.d.ts +192 -0
- package/lib/host/pricing.js +287 -0
- package/lib/host/records.d.ts +47 -0
- package/lib/host/records.js +116 -0
- package/lib/host/runtime-marker.d.ts +67 -0
- package/lib/host/runtime-marker.js +71 -0
- package/lib/host/schema.d.ts +77 -0
- package/lib/host/schema.js +158 -0
- package/lib/host/store.d.ts +171 -0
- package/lib/host/store.js +337 -0
- package/lib/host/workspace-titles.d.ts +38 -0
- package/lib/host/workspace-titles.js +136 -0
- package/lib/host/zstd.d.ts +41 -0
- package/lib/host/zstd.js +134 -0
- package/lib/index.d.ts +82 -0
- package/lib/index.js +512 -0
- package/lib/shared/format.d.ts +415 -0
- package/lib/shared/format.js +714 -0
- package/lib/shared/protocol.d.ts +646 -0
- package/lib/shared/protocol.js +129 -0
- package/package.json +85 -4
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming Turn usage fold — the exact-accounting core of the plugin.
|
|
3
|
+
*
|
|
4
|
+
* This is a faithful, incremental re-implementation of the harness's own
|
|
5
|
+
* `deriveTurnTokenUsage` (`@deepseek-ai/dsh-token-meter/client`): same lifecycle
|
|
6
|
+
* state machine, same validations, same "prove it or drop it" posture. It is
|
|
7
|
+
* re-implemented rather than imported on purpose — see README "版本兼容与升级
|
|
8
|
+
* 要点": an out-of-tree plugin that imports an internal package at runtime
|
|
9
|
+
* inherits that package's resolution and version coupling, while this fold is a
|
|
10
|
+
* few hundred lines of pure arithmetic that cannot break the conversation even
|
|
11
|
+
* if the upstream package moves. `tests/conformance.test.mjs` pins the two
|
|
12
|
+
* against each other on synthetic logs whenever the upstream package is
|
|
13
|
+
* resolvable.
|
|
14
|
+
*
|
|
15
|
+
* Cost of the streaming shape: the collector NEVER retains an event or a
|
|
16
|
+
* stream. It reads the usage sample out of the durable stream at push time and
|
|
17
|
+
* keeps only scalars, so a long Turn costs a constant number of bytes here.
|
|
18
|
+
*
|
|
19
|
+
* @module dsh-token-usage-ledger/host/collector
|
|
20
|
+
*/
|
|
21
|
+
/** `Number.isSafeInteger(value) && value >= 0`, the upstream count predicate. */
|
|
22
|
+
function isCount(value) {
|
|
23
|
+
return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Sum counts, refusing to overflow the safe-integer range.
|
|
27
|
+
* @param values - counts to add.
|
|
28
|
+
* @returns the exact sum, or undefined when any partial sum is unsafe.
|
|
29
|
+
*/
|
|
30
|
+
function safeSum(values) {
|
|
31
|
+
let total = 0;
|
|
32
|
+
for (const value of values) {
|
|
33
|
+
total += value;
|
|
34
|
+
if (!Number.isSafeInteger(total))
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
return total;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Read the LAST raw `usage` chunk out of a compact assistant stream.
|
|
41
|
+
*
|
|
42
|
+
* Matches upstream's `lastAssistantStreamChunk(stream, 'usage')` exactly: the
|
|
43
|
+
* scan is by chunk TYPE only, and it returns on the first match from the tail
|
|
44
|
+
* WITHOUT inspecting the payload. That distinction matters — skipping an empty
|
|
45
|
+
* trailing usage chunk and continuing backwards would resurrect an older sample
|
|
46
|
+
* from the same stream, which upstream never does. Here an empty tail sample
|
|
47
|
+
* simply propagates as `null`/`undefined` and the caller's `?? state.sample`
|
|
48
|
+
* fallback (or a refusal) decides, exactly as upstream's does.
|
|
49
|
+
*
|
|
50
|
+
* @param stream - compact stream records from one attempt.
|
|
51
|
+
* @returns the last usage chunk's `usage` value, verbatim.
|
|
52
|
+
*/
|
|
53
|
+
function streamUsage(stream) {
|
|
54
|
+
if (!Array.isArray(stream))
|
|
55
|
+
return undefined;
|
|
56
|
+
for (let index = stream.length - 1; index >= 0; index -= 1) {
|
|
57
|
+
const record = stream[index];
|
|
58
|
+
if (typeof record !== 'object' || record === null)
|
|
59
|
+
continue;
|
|
60
|
+
const candidate = record;
|
|
61
|
+
if (candidate.type !== 'chunk')
|
|
62
|
+
continue;
|
|
63
|
+
const chunk = candidate.chunk;
|
|
64
|
+
if (typeof chunk !== 'object' || chunk === null)
|
|
65
|
+
continue;
|
|
66
|
+
const typed = chunk;
|
|
67
|
+
if (typed.type !== 'usage')
|
|
68
|
+
continue;
|
|
69
|
+
return typed.usage;
|
|
70
|
+
}
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Validate and normalize one raw usage sample, mirroring upstream's
|
|
75
|
+
* `normalizeUsage`. Anything the provider left inconsistent — a missing
|
|
76
|
+
* required count, a reasoning count above the output count, a declared total
|
|
77
|
+
* that contradicts the buckets — makes the attempt unavailable instead of
|
|
78
|
+
* producing a plausible-looking wrong number.
|
|
79
|
+
*
|
|
80
|
+
* @param usage - raw provider usage.
|
|
81
|
+
* @returns the normalized attempt, or undefined when it cannot be proven.
|
|
82
|
+
*/
|
|
83
|
+
function normalizeUsage(usage) {
|
|
84
|
+
// Upstream destructures unguarded, so a `usage: null` chunk throws out of
|
|
85
|
+
// `deriveTurnTokenUsage`. This fold runs inside the session dispatch and must
|
|
86
|
+
// not throw, so the same situation REFUSES the attempt instead — the turn is
|
|
87
|
+
// dropped, which is the outcome upstream's caller sees either way.
|
|
88
|
+
if (typeof usage !== 'object' || usage === null)
|
|
89
|
+
return undefined;
|
|
90
|
+
const { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens, totalTokens, } = usage;
|
|
91
|
+
if (!isCount(inputTokens) || !isCount(outputTokens))
|
|
92
|
+
return undefined;
|
|
93
|
+
if (cacheReadTokens !== undefined && !isCount(cacheReadTokens))
|
|
94
|
+
return undefined;
|
|
95
|
+
if (cacheWriteTokens !== undefined && !isCount(cacheWriteTokens))
|
|
96
|
+
return undefined;
|
|
97
|
+
if (reasoningTokens !== undefined &&
|
|
98
|
+
(!isCount(reasoningTokens) || reasoningTokens > outputTokens)) {
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
const knownPrompt = safeSum([
|
|
102
|
+
inputTokens,
|
|
103
|
+
...(cacheReadTokens === undefined ? [] : [cacheReadTokens]),
|
|
104
|
+
...(cacheWriteTokens === undefined ? [] : [cacheWriteTokens]),
|
|
105
|
+
]);
|
|
106
|
+
if (knownPrompt === undefined)
|
|
107
|
+
return undefined;
|
|
108
|
+
let exactTotal;
|
|
109
|
+
if (totalTokens !== undefined) {
|
|
110
|
+
if (!isCount(totalTokens))
|
|
111
|
+
return undefined;
|
|
112
|
+
const exactPrompt = totalTokens - outputTokens;
|
|
113
|
+
if (!isCount(exactPrompt) || exactPrompt < knownPrompt)
|
|
114
|
+
return undefined;
|
|
115
|
+
if (cacheReadTokens !== undefined &&
|
|
116
|
+
cacheWriteTokens !== undefined &&
|
|
117
|
+
exactPrompt !== knownPrompt) {
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
exactTotal = totalTokens;
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
// Without a declared total both optional buckets must be present, or the
|
|
124
|
+
// prompt size is unknowable.
|
|
125
|
+
if (cacheReadTokens === undefined || cacheWriteTokens === undefined)
|
|
126
|
+
return undefined;
|
|
127
|
+
const derivedTotal = safeSum([knownPrompt, outputTokens]);
|
|
128
|
+
if (derivedTotal === undefined)
|
|
129
|
+
return undefined;
|
|
130
|
+
exactTotal = derivedTotal;
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
buckets: {
|
|
134
|
+
uncachedInput: inputTokens,
|
|
135
|
+
cacheRead: cacheReadTokens ?? 0,
|
|
136
|
+
cacheWrite: cacheWriteTokens ?? 0,
|
|
137
|
+
output: outputTokens,
|
|
138
|
+
reasoning: reasoningTokens ?? 0,
|
|
139
|
+
},
|
|
140
|
+
total: exactTotal,
|
|
141
|
+
reportedCacheRead: cacheReadTokens !== undefined,
|
|
142
|
+
reportedCacheWrite: cacheWriteTokens !== undefined,
|
|
143
|
+
reportedReasoning: reasoningTokens !== undefined,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
/** Narrow an event's `data` for one event type. */
|
|
147
|
+
function dataOf(event) {
|
|
148
|
+
return event.data;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Fold one Session's live event feed into exact per-Turn accounting.
|
|
152
|
+
*
|
|
153
|
+
* One instance per Session. The owner feeds every `session/event` it observes
|
|
154
|
+
* and reacts to the `'closed'` result. Collection begins only at a `turn/start`
|
|
155
|
+
* the collector actually saw, so attaching mid-turn drops that Turn instead of
|
|
156
|
+
* recording a half-observed one.
|
|
157
|
+
*/
|
|
158
|
+
export class TurnUsageCollector {
|
|
159
|
+
state = { kind: 'idle' };
|
|
160
|
+
attempts = [];
|
|
161
|
+
turn = undefined;
|
|
162
|
+
sawEnd = false;
|
|
163
|
+
invalid = false;
|
|
164
|
+
startedAt = 0;
|
|
165
|
+
endedAt = 0;
|
|
166
|
+
reason = '';
|
|
167
|
+
/** Event time of the event currently being consumed; stamps settled attempts. */
|
|
168
|
+
eventTime = 0;
|
|
169
|
+
/**
|
|
170
|
+
* Whether a Turn is currently being collected.
|
|
171
|
+
* @returns true while an observed Turn is open.
|
|
172
|
+
*/
|
|
173
|
+
get collecting() {
|
|
174
|
+
return this.turn !== undefined;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Consume one durable session event.
|
|
178
|
+
*
|
|
179
|
+
* Never throws for any event shape: a malformed event marks the Turn
|
|
180
|
+
* underivable, which drops it, rather than escaping into the caller's
|
|
181
|
+
* dispatch. That containment is what keeps this plugin off the conversation's
|
|
182
|
+
* failure path.
|
|
183
|
+
*
|
|
184
|
+
* @param event - the appended session event, exactly as published.
|
|
185
|
+
* @returns `ignored`, `collecting`, or `closed` with the outcome.
|
|
186
|
+
*/
|
|
187
|
+
push(event) {
|
|
188
|
+
const wasCollecting = this.turn !== undefined;
|
|
189
|
+
try {
|
|
190
|
+
return this.consume(event);
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
// Defensive: a shape this build does not understand must never escape
|
|
194
|
+
// into the session dispatch. Drop the Turn and carry on.
|
|
195
|
+
this.reset();
|
|
196
|
+
return wasCollecting ? { kind: 'closed', outcome: undefined } : { kind: 'ignored' };
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/** Discard all state; the next observed `turn/start` begins a fresh Turn. */
|
|
200
|
+
reset() {
|
|
201
|
+
this.state = { kind: 'idle' };
|
|
202
|
+
this.attempts = [];
|
|
203
|
+
this.turn = undefined;
|
|
204
|
+
this.sawEnd = false;
|
|
205
|
+
this.invalid = false;
|
|
206
|
+
this.startedAt = 0;
|
|
207
|
+
this.endedAt = 0;
|
|
208
|
+
this.reason = '';
|
|
209
|
+
this.eventTime = 0;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* The state machine itself; kept separate so `push` can armor it.
|
|
213
|
+
* @param event - event to fold.
|
|
214
|
+
* @returns the fold's progress.
|
|
215
|
+
*/
|
|
216
|
+
consume(event) {
|
|
217
|
+
// Widened to `string` because `SessionEventMap` is merge-extensible: the
|
|
218
|
+
// retry events below are contributed by an optional plugin whose type
|
|
219
|
+
// package this build does not depend on, yet must still be handled when it
|
|
220
|
+
// is mounted.
|
|
221
|
+
const type = event.type;
|
|
222
|
+
this.eventTime = event.time;
|
|
223
|
+
// Idle: nothing is collected until this collector OBSERVES a turn/start, so
|
|
224
|
+
// a session adopted mid-Turn contributes no half-measured record.
|
|
225
|
+
if (this.turn === undefined) {
|
|
226
|
+
if (type !== 'turn/start')
|
|
227
|
+
return { kind: 'ignored' };
|
|
228
|
+
this.turn = dataOf(event).turn;
|
|
229
|
+
this.startedAt = event.time;
|
|
230
|
+
return { kind: 'collecting' };
|
|
231
|
+
}
|
|
232
|
+
if (type === 'turn/end') {
|
|
233
|
+
const data = dataOf(event);
|
|
234
|
+
// Upstream refuses a closer that names another turn or arrives while a
|
|
235
|
+
// step is still open; `buildOutcome` already covers the open-step half.
|
|
236
|
+
if (data.turn !== this.turn)
|
|
237
|
+
this.invalid = true;
|
|
238
|
+
this.endedAt = event.time;
|
|
239
|
+
this.reason = typeof data.reason?.kind === 'string' ? data.reason.kind : 'unknown';
|
|
240
|
+
this.sawEnd = true;
|
|
241
|
+
const outcome = this.buildOutcome();
|
|
242
|
+
this.reset();
|
|
243
|
+
return { kind: 'closed', outcome };
|
|
244
|
+
}
|
|
245
|
+
if (this.sawEnd) {
|
|
246
|
+
// An event after the closer contradicts the log; drop the Turn.
|
|
247
|
+
this.invalid = true;
|
|
248
|
+
return { kind: 'collecting' };
|
|
249
|
+
}
|
|
250
|
+
switch (type) {
|
|
251
|
+
case 'step/start': {
|
|
252
|
+
const data = dataOf(event);
|
|
253
|
+
if (data.turn !== this.turn || this.state.kind !== 'idle')
|
|
254
|
+
this.invalid = true;
|
|
255
|
+
else
|
|
256
|
+
this.state = { kind: 'open', step: data.step, sample: undefined };
|
|
257
|
+
return { kind: 'collecting' };
|
|
258
|
+
}
|
|
259
|
+
case 'llm/retry-started': {
|
|
260
|
+
const data = dataOf(event);
|
|
261
|
+
if (data.turn !== this.turn ||
|
|
262
|
+
this.state.kind !== 'settled' ||
|
|
263
|
+
this.state.by !== 'retry' ||
|
|
264
|
+
this.state.step !== data.step) {
|
|
265
|
+
this.invalid = true;
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
this.state = { kind: 'open', step: data.step, sample: undefined };
|
|
269
|
+
}
|
|
270
|
+
return { kind: 'collecting' };
|
|
271
|
+
}
|
|
272
|
+
case 'assistant/attempt': {
|
|
273
|
+
const data = dataOf(event);
|
|
274
|
+
if (!this.sameAttempt(data.turn, data.step)) {
|
|
275
|
+
this.invalid = true;
|
|
276
|
+
return { kind: 'collecting' };
|
|
277
|
+
}
|
|
278
|
+
const state = this.state;
|
|
279
|
+
const sample = streamUsage(data.stream) ?? state.sample;
|
|
280
|
+
this.state = { kind: 'open', step: data.step, sample };
|
|
281
|
+
if (!this.closeOpen('', ''))
|
|
282
|
+
this.invalid = true;
|
|
283
|
+
else
|
|
284
|
+
this.state = { kind: 'finishClosed', step: data.step };
|
|
285
|
+
return { kind: 'collecting' };
|
|
286
|
+
}
|
|
287
|
+
case 'assistant/message': {
|
|
288
|
+
const data = dataOf(event);
|
|
289
|
+
if (!this.sameAttempt(data.turn, data.step)) {
|
|
290
|
+
this.invalid = true;
|
|
291
|
+
return { kind: 'collecting' };
|
|
292
|
+
}
|
|
293
|
+
const state = this.state;
|
|
294
|
+
const sample = data.usage ?? streamUsage(data.stream);
|
|
295
|
+
this.state =
|
|
296
|
+
sample === undefined ? state : { kind: 'open', step: data.step, sample };
|
|
297
|
+
const route = messageRoute(data.message);
|
|
298
|
+
if (!this.closeOpen(route.provider, route.model))
|
|
299
|
+
this.invalid = true;
|
|
300
|
+
else
|
|
301
|
+
this.state = { kind: 'settled', step: data.step, by: 'message' };
|
|
302
|
+
return { kind: 'collecting' };
|
|
303
|
+
}
|
|
304
|
+
case 'llm/retry': {
|
|
305
|
+
const data = dataOf(event);
|
|
306
|
+
if (data.turn !== this.turn || this.state.kind === 'idle' || this.state.step !== data.step) {
|
|
307
|
+
this.invalid = true;
|
|
308
|
+
return { kind: 'collecting' };
|
|
309
|
+
}
|
|
310
|
+
if (this.state.kind === 'settled' ||
|
|
311
|
+
(this.state.kind === 'open' && !this.closeOpen('', ''))) {
|
|
312
|
+
this.invalid = true;
|
|
313
|
+
}
|
|
314
|
+
if (!this.invalid)
|
|
315
|
+
this.state = { kind: 'settled', step: data.step, by: 'retry' };
|
|
316
|
+
return { kind: 'collecting' };
|
|
317
|
+
}
|
|
318
|
+
case 'step/end': {
|
|
319
|
+
const data = dataOf(event);
|
|
320
|
+
if (data.turn !== this.turn || this.state.kind === 'idle' || this.state.step !== data.step) {
|
|
321
|
+
this.invalid = true;
|
|
322
|
+
return { kind: 'collecting' };
|
|
323
|
+
}
|
|
324
|
+
if (this.state.kind === 'open' && !this.closeOpen('', ''))
|
|
325
|
+
this.invalid = true;
|
|
326
|
+
if (!this.invalid)
|
|
327
|
+
this.state = { kind: 'idle' };
|
|
328
|
+
return { kind: 'collecting' };
|
|
329
|
+
}
|
|
330
|
+
default:
|
|
331
|
+
// Every other event type is off the usage path and leaves the fold
|
|
332
|
+
// untouched, exactly as upstream's loop does.
|
|
333
|
+
return { kind: 'collecting' };
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Report whether an event's (turn, step) pair matches the open attempt.
|
|
338
|
+
* @param turn - event Turn.
|
|
339
|
+
* @param step - event step.
|
|
340
|
+
* @returns true when the pair matches the current open state.
|
|
341
|
+
*/
|
|
342
|
+
sameAttempt(turn, step) {
|
|
343
|
+
return this.turn === turn && this.state.kind === 'open' && this.state.step === step;
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* Close the open attempt, if it holds a usable sample.
|
|
347
|
+
*
|
|
348
|
+
* The sample is normalized HERE rather than on arrival so a raw sample can be
|
|
349
|
+
* carried across the `assistant/attempt` → `assistant/message` pair exactly
|
|
350
|
+
* as upstream does.
|
|
351
|
+
*
|
|
352
|
+
* @param provider - route provider for the settled attempt.
|
|
353
|
+
* @param model - route model for the settled attempt.
|
|
354
|
+
* @returns true when the attempt was recorded.
|
|
355
|
+
*/
|
|
356
|
+
closeOpen(provider, model) {
|
|
357
|
+
if (this.state.kind !== 'open')
|
|
358
|
+
return false;
|
|
359
|
+
const sample = this.state.sample;
|
|
360
|
+
if (sample === undefined)
|
|
361
|
+
return false;
|
|
362
|
+
const normalized = normalizeUsage(sample);
|
|
363
|
+
if (normalized === undefined)
|
|
364
|
+
return false;
|
|
365
|
+
this.attempts.push({ ...normalized, at: this.eventTime, provider, model });
|
|
366
|
+
return true;
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* Assemble the outcome for the Turn that just closed.
|
|
370
|
+
* @returns exact accounting, or undefined when the fold was invalid.
|
|
371
|
+
*/
|
|
372
|
+
buildOutcome() {
|
|
373
|
+
const turn = this.turn;
|
|
374
|
+
if (turn === undefined)
|
|
375
|
+
return undefined;
|
|
376
|
+
if (this.invalid || !this.sawEnd || this.state.kind !== 'idle')
|
|
377
|
+
return undefined;
|
|
378
|
+
if (this.attempts.length === 0)
|
|
379
|
+
return undefined;
|
|
380
|
+
const inputTokens = safeSum(this.attempts.map((a) => a.buckets.uncachedInput));
|
|
381
|
+
const outputTokens = safeSum(this.attempts.map((a) => a.buckets.output));
|
|
382
|
+
const totals = safeSum(this.attempts.map((a) => a.total));
|
|
383
|
+
if (inputTokens === undefined || outputTokens === undefined || totals === undefined) {
|
|
384
|
+
return undefined;
|
|
385
|
+
}
|
|
386
|
+
// Optional buckets are published only when EVERY attempt reported that
|
|
387
|
+
// bucket — upstream's rule, applied per bucket so a route that reports
|
|
388
|
+
// cache reads without cache writes still contributes its cache-read total.
|
|
389
|
+
const cacheRead = this.attempts.every((a) => a.reportedCacheRead)
|
|
390
|
+
? safeSum(this.attempts.map((a) => a.buckets.cacheRead))
|
|
391
|
+
: undefined;
|
|
392
|
+
const cacheWrite = this.attempts.every((a) => a.reportedCacheWrite)
|
|
393
|
+
? safeSum(this.attempts.map((a) => a.buckets.cacheWrite))
|
|
394
|
+
: undefined;
|
|
395
|
+
const reasoning = this.attempts.every((a) => a.reportedReasoning)
|
|
396
|
+
? safeSum(this.attempts.map((a) => a.buckets.reasoning))
|
|
397
|
+
: undefined;
|
|
398
|
+
const attempts = this.attempts.map((attempt) => ({
|
|
399
|
+
provider: attempt.provider,
|
|
400
|
+
model: attempt.model,
|
|
401
|
+
at: attempt.at,
|
|
402
|
+
tokens: attempt.buckets,
|
|
403
|
+
total: attempt.total,
|
|
404
|
+
}));
|
|
405
|
+
return {
|
|
406
|
+
turn,
|
|
407
|
+
reason: this.reason,
|
|
408
|
+
startedAt: this.startedAt,
|
|
409
|
+
endedAt: this.endedAt,
|
|
410
|
+
attempts,
|
|
411
|
+
tokens: {
|
|
412
|
+
uncachedInput: inputTokens,
|
|
413
|
+
cacheRead: cacheRead ?? 0,
|
|
414
|
+
cacheWrite: cacheWrite ?? 0,
|
|
415
|
+
output: outputTokens,
|
|
416
|
+
reasoning: reasoning ?? 0,
|
|
417
|
+
},
|
|
418
|
+
total: totals,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* Read the billed route out of a settled assistant message.
|
|
424
|
+
*
|
|
425
|
+
* An empty provider or model is reported as unattributed rather than as a route
|
|
426
|
+
* named `''`, so pricing falls back explicitly and the record is marked
|
|
427
|
+
* estimated.
|
|
428
|
+
*
|
|
429
|
+
* @param message - settled assistant message.
|
|
430
|
+
* @returns the route, both fields `''` when unattributed.
|
|
431
|
+
*/
|
|
432
|
+
function messageRoute(message) {
|
|
433
|
+
const provider = message.source?.provider;
|
|
434
|
+
const model = message.source?.model;
|
|
435
|
+
if (typeof provider !== 'string' || typeof model !== 'string')
|
|
436
|
+
return { provider: '', model: '' };
|
|
437
|
+
if (provider.length === 0 || model.length === 0)
|
|
438
|
+
return { provider: '', model: '' };
|
|
439
|
+
return { provider, model };
|
|
440
|
+
}
|
|
441
|
+
//# sourceMappingURL=collector.js.map
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ledger file IO for the backup/import/backfill operations.
|
|
3
|
+
*
|
|
4
|
+
* The dashboard's own reader (`UsageStore`) is bounded and de-duplicated for
|
|
5
|
+
* rendering; a backup must instead capture EVERY valid line, so these helpers
|
|
6
|
+
* read the file directly and keep the canonical records returned by the schema
|
|
7
|
+
* readers. Import/backfill use the same helpers for the duplicate key set, so
|
|
8
|
+
* all three features agree on `(sessionId, turn)`.
|
|
9
|
+
*
|
|
10
|
+
* @module dsh-token-usage-ledger/host/ledger-io
|
|
11
|
+
*/
|
|
12
|
+
import { type BackupFile, type ImportResult, type UsageRecord } from '../shared/protocol.js';
|
|
13
|
+
/** One full read of the ledger file. */
|
|
14
|
+
export interface LedgerRead {
|
|
15
|
+
/** Canonical records, in file order. */
|
|
16
|
+
records: UsageRecord[];
|
|
17
|
+
/** Every non-empty line visited. */
|
|
18
|
+
lines: number;
|
|
19
|
+
/** Lines rejected as malformed. */
|
|
20
|
+
malformedLines: number;
|
|
21
|
+
/** Lines from a newer schema this build does not understand. */
|
|
22
|
+
unsupportedRecords: number;
|
|
23
|
+
}
|
|
24
|
+
/** Duplicate-tracking view of a ledger. */
|
|
25
|
+
export interface LedgerKeys {
|
|
26
|
+
/** `${sessionId}\u0000${turn}` keys. */
|
|
27
|
+
keys: Set<string>;
|
|
28
|
+
/** Fork fingerprint of every record. */
|
|
29
|
+
fingerprints: Set<string>;
|
|
30
|
+
/** Parse tallies. */
|
|
31
|
+
malformedLines: number;
|
|
32
|
+
/** Unsupported versions. */
|
|
33
|
+
unsupportedRecords: number;
|
|
34
|
+
}
|
|
35
|
+
/** Validation outcome for one backup payload. */
|
|
36
|
+
export interface ParsedBackup {
|
|
37
|
+
/** Canonical records accepted by the schema reader. */
|
|
38
|
+
records: UsageRecord[];
|
|
39
|
+
/** Total records offered by the file. */
|
|
40
|
+
total: number;
|
|
41
|
+
/** Records rejected as malformed. */
|
|
42
|
+
invalid: number;
|
|
43
|
+
/** Records from a newer schema. */
|
|
44
|
+
unsupported: number;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Stable idempotency key for one turn.
|
|
48
|
+
* @param record - ledger record.
|
|
49
|
+
* @returns the key used by the reader and by imports.
|
|
50
|
+
*/
|
|
51
|
+
export declare function recordKey(record: UsageRecord): string;
|
|
52
|
+
/**
|
|
53
|
+
* Fork fingerprint used by the backfill scanner.
|
|
54
|
+
*
|
|
55
|
+
* A forked session copies its parent's events verbatim, so `(sessionId, turn)`
|
|
56
|
+
* alone cannot recognise that copy (the child has a different session id); the
|
|
57
|
+
* log-assigned timestamps and total token count are copied unchanged.
|
|
58
|
+
*
|
|
59
|
+
* @param turn - turn number.
|
|
60
|
+
* @param startedAt - turn start epoch ms.
|
|
61
|
+
* @param endedAt - turn end epoch ms.
|
|
62
|
+
* @param settleTimes - attempt settlement times.
|
|
63
|
+
* @param total - provider-reported total tokens.
|
|
64
|
+
* @returns the fingerprint string.
|
|
65
|
+
*/
|
|
66
|
+
export declare function fingerprintOfParts(turn: number, startedAt: number, endedAt: number, settleTimes: readonly number[], total: number): string;
|
|
67
|
+
/**
|
|
68
|
+
* Fork fingerprint of one ledger record.
|
|
69
|
+
* @param record - ledger record.
|
|
70
|
+
* @returns the fingerprint string.
|
|
71
|
+
*/
|
|
72
|
+
export declare function fingerprintOf(record: UsageRecord): string;
|
|
73
|
+
/**
|
|
74
|
+
* Read every valid record from a ledger file.
|
|
75
|
+
*
|
|
76
|
+
* An absent file is the normal first-run state and yields an empty read; any
|
|
77
|
+
* other IO failure is surfaced as an internal operation error so the caller can
|
|
78
|
+
* answer 500 without exposing the path.
|
|
79
|
+
*
|
|
80
|
+
* @param path - absolute ledger path.
|
|
81
|
+
* @returns the full ledger read.
|
|
82
|
+
*/
|
|
83
|
+
export declare function readLedgerFile(path: string): Promise<LedgerRead>;
|
|
84
|
+
/**
|
|
85
|
+
* Read the duplicate-tracking views of a ledger.
|
|
86
|
+
* @param path - absolute ledger path.
|
|
87
|
+
* @returns keys/fingerprints plus parse tallies.
|
|
88
|
+
*/
|
|
89
|
+
export declare function readLedgerKeys(path: string): Promise<LedgerKeys>;
|
|
90
|
+
/**
|
|
91
|
+
* Build the downloadable backup envelope from the full ledger.
|
|
92
|
+
* @param path - absolute ledger path.
|
|
93
|
+
* @param pluginVersion - running plugin version.
|
|
94
|
+
* @returns the backup file.
|
|
95
|
+
*/
|
|
96
|
+
export declare function createBackup(path: string, pluginVersion: string): Promise<BackupFile>;
|
|
97
|
+
/**
|
|
98
|
+
* Validate a parsed backup payload.
|
|
99
|
+
*
|
|
100
|
+
* Envelope problems throw `unsupported-format`; record problems are counted and
|
|
101
|
+
* returned so the caller can reject the whole import before writing.
|
|
102
|
+
*
|
|
103
|
+
* @param value - parsed JSON body.
|
|
104
|
+
* @returns accepted records and per-record tallies.
|
|
105
|
+
* @throws OperationError when the envelope itself is unusable.
|
|
106
|
+
*/
|
|
107
|
+
export declare function parseBackup(value: unknown): ParsedBackup;
|
|
108
|
+
/**
|
|
109
|
+
* Select records that are not already present (and de-duplicate the file itself).
|
|
110
|
+
* @param records - validated records from the backup.
|
|
111
|
+
* @param existing - duplicate-tracking view of the current ledger.
|
|
112
|
+
* @returns new records and the duplicate count.
|
|
113
|
+
*/
|
|
114
|
+
export declare function selectNewRecords(records: readonly UsageRecord[], existing: LedgerKeys): {
|
|
115
|
+
records: UsageRecord[];
|
|
116
|
+
duplicates: number;
|
|
117
|
+
};
|
|
118
|
+
/**
|
|
119
|
+
* Build the import result object.
|
|
120
|
+
* @param total - records offered by the file.
|
|
121
|
+
* @param added - records appended.
|
|
122
|
+
* @param duplicates - skipped duplicates.
|
|
123
|
+
* @returns the response.
|
|
124
|
+
*/
|
|
125
|
+
export declare function importResult(total: number, added: number, duplicates: number): ImportResult;
|