dsh-token-usage-ledger 0.0.37 → 0.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/README.md +16 -10
- package/lib/client.js +1731 -568
- package/lib/host/aggregate.js +566 -0
- package/lib/host/backfill.d.ts +14 -0
- package/lib/host/backfill.js +30 -2
- package/lib/host/collector.d.ts +54 -3
- package/lib/host/collector.js +289 -10
- package/lib/host/ledger-io.d.ts +28 -1
- package/lib/host/ledger-io.js +54 -4
- package/lib/host/records.d.ts +18 -0
- package/lib/host/records.js +143 -1
- package/lib/host/schema.js +167 -2
- package/lib/index.d.ts +1 -1
- package/lib/index.js +12 -7
- package/lib/shared/format.d.ts +19 -1
- package/lib/shared/format.js +70 -1
- package/lib/shared/protocol.d.ts +277 -3
- package/lib/shared/protocol.js +9 -2
- package/package.json +1 -1
package/lib/host/aggregate.js
CHANGED
|
@@ -89,6 +89,25 @@ export function workspaceGroupKeyOf(record, titleOf) {
|
|
|
89
89
|
return record.workspace;
|
|
90
90
|
return '(未知工作区)';
|
|
91
91
|
}
|
|
92
|
+
/**
|
|
93
|
+
* Short, non-reversible session fingerprint for the highlight table.
|
|
94
|
+
*
|
|
95
|
+
* The dashboard only needs a stable key, not the full session id; keeping the id
|
|
96
|
+
* out of the frequently-read summary reduces the data exposed to the browser.
|
|
97
|
+
* FNV-1a is intentional (no crypto dependency); collisions are harmless here
|
|
98
|
+
* because the key also carries the Turn end timestamp and number.
|
|
99
|
+
*
|
|
100
|
+
* @param sessionId - full session identity from the ledger.
|
|
101
|
+
* @returns eight lowercase hex characters.
|
|
102
|
+
*/
|
|
103
|
+
function sessionFingerprint(sessionId) {
|
|
104
|
+
let hash = 0x811c9dc5;
|
|
105
|
+
for (let index = 0; index < sessionId.length; index += 1) {
|
|
106
|
+
hash ^= sessionId.charCodeAt(index);
|
|
107
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
108
|
+
}
|
|
109
|
+
return hash.toString(16).padStart(8, '0');
|
|
110
|
+
}
|
|
92
111
|
/**
|
|
93
112
|
* Local calendar day of one instant, as `YYYY-MM-DD`.
|
|
94
113
|
*
|
|
@@ -311,6 +330,306 @@ function percentileOfSorted(values, ratio) {
|
|
|
311
330
|
return 0;
|
|
312
331
|
return values[Math.min(values.length - 1, Math.floor((values.length - 1) * ratio))] ?? 0;
|
|
313
332
|
}
|
|
333
|
+
/**
|
|
334
|
+
* Derive one reading per completed step from attempts.
|
|
335
|
+
*
|
|
336
|
+
* A step is "completed" only when one of its attempts was settled by
|
|
337
|
+
* `assistant/message`; retries and failed attempts share the step number but are
|
|
338
|
+
* not a reply. Keeping the earliest start and earliest first token reproduces
|
|
339
|
+
* the harness's own per-step TTFT semantics (`resetForRetry` preserves the first
|
|
340
|
+
* token time across retries).
|
|
341
|
+
*
|
|
342
|
+
* @param attempts - attempts selected for the current view, in settlement order.
|
|
343
|
+
* @returns one reading per message-settled step, in step order.
|
|
344
|
+
*/
|
|
345
|
+
function stepReadingsOf(attempts) {
|
|
346
|
+
const groups = new Map();
|
|
347
|
+
for (const attempt of attempts) {
|
|
348
|
+
const step = attempt.step;
|
|
349
|
+
if (typeof step !== 'number' || !Number.isSafeInteger(step) || step < 0)
|
|
350
|
+
continue;
|
|
351
|
+
const list = groups.get(step) ?? [];
|
|
352
|
+
list.push(attempt);
|
|
353
|
+
groups.set(step, list);
|
|
354
|
+
}
|
|
355
|
+
const readings = [];
|
|
356
|
+
for (const [step, list] of [...groups.entries()].sort((left, right) => left[0] - right[0])) {
|
|
357
|
+
let startedAt;
|
|
358
|
+
let firstTokenAt;
|
|
359
|
+
let final;
|
|
360
|
+
for (const attempt of list) {
|
|
361
|
+
if (typeof attempt.startedAt === 'number' && Number.isFinite(attempt.startedAt)) {
|
|
362
|
+
startedAt = startedAt === undefined ? attempt.startedAt : Math.min(startedAt, attempt.startedAt);
|
|
363
|
+
}
|
|
364
|
+
if (typeof attempt.firstTokenAt === 'number' && Number.isFinite(attempt.firstTokenAt)) {
|
|
365
|
+
firstTokenAt =
|
|
366
|
+
firstTokenAt === undefined ? attempt.firstTokenAt : Math.min(firstTokenAt, attempt.firstTokenAt);
|
|
367
|
+
}
|
|
368
|
+
if (attempt.settledBy === 'message')
|
|
369
|
+
final = attempt;
|
|
370
|
+
}
|
|
371
|
+
if (final === undefined || startedAt === undefined)
|
|
372
|
+
continue;
|
|
373
|
+
const completedAt = final.at;
|
|
374
|
+
const ttftMs = firstTokenAt === undefined ? undefined : Math.max(0, firstTokenAt - startedAt);
|
|
375
|
+
const decodeMs = firstTokenAt === undefined ? undefined : Math.max(0, completedAt - firstTokenAt);
|
|
376
|
+
const rawTokensPerSecond = decodeMs !== undefined && decodeMs > 0 && Number.isFinite(final.tokens.output)
|
|
377
|
+
? (final.tokens.output / decodeMs) * 1000
|
|
378
|
+
: undefined;
|
|
379
|
+
const tokensPerSecond = rawTokensPerSecond !== undefined && isTrustedTps(rawTokensPerSecond)
|
|
380
|
+
? rawTokensPerSecond
|
|
381
|
+
: undefined;
|
|
382
|
+
readings.push({
|
|
383
|
+
step,
|
|
384
|
+
startedAt,
|
|
385
|
+
completedAt,
|
|
386
|
+
outputTokens: final.tokens.output,
|
|
387
|
+
calls: list.length,
|
|
388
|
+
interrupted: final.interrupted === true,
|
|
389
|
+
...(firstTokenAt === undefined ? {} : { firstTokenAt }),
|
|
390
|
+
...(ttftMs === undefined ? {} : { ttftMs }),
|
|
391
|
+
...(decodeMs === undefined ? {} : { decodeMs }),
|
|
392
|
+
...(tokensPerSecond === undefined ? {} : { tokensPerSecond }),
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
return readings;
|
|
396
|
+
}
|
|
397
|
+
/** A fresh timing accumulator. */
|
|
398
|
+
function newTimingTotals() {
|
|
399
|
+
return {
|
|
400
|
+
ttftMs: 0,
|
|
401
|
+
ttftSamples: 0,
|
|
402
|
+
decodeMs: 0,
|
|
403
|
+
decodeTokens: 0,
|
|
404
|
+
modelMs: 0,
|
|
405
|
+
modelSamples: 0,
|
|
406
|
+
toolMs: 0,
|
|
407
|
+
toolCalls: 0,
|
|
408
|
+
steps: 0,
|
|
409
|
+
stepSamples: 0,
|
|
410
|
+
calls: 0,
|
|
411
|
+
callsWithTiming: 0,
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* Add one timing slice into another, in place.
|
|
416
|
+
* @param target - accumulator.
|
|
417
|
+
* @param add - slice to add.
|
|
418
|
+
*/
|
|
419
|
+
function addTimingTotals(target, add) {
|
|
420
|
+
target.ttftMs += add.ttftMs;
|
|
421
|
+
target.ttftSamples += add.ttftSamples;
|
|
422
|
+
target.decodeMs += add.decodeMs;
|
|
423
|
+
target.decodeTokens += add.decodeTokens;
|
|
424
|
+
target.modelMs += add.modelMs;
|
|
425
|
+
target.modelSamples += add.modelSamples;
|
|
426
|
+
target.toolMs += add.toolMs;
|
|
427
|
+
target.toolCalls += add.toolCalls;
|
|
428
|
+
target.steps += add.steps;
|
|
429
|
+
target.stepSamples += add.stepSamples;
|
|
430
|
+
target.calls += add.calls;
|
|
431
|
+
target.callsWithTiming += add.callsWithTiming;
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* Timing slice for one record under the active filter.
|
|
435
|
+
*
|
|
436
|
+
* Split deliberately into two scopes:
|
|
437
|
+
*
|
|
438
|
+
* - **call-level** TTFT/TPS/model time come from the attempts selected by the
|
|
439
|
+
* model/provider filter;
|
|
440
|
+
* - **turn-level** step count and tool activity come from the WHOLE record,
|
|
441
|
+
* because tools cannot be attributed to one model call and a filtered step
|
|
442
|
+
* count would silently disagree with the tool count beside it.
|
|
443
|
+
*
|
|
444
|
+
* Returns undefined when the record carries neither timing nor activity, so a
|
|
445
|
+
* legacy v1 line never contributes a fake `0ms`/`0 tok/s` sample.
|
|
446
|
+
*
|
|
447
|
+
* @param record - selected record.
|
|
448
|
+
* @param attempts - attempts selected by the model/provider filter.
|
|
449
|
+
* @param activityAttempts - attempts used for the whole-turn step count.
|
|
450
|
+
* @param readings - precomputed readings for {@link attempts}, when available.
|
|
451
|
+
* @returns the slice, or undefined when there is no v2 timing/activity data.
|
|
452
|
+
*/
|
|
453
|
+
function timingFromReadings(record, attempts, readings, activityReadings) {
|
|
454
|
+
const totals = newTimingTotals();
|
|
455
|
+
totals.calls = attempts.length;
|
|
456
|
+
totals.callsWithTiming = attempts.filter((attempt) => typeof attempt.startedAt === 'number' &&
|
|
457
|
+
Number.isFinite(attempt.startedAt) &&
|
|
458
|
+
typeof attempt.firstTokenAt === 'number' &&
|
|
459
|
+
Number.isFinite(attempt.firstTokenAt)).length;
|
|
460
|
+
for (const reading of readings) {
|
|
461
|
+
if (reading.ttftMs !== undefined) {
|
|
462
|
+
totals.ttftMs += reading.ttftMs;
|
|
463
|
+
totals.ttftSamples += 1;
|
|
464
|
+
}
|
|
465
|
+
totals.modelMs += Math.max(0, reading.completedAt - reading.startedAt);
|
|
466
|
+
totals.modelSamples += 1;
|
|
467
|
+
if (reading.decodeMs !== undefined && reading.decodeMs > 0) {
|
|
468
|
+
totals.decodeMs += reading.decodeMs;
|
|
469
|
+
totals.decodeTokens += reading.outputTokens;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
totals.steps = activityReadings.length;
|
|
473
|
+
if (activityReadings.length > 0)
|
|
474
|
+
totals.stepSamples = 1;
|
|
475
|
+
if (record.tools !== undefined) {
|
|
476
|
+
for (const tool of record.tools) {
|
|
477
|
+
totals.toolMs += tool.durationMs;
|
|
478
|
+
totals.toolCalls += tool.calls;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
return readings.length > 0 || activityReadings.length > 0 || record.tools !== undefined
|
|
482
|
+
? totals
|
|
483
|
+
: undefined;
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Timing slice for one record under the active filter.
|
|
487
|
+
*
|
|
488
|
+
* Split deliberately into two scopes:
|
|
489
|
+
*
|
|
490
|
+
* - **call-level** TTFT/TPS/model time come from the attempts selected by the
|
|
491
|
+
* model/provider filter;
|
|
492
|
+
* - **turn-level** step count and tool activity come from the WHOLE record,
|
|
493
|
+
* because tools cannot be attributed to one model call and a filtered step
|
|
494
|
+
* count would silently disagree with the tool count beside it.
|
|
495
|
+
*
|
|
496
|
+
* @param record - selected record.
|
|
497
|
+
* @param attempts - attempts selected by the model/provider filter.
|
|
498
|
+
* @param activityAttempts - attempts used for the whole-turn step count.
|
|
499
|
+
* @returns the slice, or undefined when there is no v2 timing/activity data.
|
|
500
|
+
*/
|
|
501
|
+
function timingOf(record, attempts, activityAttempts = attempts) {
|
|
502
|
+
const readings = stepReadingsOf(attempts);
|
|
503
|
+
const activityReadings = activityAttempts === attempts ? readings : stepReadingsOf(activityAttempts);
|
|
504
|
+
return timingFromReadings(record, attempts, readings, activityReadings);
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* Upper plausibility bound for a per-step output speed.
|
|
508
|
+
*
|
|
509
|
+
* Some providers/non-streaming paths deliver the whole answer in one chunk:
|
|
510
|
+
* `firstTokenAt` then lands microseconds before the final message and the raw
|
|
511
|
+
* output/decode ratio becomes thousands of tok/s. Those samples describe
|
|
512
|
+
* transport batching, not model speed, so they are excluded from speed averages,
|
|
513
|
+
* percentiles and distributions. 1000 tok/s is already far above the fastest
|
|
514
|
+
* observed model; a real future model above it would need this constant raised
|
|
515
|
+
* together with the distribution chart.
|
|
516
|
+
*/
|
|
517
|
+
const MAX_TRUSTED_TPS = 1000;
|
|
518
|
+
/** Whether a per-step output speed is physically meaningful for this UI. */
|
|
519
|
+
function isTrustedTps(value) {
|
|
520
|
+
return Number.isFinite(value) && value > 0 && value <= MAX_TRUSTED_TPS;
|
|
521
|
+
}
|
|
522
|
+
/**
|
|
523
|
+
* Output-speed bins, expanded where this workload actually sits.
|
|
524
|
+
*
|
|
525
|
+
* The old top bin was `80+`, which swallowed almost every step on a real ledger
|
|
526
|
+
* (DeepSeek Flash medians around 250 tok/s). The bins now keep the dense
|
|
527
|
+
* 150–500 tok/s region readable and isolate the long tail (slow models and rare
|
|
528
|
+
* 1000+ outliers) instead of flattening everything into one bar.
|
|
529
|
+
*/
|
|
530
|
+
const SPEED_BIN_LABELS = [
|
|
531
|
+
'<50',
|
|
532
|
+
'50–100',
|
|
533
|
+
'100–150',
|
|
534
|
+
'150–200',
|
|
535
|
+
'200–250',
|
|
536
|
+
'250–300',
|
|
537
|
+
'300–400',
|
|
538
|
+
'400–500',
|
|
539
|
+
'500–1000',
|
|
540
|
+
'1000+',
|
|
541
|
+
];
|
|
542
|
+
/**
|
|
543
|
+
* Bin one output speed.
|
|
544
|
+
* @param tps - tokens/second.
|
|
545
|
+
* @returns bin index 0–9.
|
|
546
|
+
*/
|
|
547
|
+
function speedBinOf(tps) {
|
|
548
|
+
if (tps < 50)
|
|
549
|
+
return 0;
|
|
550
|
+
if (tps < 100)
|
|
551
|
+
return 1;
|
|
552
|
+
if (tps < 150)
|
|
553
|
+
return 2;
|
|
554
|
+
if (tps < 200)
|
|
555
|
+
return 3;
|
|
556
|
+
if (tps < 250)
|
|
557
|
+
return 4;
|
|
558
|
+
if (tps < 300)
|
|
559
|
+
return 5;
|
|
560
|
+
if (tps < 400)
|
|
561
|
+
return 6;
|
|
562
|
+
if (tps < 500)
|
|
563
|
+
return 7;
|
|
564
|
+
if (tps < 1000)
|
|
565
|
+
return 8;
|
|
566
|
+
return 9;
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* Steps-per-turn bins. One-step and 2–5-step turns are split out because the old
|
|
570
|
+
* `32+` bin held nearly half of the real turns; the wide high bins keep long
|
|
571
|
+
* tool loops visible without one catch-all.
|
|
572
|
+
*/
|
|
573
|
+
const STEP_BIN_LABELS = ['1', '2', '3', '4–5', '6–10', '11–20', '21–40', '41–80', '81–160', '161+'];
|
|
574
|
+
/**
|
|
575
|
+
* Bin one Turn's step count.
|
|
576
|
+
* @param steps - step count.
|
|
577
|
+
* @returns bin index 0–9.
|
|
578
|
+
*/
|
|
579
|
+
function stepBinOf(steps) {
|
|
580
|
+
if (steps <= 1)
|
|
581
|
+
return 0;
|
|
582
|
+
if (steps === 2)
|
|
583
|
+
return 1;
|
|
584
|
+
if (steps === 3)
|
|
585
|
+
return 2;
|
|
586
|
+
if (steps <= 5)
|
|
587
|
+
return 3;
|
|
588
|
+
if (steps <= 10)
|
|
589
|
+
return 4;
|
|
590
|
+
if (steps <= 20)
|
|
591
|
+
return 5;
|
|
592
|
+
if (steps <= 40)
|
|
593
|
+
return 6;
|
|
594
|
+
if (steps <= 80)
|
|
595
|
+
return 7;
|
|
596
|
+
if (steps <= 160)
|
|
597
|
+
return 8;
|
|
598
|
+
return 9;
|
|
599
|
+
}
|
|
600
|
+
/** Tool-calls-per-turn bins, split at the dense 30–240 call region. */
|
|
601
|
+
const TOOL_BIN_LABELS = ['0', '1–5', '6–15', '16–30', '31–60', '61–120', '121–240', '241+'];
|
|
602
|
+
/**
|
|
603
|
+
* Bin one Turn's tool-call count.
|
|
604
|
+
* @param calls - tool call count.
|
|
605
|
+
* @returns bin index 0–7.
|
|
606
|
+
*/
|
|
607
|
+
function toolBinOf(calls) {
|
|
608
|
+
if (calls === 0)
|
|
609
|
+
return 0;
|
|
610
|
+
if (calls <= 5)
|
|
611
|
+
return 1;
|
|
612
|
+
if (calls <= 15)
|
|
613
|
+
return 2;
|
|
614
|
+
if (calls <= 30)
|
|
615
|
+
return 3;
|
|
616
|
+
if (calls <= 60)
|
|
617
|
+
return 4;
|
|
618
|
+
if (calls <= 120)
|
|
619
|
+
return 5;
|
|
620
|
+
if (calls <= 240)
|
|
621
|
+
return 6;
|
|
622
|
+
return 7;
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Order two highlight rows, best first (higher cost, then tokens, then duration).
|
|
626
|
+
* @param left - first row.
|
|
627
|
+
* @param right - second row.
|
|
628
|
+
* @returns negative when {@link left} should come first.
|
|
629
|
+
*/
|
|
630
|
+
function compareHighlights(left, right) {
|
|
631
|
+
return right.cost - left.cost || right.tokens - left.tokens || right.durationMs - left.durationMs;
|
|
632
|
+
}
|
|
314
633
|
/**
|
|
315
634
|
* Build the extra call/agent/outcome statistics.
|
|
316
635
|
*
|
|
@@ -344,6 +663,49 @@ function buildAnalytics(selected, model, provider, currency) {
|
|
|
344
663
|
cost: 0,
|
|
345
664
|
}));
|
|
346
665
|
let callTotal = 0;
|
|
666
|
+
/** v2 accumulators. */
|
|
667
|
+
const perfTotals = newTimingTotals();
|
|
668
|
+
const ttftValues = [];
|
|
669
|
+
const tpsValues = [];
|
|
670
|
+
const speedBins = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
|
671
|
+
const stepBins = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
|
672
|
+
const toolBins = [0, 0, 0, 0, 0, 0, 0, 0];
|
|
673
|
+
const tools = new Map();
|
|
674
|
+
const routes = new Map();
|
|
675
|
+
const weekHours = [];
|
|
676
|
+
for (let weekday = 0; weekday < 7; weekday += 1) {
|
|
677
|
+
for (let hour = 0; hour < 24; hour += 1) {
|
|
678
|
+
weekHours.push({ weekday, hour, turns: 0, calls: 0, tokens: 0, cost: 0 });
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
const hourCell = (epochMs) => {
|
|
682
|
+
const date = new Date(epochMs);
|
|
683
|
+
if (!Number.isFinite(date.getTime()))
|
|
684
|
+
return undefined;
|
|
685
|
+
const weekday = (date.getDay() + 6) % 7;
|
|
686
|
+
const hour = date.getHours();
|
|
687
|
+
return weekHours[weekday * 24 + hour];
|
|
688
|
+
};
|
|
689
|
+
const highlights = [];
|
|
690
|
+
/** Keep only the eight most expensive Turns without retaining every record. */
|
|
691
|
+
const considerHighlight = (row) => {
|
|
692
|
+
if (highlights.length < 8) {
|
|
693
|
+
highlights.push(row);
|
|
694
|
+
highlights.sort(compareHighlights);
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
const worst = highlights[highlights.length - 1];
|
|
698
|
+
if (worst === undefined || compareHighlights(row, worst) >= 0)
|
|
699
|
+
return;
|
|
700
|
+
highlights.push(row);
|
|
701
|
+
highlights.sort(compareHighlights);
|
|
702
|
+
highlights.pop();
|
|
703
|
+
};
|
|
704
|
+
let recordsWithTiming = 0;
|
|
705
|
+
let toolResultsTotal = 0;
|
|
706
|
+
let toolErrorsTotal = 0;
|
|
707
|
+
let retriesTotal = 0;
|
|
708
|
+
let interruptedTurns = 0;
|
|
347
709
|
const windowLabel = {
|
|
348
710
|
peak: '高峰',
|
|
349
711
|
'off-peak': '空闲',
|
|
@@ -403,6 +765,85 @@ function buildAnalytics(selected, model, provider, currency) {
|
|
|
403
765
|
const reasonLabel = reason === 'completed' ? '完成' : reason === 'aborted' ? '中止' : reason === 'error' ? '错误' : reason;
|
|
404
766
|
addDimensionUsage(reasons, reason, reasonLabel, contribution, matched);
|
|
405
767
|
const priced = record.cost.currency === currency;
|
|
768
|
+
const readings = stepReadingsOf(matched);
|
|
769
|
+
const activityReadings = matched === record.attempts ? readings : stepReadingsOf(record.attempts);
|
|
770
|
+
const timing = timingFromReadings(record, matched, readings, activityReadings);
|
|
771
|
+
if (timing !== undefined)
|
|
772
|
+
addTimingTotals(perfTotals, timing);
|
|
773
|
+
if (activityReadings.length > 0) {
|
|
774
|
+
bumpBin(stepBins, stepBinOf(activityReadings.length));
|
|
775
|
+
for (const reading of readings) {
|
|
776
|
+
if (reading.ttftMs !== undefined)
|
|
777
|
+
ttftValues.push(reading.ttftMs);
|
|
778
|
+
if (reading.tokensPerSecond !== undefined) {
|
|
779
|
+
tpsValues.push(reading.tokensPerSecond);
|
|
780
|
+
bumpBin(speedBins, speedBinOf(reading.tokensPerSecond));
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
if (record.v >= 2) {
|
|
785
|
+
const callsInTurn = (record.tools ?? []).reduce((sum, tool) => sum + tool.calls, 0);
|
|
786
|
+
bumpBin(toolBins, toolBinOf(callsInTurn));
|
|
787
|
+
}
|
|
788
|
+
retriesTotal += record.retries ?? 0;
|
|
789
|
+
if (record.interrupted === true)
|
|
790
|
+
interruptedTurns += 1;
|
|
791
|
+
for (const tool of record.tools ?? []) {
|
|
792
|
+
const entry = tools.get(tool.name) ?? {
|
|
793
|
+
name: tool.name,
|
|
794
|
+
calls: 0,
|
|
795
|
+
results: 0,
|
|
796
|
+
errors: 0,
|
|
797
|
+
durationMs: 0,
|
|
798
|
+
};
|
|
799
|
+
entry.calls += tool.calls;
|
|
800
|
+
entry.results += tool.results;
|
|
801
|
+
entry.errors += tool.errors;
|
|
802
|
+
entry.durationMs += tool.durationMs;
|
|
803
|
+
tools.set(tool.name, entry);
|
|
804
|
+
toolResultsTotal += tool.results;
|
|
805
|
+
toolErrorsTotal += tool.errors;
|
|
806
|
+
}
|
|
807
|
+
if (matched.some((attempt) => typeof attempt.startedAt === 'number' &&
|
|
808
|
+
Number.isFinite(attempt.startedAt) &&
|
|
809
|
+
typeof attempt.firstTokenAt === 'number' &&
|
|
810
|
+
Number.isFinite(attempt.firstTokenAt))) {
|
|
811
|
+
recordsWithTiming += 1;
|
|
812
|
+
}
|
|
813
|
+
const turnCell = hourCell(record.startedAt);
|
|
814
|
+
if (turnCell !== undefined)
|
|
815
|
+
turnCell.turns += 1;
|
|
816
|
+
const routeLabels = new Set();
|
|
817
|
+
for (const attempt of matched) {
|
|
818
|
+
routeLabels.add(attempt.model.length === 0
|
|
819
|
+
? attempt.provider.length === 0
|
|
820
|
+
? '(未标注模型)'
|
|
821
|
+
: `${attempt.provider}/(未标注模型)`
|
|
822
|
+
: attempt.provider.length === 0
|
|
823
|
+
? attempt.model
|
|
824
|
+
: `${attempt.provider}/${attempt.model}`);
|
|
825
|
+
}
|
|
826
|
+
const stepsForTurn = activityReadings.length;
|
|
827
|
+
const turnTtft = readings.find((reading) => reading.ttftMs !== undefined)?.ttftMs;
|
|
828
|
+
const turnDecodeMs = readings.reduce((sum, reading) => sum + (reading.decodeMs ?? 0), 0);
|
|
829
|
+
const turnDecodeTokens = readings.reduce((sum, reading) => sum + (reading.decodeMs === undefined ? 0 : reading.outputTokens), 0);
|
|
830
|
+
const sessionKey = sessionFingerprint(record.sessionId);
|
|
831
|
+
considerHighlight({
|
|
832
|
+
key: `${String(record.ts)}:${String(record.turn)}:${sessionKey}`,
|
|
833
|
+
sessionKey,
|
|
834
|
+
turn: record.turn,
|
|
835
|
+
ts: record.ts,
|
|
836
|
+
reason: record.reason,
|
|
837
|
+
models: [...routeLabels].join(','),
|
|
838
|
+
tokens: contribution.total,
|
|
839
|
+
cost: contribution.cost,
|
|
840
|
+
durationMs: Math.max(0, record.endedAt - record.startedAt),
|
|
841
|
+
...(stepsForTurn === 0 ? {} : { steps: stepsForTurn }),
|
|
842
|
+
...(turnTtft === undefined ? {} : { ttftMs: turnTtft }),
|
|
843
|
+
...(turnDecodeMs <= 0
|
|
844
|
+
? {}
|
|
845
|
+
: { tokensPerSecond: (turnDecodeTokens / turnDecodeMs) * 1000 }),
|
|
846
|
+
});
|
|
406
847
|
for (const attempt of matched) {
|
|
407
848
|
const row = windows.get(attempt.window);
|
|
408
849
|
if (row !== undefined) {
|
|
@@ -419,6 +860,41 @@ function buildAnalytics(selected, model, provider, currency) {
|
|
|
419
860
|
hourRow.cost += priced ? attempt.cost : 0;
|
|
420
861
|
}
|
|
421
862
|
}
|
|
863
|
+
const attemptCell = hourCell(attempt.at);
|
|
864
|
+
if (attemptCell !== undefined) {
|
|
865
|
+
attemptCell.calls += 1;
|
|
866
|
+
attemptCell.tokens += attempt.total;
|
|
867
|
+
attemptCell.cost += priced ? attempt.cost : 0;
|
|
868
|
+
}
|
|
869
|
+
const routeKey = routeKeyOf(attempt.provider, attempt.model);
|
|
870
|
+
const route = routes.get(routeKey) ?? {
|
|
871
|
+
key: routeKey,
|
|
872
|
+
provider: attempt.provider,
|
|
873
|
+
model: attempt.model,
|
|
874
|
+
calls: 0,
|
|
875
|
+
timedCalls: 0,
|
|
876
|
+
outputTokens: 0,
|
|
877
|
+
modelMs: 0,
|
|
878
|
+
ttft: [],
|
|
879
|
+
tps: [],
|
|
880
|
+
};
|
|
881
|
+
route.calls += 1;
|
|
882
|
+
route.outputTokens += attempt.tokens.output;
|
|
883
|
+
if (typeof attempt.startedAt === 'number' && Number.isFinite(attempt.startedAt)) {
|
|
884
|
+
route.modelMs += Math.max(0, attempt.at - attempt.startedAt);
|
|
885
|
+
}
|
|
886
|
+
if (typeof attempt.firstTokenAt === 'number' && Number.isFinite(attempt.firstTokenAt)) {
|
|
887
|
+
route.timedCalls += 1;
|
|
888
|
+
if (typeof attempt.startedAt === 'number' && Number.isFinite(attempt.startedAt)) {
|
|
889
|
+
route.ttft.push(Math.max(0, attempt.firstTokenAt - attempt.startedAt));
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
if (typeof attempt.tokensPerSecond === 'number' &&
|
|
893
|
+
Number.isFinite(attempt.tokensPerSecond) &&
|
|
894
|
+
isTrustedTps(attempt.tokensPerSecond)) {
|
|
895
|
+
route.tps.push(attempt.tokensPerSecond);
|
|
896
|
+
}
|
|
897
|
+
routes.set(routeKey, route);
|
|
422
898
|
}
|
|
423
899
|
const session = sessions.get(record.sessionId) ?? { turns: 0, tokens: 0, cost: 0 };
|
|
424
900
|
session.turns += 1;
|
|
@@ -464,6 +940,70 @@ function buildAnalytics(selected, model, provider, currency) {
|
|
|
464
940
|
};
|
|
465
941
|
const byCostThenTokens = (left, right) => right.cost - left.cost || right.tokens - left.tokens;
|
|
466
942
|
const byTurns = (left, right) => right.turns - left.turns || right.tokens - left.tokens;
|
|
943
|
+
const sortedTtft = [...ttftValues].sort((left, right) => left - right);
|
|
944
|
+
const sortedTps = [...tpsValues].sort((left, right) => left - right);
|
|
945
|
+
const performance = {
|
|
946
|
+
ttftMedian: medianOfSorted(sortedTtft),
|
|
947
|
+
ttftP90: percentileOfSorted(sortedTtft, 0.9),
|
|
948
|
+
ttftAverage: perfTotals.ttftSamples === 0 ? 0 : perfTotals.ttftMs / perfTotals.ttftSamples,
|
|
949
|
+
ttftSamples: perfTotals.ttftSamples,
|
|
950
|
+
tpsMedian: medianOfSorted(sortedTps),
|
|
951
|
+
tpsAverage: sortedTps.length === 0
|
|
952
|
+
? 0
|
|
953
|
+
: sortedTps.reduce((sum, value) => sum + value, 0) / sortedTps.length,
|
|
954
|
+
tpsSamples: sortedTps.length,
|
|
955
|
+
modelMs: perfTotals.modelMs,
|
|
956
|
+
modelSamples: perfTotals.modelSamples,
|
|
957
|
+
toolMs: perfTotals.toolMs,
|
|
958
|
+
toolCalls: perfTotals.toolCalls,
|
|
959
|
+
toolErrors: toolErrorsTotal,
|
|
960
|
+
toolErrorRate: toolResultsTotal === 0 ? 0 : toolErrorsTotal / toolResultsTotal,
|
|
961
|
+
stepsTotal: perfTotals.steps,
|
|
962
|
+
stepSamples: perfTotals.stepSamples,
|
|
963
|
+
stepsPerTurn: perfTotals.stepSamples === 0 ? 0 : perfTotals.steps / perfTotals.stepSamples,
|
|
964
|
+
calls: callTotal,
|
|
965
|
+
callsWithTiming: perfTotals.callsWithTiming,
|
|
966
|
+
retries: retriesTotal,
|
|
967
|
+
retryRate: callTotal === 0 ? 0 : retriesTotal / callTotal,
|
|
968
|
+
interrupted: interruptedTurns,
|
|
969
|
+
coverage: {
|
|
970
|
+
recordsWithTiming,
|
|
971
|
+
records: turns,
|
|
972
|
+
attemptsWithTiming: perfTotals.callsWithTiming,
|
|
973
|
+
attempts: callTotal,
|
|
974
|
+
},
|
|
975
|
+
};
|
|
976
|
+
const toolRows = [...tools.values()].sort((left, right) => right.calls - left.calls || right.durationMs - left.durationMs || left.name.localeCompare(right.name));
|
|
977
|
+
const routeRows = [...routes.values()]
|
|
978
|
+
.map((route) => {
|
|
979
|
+
const ttft = [...route.ttft].sort((left, right) => left - right);
|
|
980
|
+
const tps = [...route.tps].sort((left, right) => left - right);
|
|
981
|
+
return {
|
|
982
|
+
key: route.key,
|
|
983
|
+
provider: route.provider,
|
|
984
|
+
model: route.model,
|
|
985
|
+
calls: route.calls,
|
|
986
|
+
timedCalls: route.timedCalls,
|
|
987
|
+
outputTokens: route.outputTokens,
|
|
988
|
+
ttftMedian: medianOfSorted(ttft),
|
|
989
|
+
tpsMedian: medianOfSorted(tps),
|
|
990
|
+
...(tps.length === 0
|
|
991
|
+
? {}
|
|
992
|
+
: {
|
|
993
|
+
tpsP10: percentileOfSorted(tps, 0.1),
|
|
994
|
+
tpsP25: percentileOfSorted(tps, 0.25),
|
|
995
|
+
tpsP75: percentileOfSorted(tps, 0.75),
|
|
996
|
+
tpsP90: percentileOfSorted(tps, 0.9),
|
|
997
|
+
}),
|
|
998
|
+
modelMs: route.modelMs,
|
|
999
|
+
};
|
|
1000
|
+
})
|
|
1001
|
+
.sort((left, right) => right.calls - left.calls || left.key.localeCompare(right.key));
|
|
1002
|
+
const highlightRows = [...highlights]
|
|
1003
|
+
.sort((left, right) => right.cost - left.cost ||
|
|
1004
|
+
right.tokens - left.tokens ||
|
|
1005
|
+
right.durationMs - left.durationMs)
|
|
1006
|
+
.slice(0, 8);
|
|
467
1007
|
return {
|
|
468
1008
|
calls: callStats,
|
|
469
1009
|
durations: durationStats,
|
|
@@ -489,6 +1029,23 @@ function buildAnalytics(selected, model, provider, currency) {
|
|
|
489
1029
|
reasons: [...reasons.values()].sort(byTurns),
|
|
490
1030
|
windows: [...windows.values()].filter((row) => row.calls > 0),
|
|
491
1031
|
hours,
|
|
1032
|
+
performance,
|
|
1033
|
+
speedBins: SPEED_BIN_LABELS.map((label, index) => ({
|
|
1034
|
+
label,
|
|
1035
|
+
count: speedBins[index] ?? 0,
|
|
1036
|
+
})),
|
|
1037
|
+
stepBins: STEP_BIN_LABELS.map((label, index) => ({
|
|
1038
|
+
label,
|
|
1039
|
+
count: stepBins[index] ?? 0,
|
|
1040
|
+
})),
|
|
1041
|
+
toolBins: TOOL_BIN_LABELS.map((label, index) => ({
|
|
1042
|
+
label,
|
|
1043
|
+
count: toolBins[index] ?? 0,
|
|
1044
|
+
})),
|
|
1045
|
+
tools: toolRows,
|
|
1046
|
+
routes: routeRows,
|
|
1047
|
+
weekHours,
|
|
1048
|
+
highlights: highlightRows,
|
|
492
1049
|
};
|
|
493
1050
|
}
|
|
494
1051
|
/**
|
|
@@ -677,6 +1234,7 @@ export function buildSummary(input) {
|
|
|
677
1234
|
const byWorkspace = new Map();
|
|
678
1235
|
const byModel = new Map();
|
|
679
1236
|
const perDay = new Map();
|
|
1237
|
+
const perfByDay = new Map();
|
|
680
1238
|
const windowStart = addLocalDays(localDayKey(now), -(Math.max(1, days) - 1));
|
|
681
1239
|
const windowStartMs = startOfLocalDay(windowStart);
|
|
682
1240
|
for (const { record, contribution } of selected) {
|
|
@@ -706,6 +1264,12 @@ export function buildSummary(input) {
|
|
|
706
1264
|
const entry = perDay.get(key) ?? newAccumulator();
|
|
707
1265
|
accumulate(entry, contribution);
|
|
708
1266
|
perDay.set(key, entry);
|
|
1267
|
+
const timing = timingOf(record, record.attempts.filter(matchAttempt), record.attempts);
|
|
1268
|
+
if (timing !== undefined) {
|
|
1269
|
+
const perf = perfByDay.get(key) ?? newTimingTotals();
|
|
1270
|
+
addTimingTotals(perf, timing);
|
|
1271
|
+
perfByDay.set(key, perf);
|
|
1272
|
+
}
|
|
709
1273
|
}
|
|
710
1274
|
}
|
|
711
1275
|
// --- daily series: every day in the window, gaps filled, so the chart's
|
|
@@ -724,6 +1288,7 @@ export function buildSummary(input) {
|
|
|
724
1288
|
for (let offset = 0; offset < dailyDays; offset += 1) {
|
|
725
1289
|
const key = addLocalDays(windowStart, offset);
|
|
726
1290
|
const entry = perDay.get(key) ?? newAccumulator();
|
|
1291
|
+
const perf = perfByDay.get(key);
|
|
727
1292
|
daily.push({
|
|
728
1293
|
date: key,
|
|
729
1294
|
turns: entry.turns,
|
|
@@ -732,6 +1297,7 @@ export function buildSummary(input) {
|
|
|
732
1297
|
cost: entry.cost,
|
|
733
1298
|
estimated: entry.estimated,
|
|
734
1299
|
...(entry.estimatedReason === undefined ? {} : { estimatedReason: entry.estimatedReason }),
|
|
1300
|
+
...(perf === undefined ? {} : { perf }),
|
|
735
1301
|
});
|
|
736
1302
|
}
|
|
737
1303
|
const sortedDays = [...daysWithUsage].sort();
|
package/lib/host/backfill.d.ts
CHANGED
|
@@ -31,6 +31,13 @@ export interface BackfillOptions {
|
|
|
31
31
|
limit?: number | undefined;
|
|
32
32
|
/** Fold inherited prefixes too (diagnostic). */
|
|
33
33
|
noPrefilter?: boolean | undefined;
|
|
34
|
+
/**
|
|
35
|
+
* Fill v2 timing/activity fields into existing records (default true).
|
|
36
|
+
*
|
|
37
|
+
* Set false to keep the old "append missing Turns only" behavior; the ledger
|
|
38
|
+
* is still append-only either way.
|
|
39
|
+
*/
|
|
40
|
+
enrich?: boolean | undefined;
|
|
34
41
|
/** Warning sink for unreadable logs; receives no record contents. */
|
|
35
42
|
onWarning?: ((message: string) => void) | undefined;
|
|
36
43
|
}
|
|
@@ -40,6 +47,13 @@ export interface BackfillOutcome {
|
|
|
40
47
|
report: BackfillReport;
|
|
41
48
|
/** Backfillable records, timestamp-sorted. */
|
|
42
49
|
records: UsageRecord[];
|
|
50
|
+
/**
|
|
51
|
+
* Existing Turns upgraded with v2 timing/activity fields, timestamp-sorted.
|
|
52
|
+
*
|
|
53
|
+
* Appending these is safe because the reader keeps the newest line per
|
|
54
|
+
* `(sessionId, turn)`; token/cost/identity fields are never changed.
|
|
55
|
+
*/
|
|
56
|
+
upgrades: UsageRecord[];
|
|
43
57
|
}
|
|
44
58
|
/**
|
|
45
59
|
* Scan durable logs and return the missing records.
|