@oh-my-pi/omp-stats 18.16.0 → 18.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/dist/client/index.css +1 -1
  3. package/dist/client/index.js +102 -99
  4. package/dist/types/client/api.d.ts +4 -1
  5. package/dist/types/client/app/nav.d.ts +1 -1
  6. package/dist/types/client/charts/Chart.d.ts +16 -8
  7. package/dist/types/client/charts/DotWhisker.d.ts +50 -0
  8. package/dist/types/client/charts/Sparkline.d.ts +1 -1
  9. package/dist/types/client/charts/Waterfall.d.ts +26 -0
  10. package/dist/types/client/charts/index.d.ts +2 -0
  11. package/dist/types/client/charts/types.d.ts +8 -2
  12. package/dist/types/client/routes/SpotlightRoute.d.ts +7 -0
  13. package/dist/types/client/routes/index.d.ts +1 -0
  14. package/dist/types/client/spotlight/Compounding.d.ts +8 -0
  15. package/dist/types/client/spotlight/Coverage.d.ts +6 -0
  16. package/dist/types/client/spotlight/Economics.d.ts +10 -0
  17. package/dist/types/client/spotlight/FetchBack.d.ts +9 -0
  18. package/dist/types/client/spotlight/Hero.d.ts +12 -0
  19. package/dist/types/client/spotlight/Overhead.d.ts +6 -0
  20. package/dist/types/client/spotlight/Ranking.d.ts +6 -0
  21. package/dist/types/client/spotlight/ResultsFeed.d.ts +6 -0
  22. package/dist/types/client/spotlight/Savings.d.ts +10 -0
  23. package/dist/types/client/spotlight/ShareTrend.d.ts +8 -0
  24. package/dist/types/client/spotlight/Steps.d.ts +6 -0
  25. package/dist/types/client/spotlight/TimeCost.d.ts +6 -0
  26. package/dist/types/client/spotlight/chips.d.ts +16 -0
  27. package/dist/types/client/spotlight/meta.d.ts +64 -0
  28. package/dist/types/db.d.ts +26 -0
  29. package/dist/types/index.d.ts +2 -1
  30. package/dist/types/parser.d.ts +4 -0
  31. package/dist/types/rollup.d.ts +22 -1
  32. package/dist/types/shared-types.d.ts +419 -0
  33. package/dist/types/spotlight-aggregator.d.ts +189 -0
  34. package/dist/types/spotlight-bootstrap.d.ts +32 -0
  35. package/dist/types/spotlight-ingest.d.ts +258 -0
  36. package/dist/types/spotlight-measures.d.ts +51 -0
  37. package/package.json +5 -5
  38. package/src/aggregator.ts +3 -1
  39. package/src/client/App.tsx +3 -0
  40. package/src/client/api.ts +21 -0
  41. package/src/client/app/nav.ts +4 -1
  42. package/src/client/charts/Chart.tsx +105 -22
  43. package/src/client/charts/DotWhisker.tsx +149 -0
  44. package/src/client/charts/Sparkline.tsx +3 -2
  45. package/src/client/charts/Waterfall.tsx +86 -0
  46. package/src/client/charts/index.ts +2 -0
  47. package/src/client/charts/types.ts +9 -2
  48. package/src/client/react-css.d.ts +1 -1
  49. package/src/client/routes/SpotlightRoute.tsx +138 -0
  50. package/src/client/routes/index.ts +1 -0
  51. package/src/client/spotlight/Compounding.tsx +222 -0
  52. package/src/client/spotlight/Coverage.tsx +96 -0
  53. package/src/client/spotlight/Economics.tsx +167 -0
  54. package/src/client/spotlight/FetchBack.tsx +222 -0
  55. package/src/client/spotlight/Hero.tsx +177 -0
  56. package/src/client/spotlight/Overhead.tsx +135 -0
  57. package/src/client/spotlight/Ranking.tsx +248 -0
  58. package/src/client/spotlight/ResultsFeed.tsx +354 -0
  59. package/src/client/spotlight/Savings.tsx +355 -0
  60. package/src/client/spotlight/ShareTrend.tsx +103 -0
  61. package/src/client/spotlight/Steps.tsx +225 -0
  62. package/src/client/spotlight/TimeCost.tsx +64 -0
  63. package/src/client/spotlight/chips.tsx +63 -0
  64. package/src/client/spotlight/meta.ts +206 -0
  65. package/src/client/spotlight/spotlight.css +1387 -0
  66. package/src/client/styles.css +289 -1
  67. package/src/db.ts +480 -2
  68. package/src/index.ts +10 -1
  69. package/src/live.ts +21 -9
  70. package/src/parser.ts +44 -8
  71. package/src/rollup.ts +203 -10
  72. package/src/server.ts +42 -0
  73. package/src/shared-types.ts +453 -0
  74. package/src/spotlight-aggregator.ts +1738 -0
  75. package/src/spotlight-bootstrap.ts +110 -0
  76. package/src/spotlight-ingest.ts +981 -0
  77. package/src/spotlight-measures.ts +667 -0
@@ -300,6 +300,425 @@ export interface GainDashboardStats {
300
300
  /** All distinct projects seen in the data, for the selector. */
301
301
  projects: string[];
302
302
  }
303
+ /**
304
+ * A tool the Spotlight experiment shaped, by the name it was called with (`read`, `grep`, an MCP tool's name, …).
305
+ * Every tool's results are shaped (`spotlight.experiment.tools`, default all); read, bash and eval have their own
306
+ * paths in pi, every other tool goes through the generic one.
307
+ */
308
+ export type SpotlightTool = string;
309
+ /** One tool's totals, for lists ordered by the tool's saving. */
310
+ export interface SpotlightToolRow extends SpotlightToolTotals {
311
+ tool: SpotlightTool;
312
+ }
313
+ /**
314
+ * How the model got back output a shortened result hid:
315
+ * - `artifact`: a later call read or searched an artifact the shortened result named (`artifact://N`);
316
+ * - `reread`: a later `read` of the same file whose selection overlaps the hidden lines, or the whole file again;
317
+ * - `rerun`: a later call of the same tool repeating the shortened one: a bash command with a pipe, redirect or
318
+ * changed flags; any other tool with near-identical arguments (same tool, argument tokens' Jaccard ≥ 0.6).
319
+ */
320
+ export type SpotlightRecoveryKind = "artifact" | "reread" | "rerun";
321
+ /** A ratio over sessions with its bootstrap 90% interval (`lo`/`hi` null under 3 sessions; `value` null with none). */
322
+ export interface SpotlightEstimate {
323
+ value: number | null;
324
+ lo: number | null;
325
+ hi: number | null;
326
+ }
327
+ /**
328
+ * Token totals of one tool's evaluated results. Savings are measured against the `plain` text pi would have shown
329
+ * unshaped for the same call (its inline cap and read page), not against the full output shaping reads.
330
+ */
331
+ export interface SpotlightToolTotals {
332
+ /** Results the experiment evaluated. */
333
+ results: number;
334
+ /** Results it shortened. */
335
+ shaped: number;
336
+ /** The full output shaping read (a spilled bash output's whole artifact, a whole file up to 2 MiB). */
337
+ inTokens: number;
338
+ /**
339
+ * What `plain` would have shown: the trace's `plainTokens`, else (read, bash and eval traces written before pi
340
+ * recorded it) estimated from the inline cap.
341
+ */
342
+ plainTokens: number;
343
+ outTokens: number;
344
+ /**
345
+ * Σ (plainTokens − outTokens) over shortened results; a cut that showed more than `plain` would have counts
346
+ * negative (see {@link SpotlightGross.widerThanPlain}).
347
+ */
348
+ savedTokens: number;
349
+ /** Σ (inTokens − outTokens): the saving against the full output, which overstates what the model was spared. */
350
+ fullSavedTokens: number;
351
+ }
352
+ /** Strategy 1, gross savings: what the cuts removed. */
353
+ export interface SpotlightGross extends SpotlightToolTotals {
354
+ /** savedTokens / (savedTokens + every tool-output token the model saw in the same sessions). */
355
+ ofToolOutput: number | null;
356
+ /** savedTokens / every token of the same sessions' requests (input + output + cache read + cache write). */
357
+ ofSessionTokens: number | null;
358
+ /** Every tool with an evaluated result in the range, the largest saving first. */
359
+ byTool: SpotlightToolRow[];
360
+ /** Shortened results whose `plainTokens` is estimated (journaled before pi recorded it). */
361
+ estimatedPlain: number;
362
+ /**
363
+ * Shortened results whose cut was wider than what `plain` would have shown (out > plain): a loss where the cut
364
+ * was shown, a saving of 0 where pi capped it at `plain`'s text (counted by the would-be cut).
365
+ */
366
+ widerThanPlain: number;
367
+ }
368
+ /** Recovery totals of one kind. */
369
+ export interface SpotlightRecoveryTotals {
370
+ events: number;
371
+ /** Tokens of the recovery calls' results, which re-enter the context. */
372
+ tokens: number;
373
+ /** Requests that issued nothing but recovery calls. */
374
+ turns: number;
375
+ /** Provider cost of those requests. */
376
+ turnsUsd: number;
377
+ /**
378
+ * The recovery tokens carried through later requests, priced like {@link SpotlightCompounding.savedUsd}
379
+ * (excluding `turnsUsd`); Σ over kinds + clawback.turnsUsd = compounding.recoveryUsd.
380
+ */
381
+ usd: number;
382
+ }
383
+ /** Strategy 2, clawback: what getting hidden output back cost. */
384
+ export interface SpotlightClawback extends SpotlightRecoveryTotals {
385
+ /** Shortened results with hidden lines: the ones a recovery can follow. */
386
+ withHidden: number;
387
+ /** Of those, the results at least one recovery followed. */
388
+ recovered: number;
389
+ byKind: Record<SpotlightRecoveryKind, SpotlightRecoveryTotals>;
390
+ /** gross.savedTokens − tokens. */
391
+ netTokens: number;
392
+ }
393
+ /** Strategy 3, compounding: savings as every later request of the session would have re-read them. */
394
+ export interface SpotlightCompounding {
395
+ /** Σ over shortened results of saved tokens × later requests of the session until a compaction. */
396
+ avoidedTokens: number;
397
+ /** Recovery tokens carried the same way: they undo what the cut avoided. */
398
+ recoveredCarriedTokens: number;
399
+ /** Prompt tokens the same sessions' requests processed (input + cache read + cache write). */
400
+ promptTokens: number;
401
+ /** (avoided − recoveredCarried) / (promptTokens + avoided − recoveredCarried). */
402
+ avoidedShare: number | null;
403
+ /** Avoided tokens priced at each later request's own cache-read rate, plus one cache write. */
404
+ savedUsd: number;
405
+ /** recoveredCarriedTokens priced the same way, plus clawback.turnsUsd. */
406
+ recoveryUsd: number;
407
+ /** Spotlight's own bill (`spotlight:shape` costUsd). */
408
+ spotlightUsd: number;
409
+ /** savedUsd − recoveryUsd − spotlightUsd. */
410
+ netUsd: number;
411
+ /** Wall time shaping added to tool calls (Σ `spotlight:shape` ms). */
412
+ shapeMs: number;
413
+ }
414
+ /** Strategy 4: one bucket of the tool-output share of context, per session group. */
415
+ export interface SpotlightSharePoint {
416
+ /** Bucket start (the range's bucket size, as every time series). */
417
+ timestamp: number;
418
+ /** `shaped`: an experiment session from its marker on; `outside`: everything else. */
419
+ group: "shaped" | "outside";
420
+ sessions: number;
421
+ /** Tool-result tokens the model saw. */
422
+ toolTokens: number;
423
+ /** Everything that entered the context: tool results, assistant output, user prompts and the first request's prompt. */
424
+ contextTokens: number;
425
+ }
426
+ /** One bucket of savings over time. */
427
+ export interface SpotlightBucket {
428
+ timestamp: number;
429
+ results: number;
430
+ shaped: number;
431
+ savedTokens: number;
432
+ recoveries: number;
433
+ recoveredTokens: number;
434
+ netTokens: number;
435
+ savedUsd: number;
436
+ netUsd: number;
437
+ }
438
+ /** A recovery attributed to the latest shortened result it fetches from. */
439
+ export interface SpotlightRecovery {
440
+ kind: SpotlightRecoveryKind;
441
+ toolCallId: string;
442
+ tool: string;
443
+ /** The call's command, path or pattern, clipped to 200 chars. */
444
+ target: string;
445
+ timestamp: number;
446
+ tokens: number;
447
+ /** Tool calls between the shortened result and this one. */
448
+ after: number;
449
+ /** Its request issued nothing but recovery calls. */
450
+ turn: boolean;
451
+ }
452
+ /** One evaluated result that changed the output, or would have (`holdout`). */
453
+ export interface SpotlightResultRow {
454
+ sessionFile: string;
455
+ toolCallId: string;
456
+ timestamp: number;
457
+ tool: SpotlightTool;
458
+ /** Held out: the model saw the result unshaped; `savedTokens` is what the cut would have saved. */
459
+ holdout: boolean;
460
+ /** Command, path (with selector) or eval code head, clipped to 200 chars. */
461
+ target: string;
462
+ inTokens: number;
463
+ outTokens: number;
464
+ savedTokens: number;
465
+ hiddenLines: number;
466
+ steps: string[];
467
+ spotlightUsd: number;
468
+ shapeMs: number;
469
+ /** What `plain` would have shown (see {@link SpotlightToolTotals.plainTokens}). */
470
+ plainTokens: number;
471
+ /** Cut by the no-score fallback rules because the score-free coin skipped Spotlight. */
472
+ scoreFree: boolean;
473
+ recoveries: SpotlightRecovery[];
474
+ /** savedTokens − Σ recovery tokens. */
475
+ netTokens: number;
476
+ }
477
+ /**
478
+ * Who saw what for a result in the fetch-back comparison:
479
+ * - `applied`: Spotlight shortened it and the model saw the shortened text;
480
+ * - `holdout`: Spotlight would have shortened it, but the result was held out at random and the model saw it
481
+ * unshaped (the experiment's true counterfactual);
482
+ * - `reference`: shaping left it unchanged (results of the same tool and sessions, standardized to the applied
483
+ * cuts' original-size mix), the stand-in until held-out results accumulate;
484
+ * - `scoreFree`: shortened by the no-score fallback rules because a random coin skipped Spotlight
485
+ * (`spotlight.experiment.scoreFree`), the model saw the cut: against `applied`, what Spotlight's scores add;
486
+ * - `gated`: a read pi's former edit gate (removed) kept whole, the model saw plain's page; only in transcripts
487
+ * journaled while the gate ran.
488
+ * `applied` holds only cuts Spotlight scored (score-free cuts and cuts whose Spotlight calls failed are not in it).
489
+ */
490
+ export type SpotlightCutGroup = "applied" | "holdout" | "reference" | "scoreFree" | "gated";
491
+ /**
492
+ * Share of results the agent fetched more of within 12 calls, by one rule for every group: a later `read` of the
493
+ * same path overlapping the result's own window (the whole file without a selector), a rerun (see
494
+ * {@link SpotlightRecoveryKind}), or a read of an artifact the result named; not counted after an edit of that path
495
+ * (reads) or any edit (reruns), nor from the same request.
496
+ */
497
+ export interface SpotlightFetchBackCell {
498
+ tool: SpotlightTool;
499
+ group: SpotlightCutGroup;
500
+ results: number;
501
+ sessions: number;
502
+ /** Session-clustered bootstrap over the group's results. */
503
+ rate: SpotlightEstimate;
504
+ }
505
+ /** What the cuts of one tool earned, in tokens and dollars (applied cuts only; held-out results save nothing). */
506
+ export interface SpotlightCutEconomics {
507
+ tool: SpotlightTool;
508
+ cuts: number;
509
+ savedTokens: number;
510
+ recoveredTokens: number;
511
+ /** Requests that did nothing but fetch back output these cuts hid. */
512
+ extraTurns: number;
513
+ savedUsd: number;
514
+ /** Recovered tokens carried through later requests, priced like the savings. */
515
+ recoveryUsd: number;
516
+ /** Provider cost of the extra turns. */
517
+ turnUsd: number;
518
+ /** savedUsd − recoveryUsd − turnUsd − this cell's Spotlight bill. */
519
+ netUsd: number;
520
+ extraTurnsPer100Cuts: number;
521
+ netUsdPer100Cuts: number;
522
+ }
523
+ /** Calls between a cut (or held-out result) and the first fetch-back after it. */
524
+ export type SpotlightGapBin = "1" | "2" | "3" | "4–6" | "7–12";
525
+ /** Fetch-backs per gap bin and tool, for applied and held-out results: every bin of each tool with at least one. */
526
+ export interface SpotlightGapCount {
527
+ tool: SpotlightTool;
528
+ gap: SpotlightGapBin;
529
+ applied: number;
530
+ holdout: number;
531
+ }
532
+ /** What shaping costs in time and reliability (all evaluated results, held-out ones included). */
533
+ export interface SpotlightOverhead {
534
+ results: number;
535
+ /** Wall time shaping added to a tool call. */
536
+ shapeMsP50: number | null;
537
+ shapeMsP90: number | null;
538
+ /** Spotlight's own request time, over results whose calls all answered. */
539
+ spotlightMsP50: number | null;
540
+ spotlightMsP90: number | null;
541
+ /** Results whose Spotlight calls failed or timed out (shaped by the no-score fallback). */
542
+ failureRate: number | null;
543
+ /** Wall time tool calls spent shaping results whose Spotlight calls failed (mostly the 5 s timeout per call). */
544
+ failedWaitMs: number;
545
+ spotlightUsd: number;
546
+ }
547
+ /**
548
+ * The net once fetch-backs that would have happened anyway are taken out: held-out results (the model saw them
549
+ * unshaped) show how much of a would-be cut's output the agent reads again regardless, per tool; only recoveries
550
+ * beyond that baseline are the cut's cost. A tool's baseline is thin while it has fewer than `minHoldouts`.
551
+ */
552
+ export interface SpotlightAdjustedTool {
553
+ tool: SpotlightTool;
554
+ /** Applied cuts (scored and score-free) and their recovered tokens. */
555
+ cuts: number;
556
+ recoveredTokens: number;
557
+ /** Held-out would-be cuts the baseline is measured on. */
558
+ holdouts: number;
559
+ /** Recovered tokens per held-out would-be cut, over all the tool's holdouts (session-clustered bootstrap). */
560
+ baselinePerCut: SpotlightEstimate;
561
+ /** max(0, recovered − baseline per cut × cuts). */
562
+ excessTokens: SpotlightEstimate;
563
+ }
564
+ /** Holdout-adjusted net savings (tokens, against the `plain` baseline). */
565
+ export interface SpotlightAdjusted {
566
+ byTool: SpotlightAdjustedTool[];
567
+ /** Σ tools' excess (bootstrapped jointly over sessions). */
568
+ excessTokens: SpotlightEstimate;
569
+ /** gross.savedTokens − excessTokens. */
570
+ netTokens: SpotlightEstimate;
571
+ /** compounding.savedUsd − recoveryUsd × (excess / recovered) − spotlightUsd. */
572
+ netUsd: SpotlightEstimate;
573
+ /** Holdouts per tool below which its baseline is too thin to trust (the UI marks the tool). */
574
+ minHoldouts: number;
575
+ }
576
+ /**
577
+ * Lines of a shaped read's original the agent needed later: lines a later `read` of the same file selected, and lines
578
+ * a later `edit` of it changed (located by its Find text in the original), until the file's first edit or a later
579
+ * whole-file read.
580
+ */
581
+ export type SpotlightNeedSource = "reread" | "edit";
582
+ /**
583
+ * How well a group's per-line choices match what the agent needed later, over shaped reads with at least one needed
584
+ * line. Applied and score-free cuts are biased alike (the agent re-reads what it lacks), so compare them with each
585
+ * other; held-out would-be cuts are the unbiased measure.
586
+ */
587
+ export interface SpotlightRankingCell {
588
+ group: "applied" | "holdout" | "scoreFree";
589
+ results: number;
590
+ neededLines: number;
591
+ /** P(heat of a needed line > heat of an unneeded line of the same result), ties half; 0.5 is chance. Null for `scoreFree` (no heat). */
592
+ auc: SpotlightEstimate;
593
+ /** Needed-line rate among each result's hottest 10% of lines over its rate among all lines; 1 is no better than chance. */
594
+ topLift: SpotlightEstimate;
595
+ /** Share of needed lines the cut kept over the share of all lines it kept; 1 is a random cut of the same size. */
596
+ keptLift: SpotlightEstimate;
597
+ }
598
+ /** Mean heat (0–9) of one line position over shaped results of at least 40 lines. */
599
+ export interface SpotlightHeatPosition {
600
+ /** `line 1`, `lines 2–3`, then `0–10%` … `90–100%` of the output. */
601
+ label: string;
602
+ meanHeat: number;
603
+ lines: number;
604
+ }
605
+ /** Ranking quality: does Spotlight keep the lines the agent comes back for? */
606
+ export interface SpotlightRanking {
607
+ cells: SpotlightRankingCell[];
608
+ position: SpotlightHeatPosition[];
609
+ /** Results with needed lines, by how the need was observed (a result can count under both). */
610
+ sources: Record<SpotlightNeedSource, number>;
611
+ }
612
+ /** Saved-token bins (against `plain`) the step economics split cuts by; a negative saving falls in `<250`. */
613
+ export type SpotlightSizeBin = "<250" | "250–1k" | "1k–4k" | "≥4k";
614
+ /** What one shaping step (or combination) earns at one cut size: applied cuts of one tool. */
615
+ export interface SpotlightStepCell {
616
+ tool: SpotlightTool;
617
+ /** The cut's shaping steps joined with `+` (`outline+window`); combinations under 10 cuts in the range fold into `other`. */
618
+ steps: string;
619
+ size: SpotlightSizeBin;
620
+ cuts: number;
621
+ fetchBackRate: SpotlightEstimate;
622
+ /** Σ (plain − out), negative cuts (wider than `plain`) included. */
623
+ savedTokens: number;
624
+ recoveredTokens: number;
625
+ /** max(0, recovered − Σ each cut's holdout baseline (its tool's)). */
626
+ excessTokens: number;
627
+ /** (saved − excess) / saved: the share of the saving that stays saved; null when the cell saved nothing (≤ 0). */
628
+ keptShare: number | null;
629
+ }
630
+ /** What the experiment costs in agent time. */
631
+ export interface SpotlightTimeCost {
632
+ /** Model time of the experiment sessions' requests (Σ request duration). */
633
+ agentMs: number;
634
+ /** Requests that did nothing but fetch back cut output, and their model time. */
635
+ fetchBackTurns: number;
636
+ fetchBackMs: number;
637
+ /** Wall time tool calls spent shaping (Spotlight requests included). */
638
+ shapeMs: number;
639
+ /** (fetchBackMs + shapeMs) / agentMs. */
640
+ share: number | null;
641
+ }
642
+ /** One tool's share of the tool output experiment sessions' models saw (chars ÷ 4). */
643
+ export interface SpotlightCoverageRow {
644
+ tool: string;
645
+ /** The experiment evaluated results of it in the range. */
646
+ shaped: boolean;
647
+ calls: number;
648
+ tokens: number;
649
+ share: number;
650
+ }
651
+ /**
652
+ * Which transcripts the Spotlight dashboard measures (`GET /api/stats/spotlight?agent=`, default `all`): `main`
653
+ * agents only, task `subagent`s only, or everything (advisors appear only under `all`). Every metric, the share
654
+ * trend's `outside` group included, is filtered the same way.
655
+ */
656
+ export type SpotlightAgentScope = "all" | "main" | "subagent";
657
+ /** An experiment session, for the context chart's picker. */
658
+ export interface SpotlightSessionSummary {
659
+ sessionFile: string;
660
+ title: string | null;
661
+ folder: string;
662
+ /** Which agent wrote the transcript (each subagent transcript is its own experiment session). */
663
+ agentType: AgentType;
664
+ /** The session's first Spotlight marker: activity before it (a resumed pre-experiment session) is not measured. */
665
+ startedAt: number;
666
+ requests: number;
667
+ savedTokens: number;
668
+ netUsd: number;
669
+ }
670
+ /**
671
+ * Spotlight dashboard payload (`GET /api/stats/spotlight?range=&agent=`). Every experiment session shapes tool
672
+ * output; held-out results within them are the counterfactual. Only activity at or after a session's Spotlight
673
+ * marker is the experiment's; earlier activity of the same transcript counts as `outside` in the share trend and is
674
+ * left out of every other metric.
675
+ */
676
+ export interface SpotlightDashboardStats {
677
+ /** Experiment sessions with activity in the range. */
678
+ sessions: number;
679
+ /** First Spotlight marker ever seen (not range-bound), for the trend's marker; null before the experiment. */
680
+ experimentStart: number | null;
681
+ gross: SpotlightGross;
682
+ clawback: SpotlightClawback;
683
+ compounding: SpotlightCompounding;
684
+ series: SpotlightBucket[];
685
+ share: SpotlightSharePoint[];
686
+ /** Latest 60 shortened results, newest first. */
687
+ recent: SpotlightResultRow[];
688
+ /** Latest 30 experiment sessions in the range, newest first. */
689
+ sessionList: SpotlightSessionSummary[];
690
+ /** Held-out results in the range (would-be cuts the model saw unshaped). */
691
+ holdouts: number;
692
+ /** Score-free cuts in the range (shortened without Spotlight by the fallback rules). */
693
+ scoreFree: number;
694
+ adjusted: SpotlightAdjusted;
695
+ ranking: SpotlightRanking;
696
+ steps: SpotlightStepCell[];
697
+ timeCost: SpotlightTimeCost;
698
+ coverage: SpotlightCoverageRow[];
699
+ fetchBack: SpotlightFetchBackCell[];
700
+ economics: SpotlightCutEconomics[];
701
+ gaps: SpotlightGapCount[];
702
+ overhead: SpotlightOverhead;
703
+ }
704
+ /** One request of a session's context chart. */
705
+ export interface SpotlightContextPoint {
706
+ timestamp: number;
707
+ /** input + cache read + cache write of this request. */
708
+ contextTokens: number;
709
+ /** The context without shaping: contextTokens + net tokens saved so far (reset at a compaction). */
710
+ withoutTokens: number;
711
+ /** The context shrank sharply before this request (a compaction). */
712
+ compacted: boolean;
713
+ /** Tokens saved and recovered by results that landed since the previous request. */
714
+ saved: number;
715
+ recovered: number;
716
+ }
717
+ /** A session's context with and without shaping (`GET /api/stats/spotlight/session?file=`). */
718
+ export interface SpotlightSessionContext {
719
+ session: SpotlightSessionSummary;
720
+ points: SpotlightContextPoint[];
721
+ }
303
722
  /**
304
723
  * Aggregated usage for a single tool over the active range.
305
724
  *
@@ -0,0 +1,189 @@
1
+ /**
2
+ * The Spotlight dashboard: what tool-output shaping saved, measured four ways (see the doc comments of
3
+ * {@link SpotlightDashboardStats} and the types it references), from the tables `spotlight-ingest.ts` fills.
4
+ *
5
+ * Approximations, all named in the UI's captions:
6
+ * - token counts of results without a Spotlight trace are result characters / 4;
7
+ * - what pi would have shown unshaped (`plain`), on read/bash/eval traces written before pi recorded it, is the input
8
+ * scaled to pi's inline cap (see {@link estimatedPlainTokens});
9
+ * - a compaction is a request whose prompt (input + cache read + cache write) is under 60% of the previous one's;
10
+ * - a rerun is a bash command equal to, extending (pipe, redirect, `&&`), or ≥ 60% token-similar to a recent one;
11
+ * for any other tool but `read`, a call of the same tool whose argument tokens (`argumentTokens`) are ≥ 60%
12
+ * similar.
13
+ */
14
+ import type { Database } from "bun:sqlite";
15
+ import { type RangeWindow } from "./rollup.js";
16
+ import type { AgentType, SpotlightDashboardStats, SpotlightRecoveryKind, SpotlightAgentScope, SpotlightSessionContext, SpotlightSharePoint, SpotlightTool } from "./shared-types.js";
17
+ export interface SessionRow {
18
+ session_file: string;
19
+ folder: string;
20
+ marked_at: number;
21
+ cwd: string | null;
22
+ title: string | null;
23
+ }
24
+ export interface Result {
25
+ timestamp: number;
26
+ tool: string;
27
+ inTokens: number;
28
+ outTokens: number;
29
+ /** What `plain` would have shown: the trace's count, else {@link estimatedPlainTokens}. */
30
+ plainTokens: number;
31
+ plainEstimated: boolean;
32
+ /** plain − out of a cut or would-be cut (held out), negative when the cut was wider than plain; 0 if unchanged. */
33
+ savedTokens: number;
34
+ /** in − out of a cut or would-be cut: the saving against the full output. */
35
+ fullSavedTokens: number;
36
+ steps: string[];
37
+ /** Shaping changed it and the model saw the changed text (an applied cut). */
38
+ shortened: boolean;
39
+ /** Shaping would have shortened it, but it was held out: the model saw it unshaped. */
40
+ holdout: boolean;
41
+ /** The score-free coin skipped Spotlight: the no-score fallback rules shaped it. */
42
+ scoreFree: boolean;
43
+ /**
44
+ * A read pi's former edit gate (removed) kept whole: the model saw `plain`'s page, the counts describe the
45
+ * would-be cut. Only in transcripts journaled while the gate ran.
46
+ */
47
+ gated: boolean;
48
+ /**
49
+ * The cut (shown, or not applied because pi capped it at `plain`'s text) was wider than what `plain` would have
50
+ * shown.
51
+ */
52
+ widerThanPlain: boolean;
53
+ /** Some Spotlight call failed: the no-score fallback rules shaped it. */
54
+ failed: boolean;
55
+ /** Per-line heat (`0`–`9`, else unscored) and kept runs (1-based positions) of the original; see the trace. */
56
+ heat: string;
57
+ kept: [number, number][];
58
+ /** Original lines (1-based positions) later calls needed; null for results that collect none. */
59
+ needs: {
60
+ reread: [number, number][];
61
+ edit: [number, number][];
62
+ } | null;
63
+ path: string | null;
64
+ /** The original window (`details.spotlight.original`): first line and line count; 0 lines without one. */
65
+ start: number;
66
+ lines: number;
67
+ hidden: [number, number][];
68
+ hiddenLines: number;
69
+ spotlightCalls: number;
70
+ spotlightFailures: number;
71
+ spotlightMs: number;
72
+ spotlightUsd: number;
73
+ /** Wall time shaping took; null before its shape entry landed. */
74
+ shapeMs: number | null;
75
+ }
76
+ export interface Recovery {
77
+ kind: SpotlightRecoveryKind;
78
+ source: Call;
79
+ /** Tool calls between the source result and this call. */
80
+ after: number;
81
+ /** Every call of this call's request is a recovery. */
82
+ turn: boolean;
83
+ }
84
+ export interface Call {
85
+ id: string;
86
+ entryId: string;
87
+ timestamp: number;
88
+ seq: number;
89
+ tool: string;
90
+ target: string;
91
+ callsInTurn: number;
92
+ /** Tokens the result added to the context: the trace's `outTokens`, else result characters / 4. */
93
+ tokens: number;
94
+ artifacts: Set<number>;
95
+ result: Result | null;
96
+ recovery: Recovery | null;
97
+ /** Recoveries this call's result was the source of. */
98
+ recoveredBy: Call[];
99
+ /** Files the call edits, resolved against the session's working directory; empty for other calls. */
100
+ edits: string[];
101
+ /** The call's argument tokens (generic rerun rule); empty for `read`, `bash` and unrecorded calls. */
102
+ argTokens: Set<string>;
103
+ /** Calls from an evaluated result to the first call fetching more of it (the uniform rule); null for none. */
104
+ fetchBackGap: number | null;
105
+ /** Evaluated results this call is the first fetch-back of (the uniform rule). */
106
+ fetchesBack: Call[];
107
+ /** A held-out result's would-be recoveries: tokens of later calls that would have recovered the cut. */
108
+ baselineTokens: number;
109
+ /** A shortened result's saved tokens × the requests that carried them, and their price. */
110
+ savedCarry: number;
111
+ savedCarryUsd: number;
112
+ /** A recovery's tokens × the requests that carried them, and their price. */
113
+ recoveryCarry: number;
114
+ recoveryCarryUsd: number;
115
+ }
116
+ export interface Request {
117
+ entryId: string;
118
+ timestamp: number;
119
+ /** Model time of the request (ms). */
120
+ durationMs: number;
121
+ context: number;
122
+ costTotal: number;
123
+ compacted: boolean;
124
+ readRate: number;
125
+ writeRate: number;
126
+ }
127
+ export interface SessionAnalysis {
128
+ session: SessionRow;
129
+ cwd: string | null;
130
+ calls: Call[];
131
+ requests: Request[];
132
+ /** Request cost by assistant entry id (main-loop requests). */
133
+ costByEntry: Map<string, number>;
134
+ agentType: AgentType;
135
+ edits: EditLog;
136
+ }
137
+ /**
138
+ * What `plain` would have shown of a read/bash/eval result whose trace predates `plainTokens`: its input scaled to
139
+ * the inline cap ({@link PLAIN_CAP_BYTES}), in × min(1, cap / inChars). Null for any other tool: the generic path
140
+ * shapes the text `plain` shows, so its input is that text.
141
+ */
142
+ export declare function estimatedPlainTokens(tool: string, inTokens: number, inChars: number): number | null;
143
+ /** Whether bash command `next` repeats `previous`: equal, extended by a pipe/redirect/`&&`, or similar enough. */
144
+ export declare function isRerun(previous: string, next: string): boolean;
145
+ /** The session's file edits in call order: whether a file (or any file) changed between two calls. */
146
+ export declare class EditLog {
147
+ #private;
148
+ /** `calls` sorted by seq, their `edits` resolved. */
149
+ constructor(calls: readonly Call[]);
150
+ /** An edit strictly between calls `after` and `before`, of `file` (resolved) when given, else of any file. */
151
+ between(after: number, before: number, file?: string): boolean;
152
+ }
153
+ /** Raw per-bucket facts of one session over the range. */
154
+ interface SessionFacts {
155
+ /** bucket → [result chars, output tokens, user chars] */
156
+ buckets: Map<number, [number, number, number]>;
157
+ totalTokens: number;
158
+ promptTokens: number;
159
+ }
160
+ /** An experiment session as the share trend splits it: its activity from `markedAt` on is `shaped`. */
161
+ export interface ShareExperiment {
162
+ markedAt: number;
163
+ /** Prompt of its first request at or after the marker: the context shaping started from. */
164
+ firstPrompt: number;
165
+ /** Its facts from the marker (or the window start, if later) on. */
166
+ facts: SessionFacts;
167
+ }
168
+ /** Whether a transcript of `agentType` is in `scope`; advisors only count under `all`. */
169
+ export declare function inAgentScope(scope: SpotlightAgentScope, agentType: AgentType): boolean;
170
+ /**
171
+ * Strategy 4 over every transcript of `scope` in the window. Totals come from the rollups; experiment sessions are
172
+ * read raw (they are few) and subtracted to leave `outside`, so the cost does not grow with history. A transcript's
173
+ * activity before its marker stays `outside`.
174
+ */
175
+ export declare function computeShareSeries(database: Database, window: RangeWindow, experiments: ReadonlyMap<string, ShareExperiment>, scope?: SpotlightAgentScope, now?: number): SpotlightSharePoint[];
176
+ /** Build the dashboard payload for a window and agent scope; `now` pins the series end (tests). */
177
+ export declare function buildSpotlightDashboard(database: Database, window: RangeWindow, scope?: SpotlightAgentScope, now?: number): SpotlightDashboardStats;
178
+ /** An evaluated result in the range. */
179
+ export interface Evaluated {
180
+ call: Call;
181
+ result: Result;
182
+ analysis: SessionAnalysis;
183
+ tool: SpotlightTool;
184
+ }
185
+ /** `GET /api/stats/spotlight?range=&agent=`; an unknown `agent` measures every transcript. */
186
+ export declare function getSpotlightDashboardStats(range?: string | null, agent?: string | null): Promise<SpotlightDashboardStats>;
187
+ /** `GET /api/stats/spotlight/session?file=`; null when the file is not an experiment session. */
188
+ export declare function getSpotlightSessionContext(sessionFile: string): Promise<SpotlightSessionContext | null>;
189
+ export {};
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Seeded session-clustered bootstrap for the Spotlight dashboard's estimates: 1000 resamples (seeded per key, so
3
+ * every request returns the same interval), 5th and 95th percentiles, no interval under 3 sessions.
4
+ */
5
+ import type { SpotlightEstimate } from "./shared-types.js";
6
+ /** Fewest sessions an interval is drawn from. */
7
+ export declare const BOOTSTRAP_MIN_SESSIONS = 3;
8
+ /** Linear-interpolated quantile `p` of ascending `sorted` (non-empty). */
9
+ export declare function quantile(sorted: ArrayLike<number>, p: number): number;
10
+ /**
11
+ * The 90% interval of a statistic over seeded resamples: `resample` draws its own resample of the sessions with
12
+ * `random` (uniform in [0, 1)) and returns the statistic, or null when the resample cannot define it.
13
+ */
14
+ export declare function bootstrapInterval(key: string, resample: (random: () => number) => number | null): Pick<SpotlightEstimate, "lo" | "hi">;
15
+ /** 5th and 95th percentiles of bootstrap samples (sorted in place); null without any. */
16
+ export declare function percentileInterval(samples: number[] | Float64Array): Pick<SpotlightEstimate, "lo" | "hi">;
17
+ /**
18
+ * A joint bootstrap: one array of {@link BOOTSTRAP_RESAMPLES} seeded resample rounds that several statistics share.
19
+ * `round` draws one resample with `random` and stores each statistic of it at index `r` of that statistic's array
20
+ * (made by {@link jointSamples}); {@link percentileInterval} then turns every array into its interval. One pass
21
+ * replaces a {@link bootstrapInterval} per statistic.
22
+ */
23
+ export declare function bootstrapRounds(key: string, round: (random: () => number, r: number) => void): void;
24
+ /** One statistic's samples for {@link bootstrapRounds}. */
25
+ export declare function jointSamples(): Float64Array;
26
+ /**
27
+ * The means of seeded resamples of `values` (one per bootstrap round, in round order), so statistics of several
28
+ * groups can be combined round by round.
29
+ */
30
+ export declare function resampledMeans(values: readonly number[], key: string): Float64Array;
31
+ /** Ratio of sums over sessions (`[numerator, denominator]` per session) with its bootstrap interval. */
32
+ export declare function bootstrapRatio(pairs: readonly (readonly [number, number])[], key: string): SpotlightEstimate;