@oh-my-pi/omp-stats 18.16.0 → 18.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/client/index.css +1 -1
- package/dist/client/index.js +102 -99
- package/dist/types/client/api.d.ts +4 -1
- package/dist/types/client/app/nav.d.ts +1 -1
- package/dist/types/client/charts/Chart.d.ts +16 -8
- package/dist/types/client/charts/DotWhisker.d.ts +50 -0
- package/dist/types/client/charts/Sparkline.d.ts +1 -1
- package/dist/types/client/charts/Waterfall.d.ts +26 -0
- package/dist/types/client/charts/index.d.ts +2 -0
- package/dist/types/client/charts/types.d.ts +8 -2
- package/dist/types/client/routes/SpotlightRoute.d.ts +7 -0
- package/dist/types/client/routes/index.d.ts +1 -0
- package/dist/types/client/spotlight/Compounding.d.ts +8 -0
- package/dist/types/client/spotlight/Coverage.d.ts +6 -0
- package/dist/types/client/spotlight/Economics.d.ts +10 -0
- package/dist/types/client/spotlight/FetchBack.d.ts +9 -0
- package/dist/types/client/spotlight/Hero.d.ts +12 -0
- package/dist/types/client/spotlight/Overhead.d.ts +6 -0
- package/dist/types/client/spotlight/Ranking.d.ts +6 -0
- package/dist/types/client/spotlight/ResultsFeed.d.ts +6 -0
- package/dist/types/client/spotlight/Savings.d.ts +10 -0
- package/dist/types/client/spotlight/ShareTrend.d.ts +8 -0
- package/dist/types/client/spotlight/Steps.d.ts +6 -0
- package/dist/types/client/spotlight/TimeCost.d.ts +6 -0
- package/dist/types/client/spotlight/chips.d.ts +16 -0
- package/dist/types/client/spotlight/meta.d.ts +64 -0
- package/dist/types/db.d.ts +26 -0
- package/dist/types/index.d.ts +2 -1
- package/dist/types/parser.d.ts +4 -0
- package/dist/types/rollup.d.ts +22 -1
- package/dist/types/shared-types.d.ts +419 -0
- package/dist/types/spotlight-aggregator.d.ts +189 -0
- package/dist/types/spotlight-bootstrap.d.ts +32 -0
- package/dist/types/spotlight-ingest.d.ts +258 -0
- package/dist/types/spotlight-measures.d.ts +51 -0
- package/package.json +5 -5
- package/src/aggregator.ts +3 -1
- package/src/client/App.tsx +3 -0
- package/src/client/api.ts +21 -0
- package/src/client/app/nav.ts +4 -1
- package/src/client/charts/Chart.tsx +105 -22
- package/src/client/charts/DotWhisker.tsx +149 -0
- package/src/client/charts/Sparkline.tsx +3 -2
- package/src/client/charts/Waterfall.tsx +86 -0
- package/src/client/charts/index.ts +2 -0
- package/src/client/charts/types.ts +9 -2
- package/src/client/react-css.d.ts +1 -1
- package/src/client/routes/SpotlightRoute.tsx +138 -0
- package/src/client/routes/index.ts +1 -0
- package/src/client/spotlight/Compounding.tsx +222 -0
- package/src/client/spotlight/Coverage.tsx +96 -0
- package/src/client/spotlight/Economics.tsx +167 -0
- package/src/client/spotlight/FetchBack.tsx +222 -0
- package/src/client/spotlight/Hero.tsx +177 -0
- package/src/client/spotlight/Overhead.tsx +135 -0
- package/src/client/spotlight/Ranking.tsx +248 -0
- package/src/client/spotlight/ResultsFeed.tsx +354 -0
- package/src/client/spotlight/Savings.tsx +355 -0
- package/src/client/spotlight/ShareTrend.tsx +103 -0
- package/src/client/spotlight/Steps.tsx +225 -0
- package/src/client/spotlight/TimeCost.tsx +64 -0
- package/src/client/spotlight/chips.tsx +63 -0
- package/src/client/spotlight/meta.ts +206 -0
- package/src/client/spotlight/spotlight.css +1387 -0
- package/src/client/styles.css +289 -1
- package/src/db.ts +480 -2
- package/src/index.ts +10 -1
- package/src/live.ts +21 -9
- package/src/parser.ts +44 -8
- package/src/rollup.ts +203 -10
- package/src/server.ts +42 -0
- package/src/shared-types.ts +453 -0
- package/src/spotlight-aggregator.ts +1738 -0
- package/src/spotlight-bootstrap.ts +110 -0
- package/src/spotlight-ingest.ts +981 -0
- package/src/spotlight-measures.ts +667 -0
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
import { useMemo, useState } from "react";
|
|
2
|
+
import { type ChartSeries, Legend, TimeChart, Waterfall, type WaterfallStep } from "../charts";
|
|
3
|
+
import { formatCompact, formatCost, formatInteger } from "../data/formatters";
|
|
4
|
+
import { bucketAxis, rangeMeta } from "../data/range";
|
|
5
|
+
import { densify } from "../data/series";
|
|
6
|
+
import type { SpotlightAdjustedTool, SpotlightDashboardStats, SpotlightToolRow, TimeRange } from "../types";
|
|
7
|
+
import { Card, Segmented } from "../ui";
|
|
8
|
+
import {
|
|
9
|
+
bucketNoun,
|
|
10
|
+
formatInterval,
|
|
11
|
+
formatShare,
|
|
12
|
+
formatSignedCompact,
|
|
13
|
+
formatSignedCost,
|
|
14
|
+
foldTools,
|
|
15
|
+
OTHER_TOOLS,
|
|
16
|
+
RECOVERY_KIND,
|
|
17
|
+
RECOVERY_KINDS,
|
|
18
|
+
toolIcon,
|
|
19
|
+
} from "./meta";
|
|
20
|
+
|
|
21
|
+
type Unit = "tokens" | "usd";
|
|
22
|
+
|
|
23
|
+
const UNIT_OPTIONS = [
|
|
24
|
+
{ value: "tokens", label: "Tokens" },
|
|
25
|
+
{ value: "usd", label: "$" },
|
|
26
|
+
] as const;
|
|
27
|
+
|
|
28
|
+
function UnitToggle({ value, onChange }: { value: Unit; onChange: (unit: Unit) => void }) {
|
|
29
|
+
return <Segmented size="sm" aria-label="Unit" options={UNIT_OPTIONS} value={value} onChange={onChange} />;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface SavingsProps {
|
|
33
|
+
stats: SpotlightDashboardStats;
|
|
34
|
+
range: TimeRange;
|
|
35
|
+
stale: boolean;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Cumulative gross vs net per bucket, with what recoveries took back hanging below zero. */
|
|
39
|
+
export function SavingsOverTime({ stats, range, stale }: SavingsProps) {
|
|
40
|
+
const [unit, setUnit] = useState<Unit>("tokens");
|
|
41
|
+
const meta = rangeMeta(range);
|
|
42
|
+
|
|
43
|
+
const chart = useMemo(() => {
|
|
44
|
+
const buckets = bucketAxis(
|
|
45
|
+
range,
|
|
46
|
+
stats.series.map(b => b.timestamp),
|
|
47
|
+
meta.bucketMs,
|
|
48
|
+
);
|
|
49
|
+
const usd = unit === "usd";
|
|
50
|
+
const saved = densify(stats.series, buckets, b => (usd ? b.savedUsd : b.savedTokens));
|
|
51
|
+
const net = densify(stats.series, buckets, b => (usd ? b.netUsd : b.netTokens));
|
|
52
|
+
let grossRun = 0;
|
|
53
|
+
let netRun = 0;
|
|
54
|
+
const series: ChartSeries[] = [
|
|
55
|
+
{
|
|
56
|
+
key: "gross",
|
|
57
|
+
label: "Gross (cumulative)",
|
|
58
|
+
color: "var(--spot-gross)",
|
|
59
|
+
kind: "area",
|
|
60
|
+
values: saved.map(v => (grossRun += v)),
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
key: "clawback",
|
|
64
|
+
label: usd ? "Recoveries + Spotlight cost" : "Fetched back",
|
|
65
|
+
color: "var(--spot-clawback)",
|
|
66
|
+
kind: "bars",
|
|
67
|
+
values: saved.map((v, i) => net[i] - v),
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
key: "net",
|
|
71
|
+
label: "Net of every fetch-back (cumulative)",
|
|
72
|
+
color: "var(--spot-net)",
|
|
73
|
+
kind: "line",
|
|
74
|
+
values: net.map(v => (netRun += v)),
|
|
75
|
+
},
|
|
76
|
+
];
|
|
77
|
+
return { buckets, series };
|
|
78
|
+
}, [stats.series, range, meta.bucketMs, unit]);
|
|
79
|
+
|
|
80
|
+
const format = unit === "usd" ? formatSignedCost : formatSignedCompact;
|
|
81
|
+
return (
|
|
82
|
+
<Card
|
|
83
|
+
index={1}
|
|
84
|
+
title="Savings over time"
|
|
85
|
+
description={`Running gross and net per ${bucketNoun(meta.bucketMs)}; what recoveries took back hangs below zero`}
|
|
86
|
+
actions={<UnitToggle value={unit} onChange={setUnit} />}
|
|
87
|
+
stale={stale}
|
|
88
|
+
footer={
|
|
89
|
+
unit === "usd"
|
|
90
|
+
? "Dollars price each saved token at the cache-read rate of every later request that would have carried it (plus one cache write); recoveries are priced the same way."
|
|
91
|
+
: "Gross is what the cuts removed against what plain would have shown; this net subtracts every later call that fetched hidden output back, including the share held-out results show comes back anyway (the headline adds that back). Each lands in the bucket of its own timestamp."
|
|
92
|
+
}
|
|
93
|
+
>
|
|
94
|
+
<div className="spot-chart-legend">
|
|
95
|
+
<Legend items={chart.series.map(s => ({ key: s.key, label: s.label, color: s.color }))} />
|
|
96
|
+
</div>
|
|
97
|
+
<TimeChart
|
|
98
|
+
buckets={chart.buckets}
|
|
99
|
+
bucketMs={meta.bucketMs}
|
|
100
|
+
series={chart.series}
|
|
101
|
+
stacked={false}
|
|
102
|
+
// Tall enough to sit level with the bridge and per-tool list beside it.
|
|
103
|
+
height={480}
|
|
104
|
+
format={format}
|
|
105
|
+
formatTooltip={
|
|
106
|
+
unit === "usd" ? formatSignedCost : v => (v < 0 ? `−${formatInteger(-v)}` : formatInteger(v))
|
|
107
|
+
}
|
|
108
|
+
showTotal={false}
|
|
109
|
+
emptyLabel="No shortened results in this range"
|
|
110
|
+
/>
|
|
111
|
+
</Card>
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Bridge from gross to the holdout-adjusted net: every way the saving was undone, and what would have come back anyway. */
|
|
116
|
+
export function WhereSavingsWent({ stats, stale }: Omit<SavingsProps, "range">) {
|
|
117
|
+
const [unit, setUnit] = useState<Unit>("tokens");
|
|
118
|
+
const { gross, clawback, compounding, adjusted } = stats;
|
|
119
|
+
|
|
120
|
+
const steps = useMemo((): WaterfallStep[] => {
|
|
121
|
+
const usd = unit === "usd";
|
|
122
|
+
const top = usd ? compounding.savedUsd : gross.savedTokens;
|
|
123
|
+
const share = (v: number) => (top > 0 ? formatShare(v / top) : "");
|
|
124
|
+
const kindSteps = RECOVERY_KINDS.map((kind): WaterfallStep => {
|
|
125
|
+
const totals = clawback.byKind[kind];
|
|
126
|
+
const value = usd ? totals.usd : totals.tokens;
|
|
127
|
+
return {
|
|
128
|
+
key: kind,
|
|
129
|
+
label: (
|
|
130
|
+
<>
|
|
131
|
+
<span className="spot-step-swatch" style={{ background: RECOVERY_KIND[kind].color }} />
|
|
132
|
+
{RECOVERY_KIND[kind].plural}
|
|
133
|
+
<span className="dim num">{formatInteger(totals.events)}</span>
|
|
134
|
+
</>
|
|
135
|
+
),
|
|
136
|
+
kind: "delta",
|
|
137
|
+
value: -value,
|
|
138
|
+
color: RECOVERY_KIND[kind].color,
|
|
139
|
+
note: share(value),
|
|
140
|
+
title: RECOVERY_KIND[kind].rule,
|
|
141
|
+
};
|
|
142
|
+
});
|
|
143
|
+
const steps: WaterfallStep[] = [
|
|
144
|
+
{
|
|
145
|
+
key: "gross",
|
|
146
|
+
label: usd ? "Gross saved" : "Gross vs plain",
|
|
147
|
+
kind: "total",
|
|
148
|
+
value: top,
|
|
149
|
+
color: "var(--spot-gross)",
|
|
150
|
+
title: usd
|
|
151
|
+
? "Avoided context priced at each later request's cache-read rate, plus one cache write"
|
|
152
|
+
: "Σ (plain − shortened) tokens over shortened results; a cut that showed more than plain counts negative",
|
|
153
|
+
},
|
|
154
|
+
...kindSteps,
|
|
155
|
+
];
|
|
156
|
+
if (usd) {
|
|
157
|
+
steps.push(
|
|
158
|
+
{
|
|
159
|
+
key: "turns",
|
|
160
|
+
label: (
|
|
161
|
+
<>
|
|
162
|
+
<span className="spot-step-swatch" style={{ background: "var(--spot-turns)" }} />
|
|
163
|
+
Extra turns
|
|
164
|
+
<span className="dim num">{formatInteger(clawback.turns)}</span>
|
|
165
|
+
</>
|
|
166
|
+
),
|
|
167
|
+
kind: "delta",
|
|
168
|
+
value: -clawback.turnsUsd,
|
|
169
|
+
color: "var(--spot-turns)",
|
|
170
|
+
note: share(clawback.turnsUsd),
|
|
171
|
+
title: "Provider cost of requests that issued nothing but recovery calls",
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
key: "spotlight",
|
|
175
|
+
label: (
|
|
176
|
+
<>
|
|
177
|
+
<span className="spot-step-swatch" style={{ background: "var(--spot-cost)" }} />
|
|
178
|
+
Spotlight cost
|
|
179
|
+
</>
|
|
180
|
+
),
|
|
181
|
+
kind: "delta",
|
|
182
|
+
value: -compounding.spotlightUsd,
|
|
183
|
+
color: "var(--spot-cost)",
|
|
184
|
+
note: share(compounding.spotlightUsd),
|
|
185
|
+
title: "Spotlight's own scoring bill",
|
|
186
|
+
},
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
// Without holdouts there is no baseline to add back: the net is unadjusted.
|
|
190
|
+
const unadjusted = usd ? compounding.netUsd : clawback.netTokens;
|
|
191
|
+
const adjustedNet = usd ? adjusted.netUsd.value : adjusted.netTokens.value;
|
|
192
|
+
if (adjustedNet !== null) {
|
|
193
|
+
const anyway = adjustedNet - unadjusted;
|
|
194
|
+
steps.push({
|
|
195
|
+
key: "anyway",
|
|
196
|
+
label: (
|
|
197
|
+
<>
|
|
198
|
+
<span className="spot-step-swatch" style={{ background: "var(--spot-anyway)" }} />
|
|
199
|
+
Back anyway
|
|
200
|
+
</>
|
|
201
|
+
),
|
|
202
|
+
kind: "delta",
|
|
203
|
+
value: anyway,
|
|
204
|
+
color: "var(--spot-anyway)",
|
|
205
|
+
note: share(anyway),
|
|
206
|
+
title: "Fetch-backs held-out results (shown whole) show the agent makes regardless, per tool: not the cut's cost",
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
const net = adjustedNet ?? unadjusted;
|
|
210
|
+
steps.push({
|
|
211
|
+
key: "net",
|
|
212
|
+
label: adjustedNet === null ? "Net (no holdout baseline yet)" : "Net",
|
|
213
|
+
kind: "total",
|
|
214
|
+
value: net,
|
|
215
|
+
color: "var(--spot-net-fill)",
|
|
216
|
+
note: share(net),
|
|
217
|
+
});
|
|
218
|
+
return steps;
|
|
219
|
+
}, [unit, gross, clawback, compounding, adjusted]);
|
|
220
|
+
|
|
221
|
+
return (
|
|
222
|
+
<Card
|
|
223
|
+
index={2}
|
|
224
|
+
title="Where the savings went"
|
|
225
|
+
description="Gross, minus each way hidden output came back"
|
|
226
|
+
actions={<UnitToggle value={unit} onChange={setUnit} />}
|
|
227
|
+
stale={stale}
|
|
228
|
+
footer={
|
|
229
|
+
unit === "usd"
|
|
230
|
+
? "Recoveries are priced as carried context; extra turns are requests that only recovered; Spotlight cost is its own scoring bill. What comes back anyway is the recovery $ in proportion to the tokens held-out results fetch regardless."
|
|
231
|
+
: "Recovered tokens are the full results of the recovery calls (estimated as chars ÷ 4 where no exact count exists). What comes back anyway is, per tool, the tokens a held-out result gets fetched back times the tool's cuts."
|
|
232
|
+
}
|
|
233
|
+
>
|
|
234
|
+
<Waterfall steps={steps} format={unit === "usd" ? formatCost : formatCompact} />
|
|
235
|
+
<div className="spot-by-tool">
|
|
236
|
+
<div className="section-label">Net by tool, tokens</div>
|
|
237
|
+
<ToolNetList byTool={gross.byTool} adjusted={adjusted.byTool} />
|
|
238
|
+
</div>
|
|
239
|
+
</Card>
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Tools the net-by-tool list names before folding the rest into `other`. */
|
|
244
|
+
const NET_LIST_TOOLS = 3;
|
|
245
|
+
|
|
246
|
+
/** One row of the net-by-tool list: a tool, or the folded rest summed. */
|
|
247
|
+
interface ToolNetRow {
|
|
248
|
+
tool: string;
|
|
249
|
+
totals: SpotlightToolRow;
|
|
250
|
+
excess: number | null;
|
|
251
|
+
title: string | undefined;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Each tool's gross as a bar, its fetch-back beyond the holdout baseline hatched off the end: what is left solid is the
|
|
256
|
+
* tool's net. The top tools by saving are listed; the rest fold into one `other` row.
|
|
257
|
+
*/
|
|
258
|
+
function ToolNetList({
|
|
259
|
+
byTool,
|
|
260
|
+
adjusted,
|
|
261
|
+
}: {
|
|
262
|
+
byTool: readonly SpotlightToolRow[];
|
|
263
|
+
adjusted: readonly SpotlightAdjustedTool[];
|
|
264
|
+
}) {
|
|
265
|
+
const adjustedByTool = new Map(adjusted.map(row => [row.tool, row]));
|
|
266
|
+
// Fewer rows than other cards: the list sits under the waterfall, beside the savings chart it should not outgrow.
|
|
267
|
+
const { listed, rest } = foldTools(byTool, NET_LIST_TOOLS);
|
|
268
|
+
const rows: ToolNetRow[] = listed.map(totals => {
|
|
269
|
+
const row = adjustedByTool.get(totals.tool);
|
|
270
|
+
const excessInterval = row === undefined ? "" : formatInterval(row.excessTokens, formatCompact);
|
|
271
|
+
const baseline = row?.baselinePerCut.value ?? null;
|
|
272
|
+
return {
|
|
273
|
+
tool: totals.tool,
|
|
274
|
+
totals,
|
|
275
|
+
excess: row?.excessTokens.value ?? null,
|
|
276
|
+
title:
|
|
277
|
+
row === undefined
|
|
278
|
+
? undefined
|
|
279
|
+
: [
|
|
280
|
+
`${formatInteger(row.cuts)} cuts fetched back ${formatCompact(row.recoveredTokens)} tokens`,
|
|
281
|
+
baseline === null
|
|
282
|
+
? "No held-out baseline yet"
|
|
283
|
+
: `${formatInteger(row.holdouts)} held-out results get ${formatCompact(baseline)} each fetched back anyway`,
|
|
284
|
+
excessInterval === "" ? "" : `Beyond the baseline: ${excessInterval} (90%)`,
|
|
285
|
+
]
|
|
286
|
+
.filter(Boolean)
|
|
287
|
+
.join("\n"),
|
|
288
|
+
};
|
|
289
|
+
});
|
|
290
|
+
if (rest.length > 0) {
|
|
291
|
+
const totals: SpotlightToolRow = {
|
|
292
|
+
tool: OTHER_TOOLS,
|
|
293
|
+
results: 0,
|
|
294
|
+
shaped: 0,
|
|
295
|
+
inTokens: 0,
|
|
296
|
+
plainTokens: 0,
|
|
297
|
+
outTokens: 0,
|
|
298
|
+
savedTokens: 0,
|
|
299
|
+
fullSavedTokens: 0,
|
|
300
|
+
};
|
|
301
|
+
let excess: number | null = null;
|
|
302
|
+
for (const row of rest) {
|
|
303
|
+
totals.results += row.results;
|
|
304
|
+
totals.shaped += row.shaped;
|
|
305
|
+
totals.savedTokens += row.savedTokens;
|
|
306
|
+
const own = adjustedByTool.get(row.tool)?.excessTokens.value ?? null;
|
|
307
|
+
if (own !== null) excess = (excess ?? 0) + own;
|
|
308
|
+
}
|
|
309
|
+
rows.push({
|
|
310
|
+
tool: OTHER_TOOLS,
|
|
311
|
+
totals,
|
|
312
|
+
excess,
|
|
313
|
+
title: `${formatInteger(rest.length)} more tools: ${rest.map(row => row.tool).join(", ")}`,
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
const top = Math.max(0, ...rows.map(row => row.totals.savedTokens));
|
|
317
|
+
return (
|
|
318
|
+
<ul className="spot-tool-net">
|
|
319
|
+
{rows.map(({ tool, totals, excess, title }) => {
|
|
320
|
+
const net = excess === null ? null : totals.savedTokens - excess;
|
|
321
|
+
const Icon = toolIcon(tool);
|
|
322
|
+
// A tool whose cuts showed more than plain nets below zero: no bar.
|
|
323
|
+
const width = top > 0 ? (Math.max(0, totals.savedTokens) / top) * 100 : 0;
|
|
324
|
+
const excessShare =
|
|
325
|
+
excess !== null && totals.savedTokens > 0 ? Math.min(1, excess / totals.savedTokens) : 0;
|
|
326
|
+
return (
|
|
327
|
+
<li key={tool} className="spot-tool-net-row" title={title}>
|
|
328
|
+
<div className="row spot-tool-net-head">
|
|
329
|
+
<Icon size={13} aria-hidden="true" />
|
|
330
|
+
<span className="cell-primary">{tool}</span>
|
|
331
|
+
<span className="dim num">
|
|
332
|
+
{formatInteger(totals.shaped)} of {formatInteger(totals.results)} shortened
|
|
333
|
+
</span>
|
|
334
|
+
<span
|
|
335
|
+
className="spot-tool-net-value num"
|
|
336
|
+
title={net === null ? "No holdout baseline yet" : undefined}
|
|
337
|
+
>
|
|
338
|
+
{formatSignedCompact(net ?? totals.savedTokens)}
|
|
339
|
+
</span>
|
|
340
|
+
</div>
|
|
341
|
+
<div className="spot-tool-net-track" aria-hidden="true">
|
|
342
|
+
<span className="spot-tool-net-bar" style={{ width: `${width}%` }}>
|
|
343
|
+
<span className="spot-tool-net-excess" style={{ width: `${excessShare * 100}%` }} />
|
|
344
|
+
</span>
|
|
345
|
+
</div>
|
|
346
|
+
<div className="micro spot-tool-net-detail">
|
|
347
|
+
{formatSignedCompact(totals.savedTokens)} gross
|
|
348
|
+
{excess !== null && <> − {formatCompact(excess)} beyond baseline</>}
|
|
349
|
+
</div>
|
|
350
|
+
</li>
|
|
351
|
+
);
|
|
352
|
+
})}
|
|
353
|
+
</ul>
|
|
354
|
+
);
|
|
355
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { useMemo } from "react";
|
|
2
|
+
import { type ChartSeries, Legend, type SlotMarker, TimeChart } from "../charts";
|
|
3
|
+
import { formatPercent } from "../data/formatters";
|
|
4
|
+
import { bucketAxis, formatBucket, rangeMeta } from "../data/range";
|
|
5
|
+
import { densify } from "../data/series";
|
|
6
|
+
import type { SpotlightDashboardStats, SpotlightSharePoint, TimeRange } from "../types";
|
|
7
|
+
import { Card } from "../ui";
|
|
8
|
+
import { bucketNoun, formatShare } from "./meta";
|
|
9
|
+
|
|
10
|
+
/** Trailing buckets in each rolling share. */
|
|
11
|
+
const ROLLING_WINDOW = 7;
|
|
12
|
+
|
|
13
|
+
type ShareGroup = SpotlightSharePoint["group"];
|
|
14
|
+
|
|
15
|
+
const GROUPS: readonly { group: ShareGroup; label: string; color: string }[] = [
|
|
16
|
+
{ group: "shaped", label: "Shaped sessions", color: "var(--spot-shaped)" },
|
|
17
|
+
{ group: "outside", label: "Outside the experiment", color: "var(--spot-outside)" },
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
export interface ShareTrendProps {
|
|
21
|
+
stats: SpotlightDashboardStats;
|
|
22
|
+
range: TimeRange;
|
|
23
|
+
stale: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Share of context that is tool output, per session group, raw per bucket and as a rolling ratio. */
|
|
27
|
+
export function ShareTrend({ stats, range, stale }: ShareTrendProps) {
|
|
28
|
+
const meta = rangeMeta(range);
|
|
29
|
+
const chart = useMemo(() => {
|
|
30
|
+
const buckets = bucketAxis(
|
|
31
|
+
range,
|
|
32
|
+
stats.share.map(p => p.timestamp),
|
|
33
|
+
meta.bucketMs,
|
|
34
|
+
);
|
|
35
|
+
const series: ChartSeries[] = [];
|
|
36
|
+
const legend: { key: string; label: string; color: string; value?: string }[] = [];
|
|
37
|
+
for (const { group, label, color } of GROUPS) {
|
|
38
|
+
const points = stats.share.filter(p => p.group === group);
|
|
39
|
+
const tool = densify(points, buckets, p => p.toolTokens);
|
|
40
|
+
const ctx = densify(points, buckets, p => p.contextTokens);
|
|
41
|
+
let toolSum = 0;
|
|
42
|
+
let ctxSum = 0;
|
|
43
|
+
const rolling = buckets.map((_, i) => {
|
|
44
|
+
let t = 0;
|
|
45
|
+
let c = 0;
|
|
46
|
+
for (let j = Math.max(0, i - ROLLING_WINDOW + 1); j <= i; j++) {
|
|
47
|
+
t += tool[j];
|
|
48
|
+
c += ctx[j];
|
|
49
|
+
}
|
|
50
|
+
toolSum += tool[i];
|
|
51
|
+
ctxSum += ctx[i];
|
|
52
|
+
return c > 0 ? t / c : null;
|
|
53
|
+
});
|
|
54
|
+
if (ctxSum === 0) continue;
|
|
55
|
+
series.push(
|
|
56
|
+
{
|
|
57
|
+
key: `${group}:raw`,
|
|
58
|
+
label: `${label} · bucket`,
|
|
59
|
+
color,
|
|
60
|
+
kind: "dots",
|
|
61
|
+
values: buckets.map((_, i) => (ctx[i] > 0 ? tool[i] / ctx[i] : null)),
|
|
62
|
+
},
|
|
63
|
+
{ key: `${group}:avg`, label: `${label} · rolling`, color, kind: "line", values: rolling },
|
|
64
|
+
);
|
|
65
|
+
legend.push({ key: group, label, color, value: formatShare(toolSum / ctxSum) });
|
|
66
|
+
}
|
|
67
|
+
const markers: SlotMarker[] = [];
|
|
68
|
+
if (stats.experimentStart !== null) {
|
|
69
|
+
const start = Math.floor(stats.experimentStart / meta.bucketMs) * meta.bucketMs;
|
|
70
|
+
const index = buckets.indexOf(start);
|
|
71
|
+
if (index >= 0) markers.push({ index, label: "experiment starts" });
|
|
72
|
+
}
|
|
73
|
+
return { buckets, series, legend, markers };
|
|
74
|
+
}, [stats.share, stats.experimentStart, range, meta.bucketMs]);
|
|
75
|
+
|
|
76
|
+
return (
|
|
77
|
+
<Card
|
|
78
|
+
index={4}
|
|
79
|
+
title="Tool-output share of context"
|
|
80
|
+
description={
|
|
81
|
+
stats.experimentStart !== null && chart.markers.length === 0
|
|
82
|
+
? `Since ${formatBucket(stats.experimentStart, 86_400_000)}, shaped sessions should carry less tool output`
|
|
83
|
+
: "Shaped sessions should carry less tool output than the rest of your history"
|
|
84
|
+
}
|
|
85
|
+
actions={<Legend items={chart.legend} />}
|
|
86
|
+
stale={stale}
|
|
87
|
+
footer={`Tool tokens are result chars ÷ 4; context adds assistant output, your prompts (chars ÷ 4) and each session's first prompt. Dots are single buckets (one ${bucketNoun(meta.bucketMs)}); lines are the trailing ${ROLLING_WINDOW}-bucket ratio of sums.`}
|
|
88
|
+
>
|
|
89
|
+
<TimeChart
|
|
90
|
+
buckets={chart.buckets}
|
|
91
|
+
bucketMs={meta.bucketMs}
|
|
92
|
+
series={chart.series}
|
|
93
|
+
markers={chart.markers}
|
|
94
|
+
stacked={false}
|
|
95
|
+
height={260}
|
|
96
|
+
format={v => formatPercent(v, 0)}
|
|
97
|
+
formatTooltip={v => formatPercent(v)}
|
|
98
|
+
showTotal={false}
|
|
99
|
+
emptyLabel="No tool calls in this range"
|
|
100
|
+
/>
|
|
101
|
+
</Card>
|
|
102
|
+
);
|
|
103
|
+
}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import { Fragment, type ReactNode, useState } from "react";
|
|
2
|
+
import { formatCompact, formatInteger } from "../data/formatters";
|
|
3
|
+
import type { SpotlightDashboardStats, SpotlightSizeBin, SpotlightStepCell, SpotlightTool } from "../types";
|
|
4
|
+
import { Card } from "../ui";
|
|
5
|
+
import { MoreTools } from "./chips";
|
|
6
|
+
import { foldTools, formatInterval, formatShare, SIZE_BINS, toolIcon, toolsBySaving } from "./meta";
|
|
7
|
+
|
|
8
|
+
/** Under this many cuts a cell's share swings on a handful of fetch-backs: it renders dimmed. */
|
|
9
|
+
const THIN_CUTS = 10;
|
|
10
|
+
|
|
11
|
+
/** Tint of a cell kept entirely (or lost entirely, or worse); the rest scale with the distance from half kept. */
|
|
12
|
+
const MAX_TINT = 30;
|
|
13
|
+
const MIN_TINT = 4;
|
|
14
|
+
|
|
15
|
+
/** Σ over a step's sizes, for its "all sizes" column. */
|
|
16
|
+
interface StepTotal {
|
|
17
|
+
cuts: number;
|
|
18
|
+
savedTokens: number;
|
|
19
|
+
excessTokens: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface StepRow {
|
|
23
|
+
steps: string;
|
|
24
|
+
bySize: Map<SpotlightSizeBin, SpotlightStepCell>;
|
|
25
|
+
total: StepTotal;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Steps of one tool, the biggest saving first and `other` last. */
|
|
29
|
+
function stepRows(cells: readonly SpotlightStepCell[], tool: SpotlightTool): StepRow[] {
|
|
30
|
+
const rows = new Map<string, StepRow>();
|
|
31
|
+
for (const cell of cells) {
|
|
32
|
+
if (cell.tool !== tool) continue;
|
|
33
|
+
let row = rows.get(cell.steps);
|
|
34
|
+
if (row === undefined) {
|
|
35
|
+
row = { steps: cell.steps, bySize: new Map(), total: { cuts: 0, savedTokens: 0, excessTokens: 0 } };
|
|
36
|
+
rows.set(cell.steps, row);
|
|
37
|
+
}
|
|
38
|
+
row.bySize.set(cell.size, cell);
|
|
39
|
+
row.total.cuts += cell.cuts;
|
|
40
|
+
row.total.savedTokens += cell.savedTokens;
|
|
41
|
+
row.total.excessTokens += cell.excessTokens;
|
|
42
|
+
}
|
|
43
|
+
return [...rows.values()].sort(
|
|
44
|
+
(a, b) => Number(a.steps === "other") - Number(b.steps === "other") || b.total.savedTokens - a.total.savedTokens,
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Green above half the saving kept, red below; strength grows toward all or nothing. */
|
|
49
|
+
function KeptCell({
|
|
50
|
+
kept,
|
|
51
|
+
cuts,
|
|
52
|
+
title,
|
|
53
|
+
children,
|
|
54
|
+
}: {
|
|
55
|
+
kept: number | null;
|
|
56
|
+
cuts: number;
|
|
57
|
+
title: string;
|
|
58
|
+
children: ReactNode;
|
|
59
|
+
}) {
|
|
60
|
+
if (kept === null) {
|
|
61
|
+
return (
|
|
62
|
+
<div className="spot-econ-cell" data-empty="true" title={title}>
|
|
63
|
+
<span className="dim">No saving</span>
|
|
64
|
+
</div>
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
// keptShare goes negative when the excess fetch-back outweighs the saving: as red as a total loss.
|
|
68
|
+
const tint = MIN_TINT + Math.min(1, Math.abs(kept - 0.5) * 2) * (MAX_TINT - MIN_TINT);
|
|
69
|
+
return (
|
|
70
|
+
<div
|
|
71
|
+
className="spot-econ-cell"
|
|
72
|
+
data-tone={kept >= 0.5 ? "ok" : "bad"}
|
|
73
|
+
data-thin={cuts < THIN_CUTS}
|
|
74
|
+
style={{ "--tint": `${tint}%` }}
|
|
75
|
+
title={title}
|
|
76
|
+
>
|
|
77
|
+
<span className="spot-econ-net num">
|
|
78
|
+
{formatShare(kept)}
|
|
79
|
+
<span className="spot-steps-unit"> kept</span>
|
|
80
|
+
</span>
|
|
81
|
+
<dl className="spot-econ-facts">{children}</dl>
|
|
82
|
+
</div>
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Tool × shaping steps × cut size: how much of each kind of cut's saving stays saved after fetch-backs. */
|
|
87
|
+
export function Steps({ stats, stale }: { stats: SpotlightDashboardStats; stale: boolean }) {
|
|
88
|
+
const [expanded, setExpanded] = useState(false);
|
|
89
|
+
const all = toolsBySaving(stats).flatMap(tool => {
|
|
90
|
+
const rows = stepRows(stats.steps, tool);
|
|
91
|
+
return rows.length === 0 ? [] : [{ tool, rows }];
|
|
92
|
+
});
|
|
93
|
+
const { listed, rest } = foldTools(all);
|
|
94
|
+
const tools = expanded ? all : listed;
|
|
95
|
+
|
|
96
|
+
return (
|
|
97
|
+
<Card
|
|
98
|
+
index={5}
|
|
99
|
+
title="Which cuts pay"
|
|
100
|
+
description="Share of each kind of cut's saving that stays saved after fetch-backs beyond the holdout baseline, by size"
|
|
101
|
+
actions={
|
|
102
|
+
<span className="spot-econ-scale" aria-hidden="true">
|
|
103
|
+
<span className="num">0%</span>
|
|
104
|
+
<span className="spot-econ-ramp" />
|
|
105
|
+
<span className="num">100% kept</span>
|
|
106
|
+
</span>
|
|
107
|
+
}
|
|
108
|
+
stale={stale}
|
|
109
|
+
footer={`Scored and score-free cuts. Kept = (saved − fetch-back beyond the tool's holdout baseline) ÷ saved; fetched back is the share of cuts the agent came back for within 12 calls. Sizes are the tokens a cut saved against plain; step combinations under 10 cuts fold into "other". Cells under ${THIN_CUTS} cuts are dimmed.`}
|
|
110
|
+
>
|
|
111
|
+
{tools.length === 0 ? (
|
|
112
|
+
<div className="empty">No shortened results in this range.</div>
|
|
113
|
+
) : (
|
|
114
|
+
<>
|
|
115
|
+
<div className="spot-steps-scroll">
|
|
116
|
+
<table className="spot-econ spot-steps">
|
|
117
|
+
<thead>
|
|
118
|
+
<tr>
|
|
119
|
+
<th scope="col">Steps</th>
|
|
120
|
+
{SIZE_BINS.map(size => (
|
|
121
|
+
<th key={size} scope="col" className="num">
|
|
122
|
+
{size} <span className="dim">saved</span>
|
|
123
|
+
</th>
|
|
124
|
+
))}
|
|
125
|
+
<th scope="col">All sizes</th>
|
|
126
|
+
</tr>
|
|
127
|
+
</thead>
|
|
128
|
+
<tbody>
|
|
129
|
+
{tools.map(({ tool, rows }) => {
|
|
130
|
+
const Icon = toolIcon(tool);
|
|
131
|
+
return (
|
|
132
|
+
<Fragment key={tool}>
|
|
133
|
+
<tr className="spot-steps-tool">
|
|
134
|
+
<th scope="rowgroup" colSpan={SIZE_BINS.length + 2}>
|
|
135
|
+
<span className="row spot-econ-tool">
|
|
136
|
+
<Icon size={13} aria-hidden="true" />
|
|
137
|
+
{tool}
|
|
138
|
+
</span>
|
|
139
|
+
</th>
|
|
140
|
+
</tr>
|
|
141
|
+
{rows.map(row => {
|
|
142
|
+
const { total } = row;
|
|
143
|
+
const totalKept =
|
|
144
|
+
total.savedTokens > 0
|
|
145
|
+
? (total.savedTokens - total.excessTokens) / total.savedTokens
|
|
146
|
+
: null;
|
|
147
|
+
return (
|
|
148
|
+
<tr key={row.steps}>
|
|
149
|
+
<th scope="row" className="spot-steps-name" title={row.steps}>
|
|
150
|
+
{row.steps}
|
|
151
|
+
</th>
|
|
152
|
+
{SIZE_BINS.map(size => {
|
|
153
|
+
const cell = row.bySize.get(size);
|
|
154
|
+
if (cell === undefined || cell.cuts === 0) {
|
|
155
|
+
return (
|
|
156
|
+
<td key={size}>
|
|
157
|
+
<div className="spot-econ-cell" data-empty="true">
|
|
158
|
+
<span className="dim">–</span>
|
|
159
|
+
</div>
|
|
160
|
+
</td>
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
const interval = formatInterval(cell.fetchBackRate, formatShare);
|
|
164
|
+
return (
|
|
165
|
+
<td key={size}>
|
|
166
|
+
<KeptCell
|
|
167
|
+
kept={cell.keptShare}
|
|
168
|
+
cuts={cell.cuts}
|
|
169
|
+
title={[
|
|
170
|
+
`${tool} · ${row.steps} · ${size} saved: ${formatInteger(cell.cuts)} cuts`,
|
|
171
|
+
`Saved ${formatCompact(cell.savedTokens)}, fetched back ${formatCompact(cell.recoveredTokens)}, ${formatCompact(cell.excessTokens)} beyond the baseline`,
|
|
172
|
+
`Fetched back after ${formatShare(cell.fetchBackRate.value)} of cuts${interval === "" ? "" : ` (${interval})`}`,
|
|
173
|
+
].join("\n")}
|
|
174
|
+
>
|
|
175
|
+
<div>
|
|
176
|
+
<dt>cuts</dt>
|
|
177
|
+
<dd className="num">{formatInteger(cell.cuts)}</dd>
|
|
178
|
+
</div>
|
|
179
|
+
<div>
|
|
180
|
+
<dt>fetched back</dt>
|
|
181
|
+
<dd className="num">
|
|
182
|
+
{formatShare(cell.fetchBackRate.value)}
|
|
183
|
+
</dd>
|
|
184
|
+
</div>
|
|
185
|
+
</KeptCell>
|
|
186
|
+
</td>
|
|
187
|
+
);
|
|
188
|
+
})}
|
|
189
|
+
<td>
|
|
190
|
+
<KeptCell
|
|
191
|
+
kept={totalKept}
|
|
192
|
+
cuts={total.cuts}
|
|
193
|
+
title={`${tool} · ${row.steps}: ${formatInteger(total.cuts)} cuts saved ${formatCompact(total.savedTokens)}, ${formatCompact(total.excessTokens)} fetched back beyond the baseline`}
|
|
194
|
+
>
|
|
195
|
+
<div>
|
|
196
|
+
<dt>cuts</dt>
|
|
197
|
+
<dd className="num">{formatInteger(total.cuts)}</dd>
|
|
198
|
+
</div>
|
|
199
|
+
<div>
|
|
200
|
+
<dt>saved</dt>
|
|
201
|
+
<dd className="num">{formatCompact(total.savedTokens)}</dd>
|
|
202
|
+
</div>
|
|
203
|
+
</KeptCell>
|
|
204
|
+
</td>
|
|
205
|
+
</tr>
|
|
206
|
+
);
|
|
207
|
+
})}
|
|
208
|
+
</Fragment>
|
|
209
|
+
);
|
|
210
|
+
})}
|
|
211
|
+
</tbody>
|
|
212
|
+
</table>
|
|
213
|
+
</div>
|
|
214
|
+
{rest.length > 0 && (
|
|
215
|
+
<MoreTools
|
|
216
|
+
hidden={rest.map(item => item.tool)}
|
|
217
|
+
expanded={expanded}
|
|
218
|
+
onToggle={() => setExpanded(value => !value)}
|
|
219
|
+
/>
|
|
220
|
+
)}
|
|
221
|
+
</>
|
|
222
|
+
)}
|
|
223
|
+
</Card>
|
|
224
|
+
);
|
|
225
|
+
}
|