@oh-my-pi/omp-stats 18.16.0 → 18.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/client/index.css +1 -1
- package/dist/client/index.js +102 -99
- package/dist/types/client/api.d.ts +4 -1
- package/dist/types/client/app/nav.d.ts +1 -1
- package/dist/types/client/charts/Chart.d.ts +16 -8
- package/dist/types/client/charts/DotWhisker.d.ts +50 -0
- package/dist/types/client/charts/Sparkline.d.ts +1 -1
- package/dist/types/client/charts/Waterfall.d.ts +26 -0
- package/dist/types/client/charts/index.d.ts +2 -0
- package/dist/types/client/charts/types.d.ts +8 -2
- package/dist/types/client/routes/SpotlightRoute.d.ts +7 -0
- package/dist/types/client/routes/index.d.ts +1 -0
- package/dist/types/client/spotlight/Compounding.d.ts +8 -0
- package/dist/types/client/spotlight/Coverage.d.ts +6 -0
- package/dist/types/client/spotlight/Economics.d.ts +10 -0
- package/dist/types/client/spotlight/FetchBack.d.ts +9 -0
- package/dist/types/client/spotlight/Hero.d.ts +12 -0
- package/dist/types/client/spotlight/Overhead.d.ts +6 -0
- package/dist/types/client/spotlight/Ranking.d.ts +6 -0
- package/dist/types/client/spotlight/ResultsFeed.d.ts +6 -0
- package/dist/types/client/spotlight/Savings.d.ts +10 -0
- package/dist/types/client/spotlight/ShareTrend.d.ts +8 -0
- package/dist/types/client/spotlight/Steps.d.ts +6 -0
- package/dist/types/client/spotlight/TimeCost.d.ts +6 -0
- package/dist/types/client/spotlight/chips.d.ts +16 -0
- package/dist/types/client/spotlight/meta.d.ts +64 -0
- package/dist/types/db.d.ts +26 -0
- package/dist/types/index.d.ts +2 -1
- package/dist/types/parser.d.ts +4 -0
- package/dist/types/rollup.d.ts +22 -1
- package/dist/types/shared-types.d.ts +419 -0
- package/dist/types/spotlight-aggregator.d.ts +189 -0
- package/dist/types/spotlight-bootstrap.d.ts +32 -0
- package/dist/types/spotlight-ingest.d.ts +258 -0
- package/dist/types/spotlight-measures.d.ts +51 -0
- package/package.json +5 -5
- package/src/aggregator.ts +3 -1
- package/src/client/App.tsx +3 -0
- package/src/client/api.ts +21 -0
- package/src/client/app/nav.ts +4 -1
- package/src/client/charts/Chart.tsx +105 -22
- package/src/client/charts/DotWhisker.tsx +149 -0
- package/src/client/charts/Sparkline.tsx +3 -2
- package/src/client/charts/Waterfall.tsx +86 -0
- package/src/client/charts/index.ts +2 -0
- package/src/client/charts/types.ts +9 -2
- package/src/client/react-css.d.ts +1 -1
- package/src/client/routes/SpotlightRoute.tsx +138 -0
- package/src/client/routes/index.ts +1 -0
- package/src/client/spotlight/Compounding.tsx +222 -0
- package/src/client/spotlight/Coverage.tsx +96 -0
- package/src/client/spotlight/Economics.tsx +167 -0
- package/src/client/spotlight/FetchBack.tsx +222 -0
- package/src/client/spotlight/Hero.tsx +177 -0
- package/src/client/spotlight/Overhead.tsx +135 -0
- package/src/client/spotlight/Ranking.tsx +248 -0
- package/src/client/spotlight/ResultsFeed.tsx +354 -0
- package/src/client/spotlight/Savings.tsx +355 -0
- package/src/client/spotlight/ShareTrend.tsx +103 -0
- package/src/client/spotlight/Steps.tsx +225 -0
- package/src/client/spotlight/TimeCost.tsx +64 -0
- package/src/client/spotlight/chips.tsx +63 -0
- package/src/client/spotlight/meta.ts +206 -0
- package/src/client/spotlight/spotlight.css +1387 -0
- package/src/client/styles.css +289 -1
- package/src/db.ts +480 -2
- package/src/index.ts +10 -1
- package/src/live.ts +21 -9
- package/src/parser.ts +44 -8
- package/src/rollup.ts +203 -10
- package/src/server.ts +42 -0
- package/src/shared-types.ts +453 -0
- package/src/spotlight-aggregator.ts +1738 -0
- package/src/spotlight-bootstrap.ts +110 -0
- package/src/spotlight-ingest.ts +981 -0
- package/src/spotlight-measures.ts +667 -0
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spotlight experiment facts the parser extracts from a session journal, for the Spotlight dashboard
|
|
3
|
+
* (`spotlight-aggregator.ts`).
|
|
4
|
+
*
|
|
5
|
+
* A session is an experiment session from its first Spotlight marker on (see {@link experimentMarker}); from then the
|
|
6
|
+
* parser records every evaluated result, every tool call (target, order, result size, the artifacts its result names)
|
|
7
|
+
* and every main-loop request, so recoveries of hidden output can be detected later. Evaluated results come from two
|
|
8
|
+
* entries that may land in different tail parses: the `spotlight:shape` journal entry (Spotlight's bill, timing) and
|
|
9
|
+
* the result message's `details.spotlight` (what the model saw, the hidden lines). Both upsert the same row.
|
|
10
|
+
*
|
|
11
|
+
* Each shaped read (applied, held out or score-free) also collects the original lines the agent needed later (see
|
|
12
|
+
* {@link SpotlightCollector}): its open reads ride in the parser state, so a later tail parse keeps collecting, and the
|
|
13
|
+
* lines land in `spotlight_needs`.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Whether a custom entry marks its session as a Spotlight experiment session: pi's `spotlight:session` entry, which
|
|
17
|
+
* every session with the experiment on writes. Journals from before the experiment became a toggle carry
|
|
18
|
+
* `spotlight:assignment` instead, naming the arm the session drew; only the arm that shipped shaped with the shipped
|
|
19
|
+
* algorithm, so sessions of any other arm (`native`, `plain`, …) are not experiment sessions.
|
|
20
|
+
*/
|
|
21
|
+
export declare function experimentMarker(customType: unknown, data: Record<string, unknown>): boolean;
|
|
22
|
+
/** Longest stored call target. */
|
|
23
|
+
export declare const SPOTLIGHT_TARGET_CHARS = 500;
|
|
24
|
+
/** A shaped read still collecting the lines the agent needs later (see {@link SpotlightCollector}). */
|
|
25
|
+
export interface SpotlightOpenRead {
|
|
26
|
+
toolCallId: string;
|
|
27
|
+
/** The read file, resolved against the session's working directory. */
|
|
28
|
+
file: string;
|
|
29
|
+
/** First file line of the original and its line count. */
|
|
30
|
+
start: number;
|
|
31
|
+
lines: number;
|
|
32
|
+
/** `original.url`: the session artifact holding the original text. */
|
|
33
|
+
url: string | null;
|
|
34
|
+
}
|
|
35
|
+
/** Spotlight state a tail parse needs from earlier bytes of the same transcript. */
|
|
36
|
+
export interface SpotlightParserState {
|
|
37
|
+
/** A Spotlight marker was seen: results, calls and requests are recorded from here on. */
|
|
38
|
+
experiment: boolean;
|
|
39
|
+
/** Tool calls recorded so far (the next call's order). */
|
|
40
|
+
seq: number;
|
|
41
|
+
/** The session header's working directory, for resolving relative read paths. */
|
|
42
|
+
cwd: string | null;
|
|
43
|
+
/** Latest session title. */
|
|
44
|
+
title: string | null;
|
|
45
|
+
/** Shaped reads still collecting needed lines; absent in cursors persisted before them. */
|
|
46
|
+
open?: SpotlightOpenRead[];
|
|
47
|
+
}
|
|
48
|
+
export declare function emptySpotlightState(): SpotlightParserState;
|
|
49
|
+
/** `spotlight_sessions` row: the session's first Spotlight marker. */
|
|
50
|
+
export interface SpotlightSessionRow {
|
|
51
|
+
sessionFile: string;
|
|
52
|
+
folder: string;
|
|
53
|
+
markedAt: number;
|
|
54
|
+
cwd: string | null;
|
|
55
|
+
title: string | null;
|
|
56
|
+
}
|
|
57
|
+
/** `spotlight_results` row (upserted): one evaluated result, from its shape entry, its result details, or both. */
|
|
58
|
+
export interface SpotlightShapeRow {
|
|
59
|
+
sessionFile: string;
|
|
60
|
+
toolCallId: string;
|
|
61
|
+
timestamp: number;
|
|
62
|
+
tool: string;
|
|
63
|
+
inChars: number;
|
|
64
|
+
outChars: number;
|
|
65
|
+
inTokens: number;
|
|
66
|
+
outTokens: number;
|
|
67
|
+
/** What pi would have shown unshaped (`plain`) for the call; null on traces written before pi recorded it. */
|
|
68
|
+
plainChars: number | null;
|
|
69
|
+
plainTokens: number | null;
|
|
70
|
+
/**
|
|
71
|
+
* A cut wider than `plain` that pi did not apply (steps end with `plain-cap`): the would-be cut's tokens; the model
|
|
72
|
+
* got `plain`'s text (`outTokens`). Null otherwise.
|
|
73
|
+
*/
|
|
74
|
+
cappedTokens: number | null;
|
|
75
|
+
/** The score-free coin skipped Spotlight: the no-score fallback rules shaped the result. */
|
|
76
|
+
scoreFree: boolean;
|
|
77
|
+
steps: string[];
|
|
78
|
+
/** The result message carried `details.spotlight`. */
|
|
79
|
+
hasDetails: boolean;
|
|
80
|
+
/**
|
|
81
|
+
* Held out at random: Spotlight would have shortened it, but the model saw it unshaped; the token counts and
|
|
82
|
+
* `original` describe the would-be cut.
|
|
83
|
+
*/
|
|
84
|
+
holdout: boolean;
|
|
85
|
+
/**
|
|
86
|
+
* A read pi's former edit gate (removed) kept whole, in transcripts journaled while it ran: the model saw
|
|
87
|
+
* `plain`'s page; like a held-out result, the token counts and `original` describe the would-be cut.
|
|
88
|
+
*/
|
|
89
|
+
gated: boolean;
|
|
90
|
+
/** `details.spotlight.original` (present only when shaping changed the text). */
|
|
91
|
+
original: SpotlightOriginalFacts | null;
|
|
92
|
+
/** Artifact ids the result names (its text's `artifact://N` plus `original.url`); null before the result lands. */
|
|
93
|
+
artifacts: number[] | null;
|
|
94
|
+
/** Spotlight's own usage, from the shape entry; null before it lands. */
|
|
95
|
+
spotlight: {
|
|
96
|
+
calls: number;
|
|
97
|
+
failures: number;
|
|
98
|
+
ms: number;
|
|
99
|
+
costUsd: number;
|
|
100
|
+
} | null;
|
|
101
|
+
/** Wall time shaping took, from the shape entry. */
|
|
102
|
+
shapeMs: number | null;
|
|
103
|
+
}
|
|
104
|
+
export interface SpotlightOriginalFacts {
|
|
105
|
+
url: string | null;
|
|
106
|
+
path: string | null;
|
|
107
|
+
/** First original line of the window (1-based file line for reads, else 1). */
|
|
108
|
+
start: number;
|
|
109
|
+
lines: number;
|
|
110
|
+
/** Hidden absolute line ranges, inclusive: the window minus the kept ranges. */
|
|
111
|
+
hidden: [number, number][];
|
|
112
|
+
hiddenLines: number;
|
|
113
|
+
/** Spotlight's per-line heat (`0`–`9`, `-` unscored) over the first lines; null when the trace has none. */
|
|
114
|
+
heat: string | null;
|
|
115
|
+
/** Kept runs, 1-based positions in the original (of a held-out result: what the cut would have kept). */
|
|
116
|
+
kept: [number, number][];
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* `spotlight_needs` row (merged on upsert): original lines (1-based positions in the original) a shaped read's later
|
|
120
|
+
* calls needed. Ranges of later parses are added to the stored ones; `open` is the latest.
|
|
121
|
+
*/
|
|
122
|
+
export interface SpotlightNeedRow {
|
|
123
|
+
sessionFile: string;
|
|
124
|
+
toolCallId: string;
|
|
125
|
+
file: string;
|
|
126
|
+
/** Still collecting: no edit, whole-file read or later shaped read of the file yet. */
|
|
127
|
+
open: boolean;
|
|
128
|
+
/** Lines later `read` selections covered. */
|
|
129
|
+
reread: [number, number][];
|
|
130
|
+
/** Lines later `edit` Find blocks located in the original. */
|
|
131
|
+
edit: [number, number][];
|
|
132
|
+
}
|
|
133
|
+
/** `spotlight_calls` row: one tool call of an experiment session. */
|
|
134
|
+
export interface SpotlightCallRow {
|
|
135
|
+
sessionFile: string;
|
|
136
|
+
toolCallId: string;
|
|
137
|
+
entryId: string;
|
|
138
|
+
timestamp: number;
|
|
139
|
+
seq: number;
|
|
140
|
+
toolName: string;
|
|
141
|
+
callsInTurn: number;
|
|
142
|
+
target: string;
|
|
143
|
+
/** Files an `edit`/`write`/`ast_edit` call changes, as written (relative paths unresolved); null for other calls. */
|
|
144
|
+
edits: string[] | null;
|
|
145
|
+
/** {@link argumentTokens} of the call, for the generic rerun rule; null for `read` and `bash` (own rules). */
|
|
146
|
+
argTokens: string[] | null;
|
|
147
|
+
}
|
|
148
|
+
/** Result facts of a recorded call, applied as an update (the result can land in a later tail parse). */
|
|
149
|
+
export interface SpotlightCallResultLink {
|
|
150
|
+
sessionFile: string;
|
|
151
|
+
toolCallId: string;
|
|
152
|
+
resultChars: number;
|
|
153
|
+
artifacts: number[];
|
|
154
|
+
}
|
|
155
|
+
/** `spotlight_requests` row: a main-loop assistant message of an experiment session (side requests excluded). */
|
|
156
|
+
export interface SpotlightRequestRow {
|
|
157
|
+
sessionFile: string;
|
|
158
|
+
entryId: string;
|
|
159
|
+
timestamp: number;
|
|
160
|
+
}
|
|
161
|
+
/** Everything one parse extracted for the Spotlight tables. */
|
|
162
|
+
export interface SpotlightParseRows {
|
|
163
|
+
sessions: SpotlightSessionRow[];
|
|
164
|
+
/** Title changes of experiment sessions seen after their marker. */
|
|
165
|
+
titles: {
|
|
166
|
+
sessionFile: string;
|
|
167
|
+
title: string;
|
|
168
|
+
}[];
|
|
169
|
+
shapes: SpotlightShapeRow[];
|
|
170
|
+
calls: SpotlightCallRow[];
|
|
171
|
+
callResults: SpotlightCallResultLink[];
|
|
172
|
+
requests: SpotlightRequestRow[];
|
|
173
|
+
needs: SpotlightNeedRow[];
|
|
174
|
+
}
|
|
175
|
+
export declare function emptySpotlightRows(): SpotlightParseRows;
|
|
176
|
+
export declare function spotlightRowCount(rows: SpotlightParseRows): number;
|
|
177
|
+
/** Distinct `artifact://N` ids a text names, ascending. */
|
|
178
|
+
export declare function artifactIds(text: string): number[];
|
|
179
|
+
/**
|
|
180
|
+
* What a call asked for, as recovery detection compares it: a read's path with its selector, a bash command, a
|
|
181
|
+
* search pattern with its path, eval code. Clipped to {@link SPOTLIGHT_TARGET_CHARS}.
|
|
182
|
+
*/
|
|
183
|
+
export declare function callTarget(toolName: string, args: unknown): string;
|
|
184
|
+
/**
|
|
185
|
+
* A call's arguments as a token set for the generic rerun rule (two calls of the same tool rerun when their sets'
|
|
186
|
+
* Jaccard similarity is high): every JSON leaf as `key=value`, the key being the nearest object key; strings split
|
|
187
|
+
* on whitespace except under path keys (`path`, `paths`, `file`, `cwd`, `url`, …), which stay whole; nulls dropped;
|
|
188
|
+
* the top-level intent note `i` left out. Distinct tokens in first-seen order, at most
|
|
189
|
+
* {@link ARGUMENT_TOKEN_LIMIT}, each clipped to {@link ARGUMENT_TOKEN_CHARS} chars.
|
|
190
|
+
*/
|
|
191
|
+
export declare function argumentTokens(args: unknown): string[];
|
|
192
|
+
/**
|
|
193
|
+
* Files a call changes: an `edit`'s patch headers (or its `path`), a `write`'s `path`, an `ast_edit`'s `paths`
|
|
194
|
+
* (globs kept as written), also when `ast_edit` runs through `write xd://ast_edit`. Null when the call is no file
|
|
195
|
+
* edit (other tools, or a write to an internal URL such as `proc://…/kill`).
|
|
196
|
+
*/
|
|
197
|
+
export declare function editedPaths(toolName: string, args: unknown): string[] | null;
|
|
198
|
+
/** A read target split into its path and selected lines. */
|
|
199
|
+
export interface ReadSelection {
|
|
200
|
+
path: string;
|
|
201
|
+
/** Selected inclusive line ranges; empty with `tail` null means the whole file. */
|
|
202
|
+
ranges: [number, number][];
|
|
203
|
+
/** `:-N`: the last N lines. */
|
|
204
|
+
tail: number | null;
|
|
205
|
+
}
|
|
206
|
+
/** Split a read path into the path and its selector (`file.ts:10-40`, `:50+20`, `:-30`, `:5-9,20`, `:raw`). */
|
|
207
|
+
export declare function parseReadSelection(target: string): ReadSelection;
|
|
208
|
+
/** A read selection's lines; null for the whole file. */
|
|
209
|
+
export declare function selectedLines(selection: ReadSelection): [number, number][] | null;
|
|
210
|
+
/** A path as written, resolved against the session's working directory (URLs and absolute paths kept). */
|
|
211
|
+
export declare function resolveReadPath(target: string, cwd: string | null): string;
|
|
212
|
+
/** One `*** Find` block of an `edit` input and the file its section edits (null: the call's `path`). */
|
|
213
|
+
export interface EditFindBlock {
|
|
214
|
+
file: string | null;
|
|
215
|
+
/** `*** Edit File: PATH all`: every occurrence is replaced. */
|
|
216
|
+
all: boolean;
|
|
217
|
+
lines: string[];
|
|
218
|
+
}
|
|
219
|
+
/** The `*** Find` blocks of an `edit` call's input, with the file each one's section edits. */
|
|
220
|
+
export declare function editFindBlocks(input: string): EditFindBlock[];
|
|
221
|
+
/**
|
|
222
|
+
* Where a Find block's text lies in `original` (lines): every match as 1-based inclusive `[first, last]` positions.
|
|
223
|
+
* A Find line ending in `…` (or a lone `…` line) may skip original lines before the next Find line matches; blank
|
|
224
|
+
* lines around the block are ignored.
|
|
225
|
+
*/
|
|
226
|
+
export declare function locateFind(original: readonly string[], find: readonly string[]): [number, number][];
|
|
227
|
+
/** Union of inclusive ranges, sorted and merged (touching ranges join). */
|
|
228
|
+
export declare function mergeRanges(ranges: readonly (readonly [number, number])[]): [number, number][];
|
|
229
|
+
/** Hidden absolute ranges of a window `[start, start + lines - 1]` whose kept ranges are 1-based within it. */
|
|
230
|
+
export declare function hiddenRanges(start: number, lines: number, kept: readonly [number, number][]): [number, number][];
|
|
231
|
+
/** The row a `spotlight:shape` entry's data describes, or null when it names no call. */
|
|
232
|
+
export declare function shapeFromEntry(sessionFile: string, data: Record<string, unknown>, timestamp: number): SpotlightShapeRow | null;
|
|
233
|
+
/** The row a result message's `details.spotlight` describes, or null when it carries none. */
|
|
234
|
+
export declare function shapeFromDetails(sessionFile: string, toolCallId: string, details: unknown, text: string, timestamp: number): SpotlightShapeRow | null;
|
|
235
|
+
/**
|
|
236
|
+
* Reduces one transcript's entries to {@link SpotlightParseRows}, carrying {@link SpotlightParserState} across tail
|
|
237
|
+
* parses. With `record` false it only advances the state (a prefix scan that rebuilds the state of a cursor without
|
|
238
|
+
* one).
|
|
239
|
+
*
|
|
240
|
+
* Needed lines: once a shaped read's result lands (applied, held out or score-free: any read with
|
|
241
|
+
* `details.spotlight.original`), later calls of the session, all from later requests, add to its needs: a `read` of the
|
|
242
|
+
* same file with a line selection adds the selected lines; an `edit` of it adds the lines its `*** Find` blocks
|
|
243
|
+
* locate in the original (read from the session's artifact directory in {@link finish}). Collecting stops at the
|
|
244
|
+
* file's first edit (after recording its lines; any edit tool), a later whole-file read, or a later shaped read of
|
|
245
|
+
* the file (after recording its selection), which starts collecting for itself.
|
|
246
|
+
*/
|
|
247
|
+
export declare class SpotlightCollector {
|
|
248
|
+
#private;
|
|
249
|
+
readonly rows: SpotlightParseRows;
|
|
250
|
+
readonly state: SpotlightParserState;
|
|
251
|
+
constructor(sessionFile: string, folder: string, state: SpotlightParserState, record?: boolean);
|
|
252
|
+
/**
|
|
253
|
+
* Locate the parse's edit Find blocks in their reads' originals (artifact files) and add the lines to the needs;
|
|
254
|
+
* an original that cannot be read adds none.
|
|
255
|
+
*/
|
|
256
|
+
finish(): Promise<void>;
|
|
257
|
+
visit(entry: unknown): void;
|
|
258
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Spotlight dashboard's round-2 measures over the analyzed sessions of `spotlight-aggregator.ts`: the
|
|
3
|
+
* holdout-adjusted net, ranking quality against the lines the agent needed later, step × size economics, time cost
|
|
4
|
+
* and tool-output coverage. Metric definitions live on the types in `shared-types.ts`.
|
|
5
|
+
*/
|
|
6
|
+
import type { Database } from "bun:sqlite";
|
|
7
|
+
import type { SpotlightAdjusted, SpotlightCompounding, SpotlightCoverageRow, SpotlightRanking, SpotlightStepCell, SpotlightTimeCost, SpotlightTool } from "./shared-types.js";
|
|
8
|
+
import type { Evaluated, SessionAnalysis } from "./spotlight-aggregator.js";
|
|
9
|
+
/** One experiment session's share of the range's savings and costs, for the adjusted net's joint bootstrap. */
|
|
10
|
+
export interface SessionMoney {
|
|
11
|
+
sessionFile: string;
|
|
12
|
+
savedTokens: number;
|
|
13
|
+
savedUsd: number;
|
|
14
|
+
/** Recovery tokens carried and priced plus extra-turn cost (its share of compounding.recoveryUsd). */
|
|
15
|
+
recoveryUsd: number;
|
|
16
|
+
spotlightUsd: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Per-cut holdout baselines of the range's evaluated results (see {@link baselinePerCut}): what a cut of `tool`
|
|
20
|
+
* gets fetched back anyway.
|
|
21
|
+
*/
|
|
22
|
+
export declare function holdoutBaselines(evaluated: readonly Evaluated[]): (tool: SpotlightTool) => number;
|
|
23
|
+
export declare function emptyAdjusted(): SpotlightAdjusted;
|
|
24
|
+
/**
|
|
25
|
+
* Holdout-adjusted net savings: each tool's applied cuts (scored and score-free) recover tokens; held-out would-be
|
|
26
|
+
* cuts of the same tool show how many the agent fetches back anyway. Intervals resample experiment sessions
|
|
27
|
+
* jointly. `byTool` lists `tools` in their order.
|
|
28
|
+
*/
|
|
29
|
+
export declare function computeAdjusted(evaluated: readonly Evaluated[], money: readonly SessionMoney[], grossSavedTokens: number, compounding: SpotlightCompounding, tools: readonly SpotlightTool[]): SpotlightAdjusted;
|
|
30
|
+
export declare function emptyRanking(): SpotlightRanking;
|
|
31
|
+
/**
|
|
32
|
+
* Ranking quality per group over shaped reads with needed lines (see {@link SpotlightRankingCell}), mean heat by
|
|
33
|
+
* line position over every scored cut or would-be cut, and how the needs were observed.
|
|
34
|
+
*/
|
|
35
|
+
export declare function computeRanking(evaluated: readonly Evaluated[]): SpotlightRanking;
|
|
36
|
+
/**
|
|
37
|
+
* What each step combination earns per tool and saved-token bin over the range's cuts; combinations with
|
|
38
|
+
* under {@link MIN_STEP_CUTS} cuts of a tool fold into `other`. The excess subtracts each cut's holdout baseline
|
|
39
|
+
* ({@link holdoutBaselines}). Cells follow `tools`' order, then the most common combination first.
|
|
40
|
+
*/
|
|
41
|
+
export declare function computeSteps(evaluated: readonly Evaluated[], baselineOf: (tool: SpotlightTool) => number, tools: readonly SpotlightTool[]): SpotlightStepCell[];
|
|
42
|
+
/**
|
|
43
|
+
* Agent time of the experiment sessions in the range: the model time of their requests, of the requests that did
|
|
44
|
+
* nothing but fetch back cut output, and the time tool calls spent shaping.
|
|
45
|
+
*/
|
|
46
|
+
export declare function computeTimeCost(active: readonly SessionAnalysis[], cutoff: number): SpotlightTimeCost;
|
|
47
|
+
/**
|
|
48
|
+
* Each tool's share of the tool output experiment sessions' models saw from their marker (and the range) on;
|
|
49
|
+
* `shaped` marks the tools with evaluated results in the range.
|
|
50
|
+
*/
|
|
51
|
+
export declare function computeCoverage(database: Database, active: readonly SessionAnalysis[], cutoff: number, shaped: ReadonlySet<SpotlightTool>): SpotlightCoverageRow[];
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@oh-my-pi/omp-stats",
|
|
4
|
-
"version": "18.
|
|
4
|
+
"version": "18.17.0",
|
|
5
5
|
"description": "Local observability dashboard for omp AI usage statistics",
|
|
6
6
|
"homepage": "https://omp.sh",
|
|
7
7
|
"author": {
|
|
@@ -36,15 +36,15 @@
|
|
|
36
36
|
"build": "bun run build.ts",
|
|
37
37
|
"dev": "bun run src/index.ts",
|
|
38
38
|
"check": "oxlint . && oxfmt --check --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts' && bun run check:types",
|
|
39
|
-
"check:types": "
|
|
39
|
+
"check:types": "bun check --no-script -p tsconfig.json && bun check --no-script -p tsconfig.client.json",
|
|
40
40
|
"lint": "oxlint .",
|
|
41
41
|
"fix": "oxlint --fix --fix-suggestions . && bun run fmt",
|
|
42
42
|
"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@oh-my-pi/pi-ai": "18.
|
|
46
|
-
"@oh-my-pi/pi-catalog": "18.
|
|
47
|
-
"@oh-my-pi/pi-utils": "18.
|
|
45
|
+
"@oh-my-pi/pi-ai": "18.17.0",
|
|
46
|
+
"@oh-my-pi/pi-catalog": "18.17.0",
|
|
47
|
+
"@oh-my-pi/pi-utils": "18.17.0",
|
|
48
48
|
"lucide-react": "^1.24.0",
|
|
49
49
|
"react": "19.2.7",
|
|
50
50
|
"react-dom": "19.2.7"
|
package/src/aggregator.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
getToolTimeSeries,
|
|
42
42
|
type RangeWindow,
|
|
43
43
|
} from "./rollup";
|
|
44
|
+
import { spotlightRowCount } from "./spotlight-ingest";
|
|
44
45
|
import type { SyncWorkerRequest, SyncWorkerResponse } from "./sync-worker";
|
|
45
46
|
// Coding-agent binary/bundle workers route through the CLI entrypoint with a
|
|
46
47
|
// hidden argv mode, so the compiled binary and npm bundle only need one
|
|
@@ -417,7 +418,8 @@ async function syncAllSessionsLocked(
|
|
|
417
418
|
result.userStats.length +
|
|
418
419
|
result.userLinks.length +
|
|
419
420
|
result.toolCalls.length +
|
|
420
|
-
result.toolResults.length
|
|
421
|
+
result.toolResults.length +
|
|
422
|
+
spotlightRowCount(result.spotlight);
|
|
421
423
|
if (pending.length >= SYNC_BATCH_FILES || pendingRows >= SYNC_BATCH_ROWS) flush();
|
|
422
424
|
};
|
|
423
425
|
|
package/src/client/App.tsx
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
ProjectsRoute,
|
|
14
14
|
ProvidersRoute,
|
|
15
15
|
RequestsRoute,
|
|
16
|
+
SpotlightRoute,
|
|
16
17
|
ToolsRoute,
|
|
17
18
|
TracesRoute,
|
|
18
19
|
} from "./routes";
|
|
@@ -62,6 +63,8 @@ function Dashboard() {
|
|
|
62
63
|
return <ProjectsRoute active={active} range={range} />;
|
|
63
64
|
case "gain":
|
|
64
65
|
return <GainRoute active={active} range={range} />;
|
|
66
|
+
case "spotlight":
|
|
67
|
+
return <SpotlightRoute active={active} range={range} />;
|
|
65
68
|
}
|
|
66
69
|
};
|
|
67
70
|
|
package/src/client/api.ts
CHANGED
|
@@ -14,6 +14,9 @@ import type {
|
|
|
14
14
|
RequestDetails,
|
|
15
15
|
SessionSummary,
|
|
16
16
|
SessionTrace,
|
|
17
|
+
SpotlightAgentScope,
|
|
18
|
+
SpotlightDashboardStats,
|
|
19
|
+
SpotlightSessionContext,
|
|
17
20
|
TimeRange,
|
|
18
21
|
ToolDashboardStats,
|
|
19
22
|
} from "./types";
|
|
@@ -184,6 +187,24 @@ export async function getGainDashboardStats(
|
|
|
184
187
|
return fetchJson<GainDashboardStats>(`${API_BASE}/stats/gain?${params}`, { signal });
|
|
185
188
|
}
|
|
186
189
|
|
|
190
|
+
export async function getSpotlightDashboardStats(
|
|
191
|
+
range: TimeRange = "24h",
|
|
192
|
+
agent: SpotlightAgentScope = "all",
|
|
193
|
+
signal?: AbortSignal,
|
|
194
|
+
): Promise<SpotlightDashboardStats> {
|
|
195
|
+
return fetchJson<SpotlightDashboardStats>(
|
|
196
|
+
`${API_BASE}/stats/spotlight?range=${encodeURIComponent(range)}&agent=${encodeURIComponent(agent)}`,
|
|
197
|
+
{ signal },
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** One experiment session's context per request, with and without shaping. */
|
|
202
|
+
export async function getSpotlightSessionContext(file: string, signal?: AbortSignal): Promise<SpotlightSessionContext> {
|
|
203
|
+
return fetchJson<SpotlightSessionContext>(`${API_BASE}/stats/spotlight/session?file=${encodeURIComponent(file)}`, {
|
|
204
|
+
signal,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
|
|
187
208
|
export async function getToolDashboardStats(
|
|
188
209
|
range: TimeRange = "24h",
|
|
189
210
|
signal?: AbortSignal,
|
package/src/client/app/nav.ts
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
LayoutGrid,
|
|
9
9
|
type LucideIcon,
|
|
10
10
|
PlugZap,
|
|
11
|
+
ScanSearch,
|
|
11
12
|
Sparkles,
|
|
12
13
|
SquareChartGantt,
|
|
13
14
|
Wrench,
|
|
@@ -24,7 +25,8 @@ export type DashboardSection =
|
|
|
24
25
|
| "tools"
|
|
25
26
|
| "frustration"
|
|
26
27
|
| "projects"
|
|
27
|
-
| "gain"
|
|
28
|
+
| "gain"
|
|
29
|
+
| "spotlight";
|
|
28
30
|
|
|
29
31
|
export interface NavItem {
|
|
30
32
|
id: DashboardSection;
|
|
@@ -65,6 +67,7 @@ export const NAV: readonly NavGroup[] = [
|
|
|
65
67
|
{ id: "frustration", label: "Frustration", icon: Frown, hotkey: "f" },
|
|
66
68
|
{ id: "projects", label: "Projects", icon: FolderGit2, hotkey: "j" },
|
|
67
69
|
{ id: "gain", label: "Gain", icon: Sparkles, hotkey: "g" },
|
|
70
|
+
{ id: "spotlight", label: "Spotlight", icon: ScanSearch, hotkey: "s" },
|
|
68
71
|
],
|
|
69
72
|
},
|
|
70
73
|
];
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Core SVG chart: categorical x slots, stacked or grouped bars, stacked areas
|
|
3
|
-
* and
|
|
4
|
-
* Marks are crisp and unsmoothed — square bars, straight segments.
|
|
2
|
+
* Core SVG chart: categorical x slots, stacked or grouped bars, stacked areas,
|
|
3
|
+
* lines and dots, an optional right axis, a hover cursor and a floating tooltip.
|
|
4
|
+
* Marks are crisp and unsmoothed — square bars, straight segments. Negative
|
|
5
|
+
* values extend the axis below zero (bars hang down from the baseline).
|
|
5
6
|
*
|
|
6
7
|
* `TimeChart` wraps this for bucketed time series; use `Chart` directly for
|
|
7
8
|
* categorical axes (hour of day, model names).
|
|
@@ -9,7 +10,7 @@
|
|
|
9
10
|
|
|
10
11
|
import { Fragment, type ReactNode, useId, useMemo, useState } from "react";
|
|
11
12
|
import { formatCompact } from "../data/formatters";
|
|
12
|
-
import type { ChartKind, ChartSeries, ReferenceLine } from "./types";
|
|
13
|
+
import type { ChartKind, ChartSeries, ReferenceLine, SlotMarker } from "./types";
|
|
13
14
|
import { useWidth } from "./useWidth";
|
|
14
15
|
|
|
15
16
|
export interface ChartProps {
|
|
@@ -39,6 +40,8 @@ export interface ChartProps {
|
|
|
39
40
|
/** Show a total row in the tooltip. Default: stacked with 2+ series. */
|
|
40
41
|
showTotal?: boolean;
|
|
41
42
|
references?: readonly ReferenceLine[];
|
|
43
|
+
/** Vertical guides at x slots, labelled along the top edge. */
|
|
44
|
+
markers?: readonly SlotMarker[];
|
|
42
45
|
onSelect?: (index: number) => void;
|
|
43
46
|
/** Shown over empty axes when every visible value is zero. */
|
|
44
47
|
emptyLabel?: ReactNode;
|
|
@@ -67,6 +70,7 @@ export function Chart({
|
|
|
67
70
|
hidden,
|
|
68
71
|
showTotal,
|
|
69
72
|
references,
|
|
73
|
+
markers,
|
|
70
74
|
onSelect,
|
|
71
75
|
emptyLabel = "No data in this range",
|
|
72
76
|
tooltipExtra,
|
|
@@ -80,31 +84,58 @@ export function Chart({
|
|
|
80
84
|
const layout = useMemo(() => {
|
|
81
85
|
const left = visible.filter(s => s.axis !== "right");
|
|
82
86
|
const right = visible.filter(s => s.axis === "right");
|
|
83
|
-
const stackable = (s: ChartSeries) =>
|
|
87
|
+
const stackable = (s: ChartSeries) => {
|
|
88
|
+
const mark = s.kind ?? kind;
|
|
89
|
+
return stacked && (mark === "bars" || mark === "area");
|
|
90
|
+
};
|
|
84
91
|
|
|
85
92
|
// Per-slot stack bases for stacked left series, in series order (first = bottom).
|
|
93
|
+
// Positive and negative values stack away from zero separately.
|
|
86
94
|
const bases = new Map<string, number[]>();
|
|
87
95
|
const tops = Array.from({ length: slots }, () => 0);
|
|
96
|
+
const bottoms = Array.from({ length: slots }, () => 0);
|
|
88
97
|
let leftMax = 0;
|
|
98
|
+
let leftMin = 0;
|
|
89
99
|
for (const s of left) {
|
|
90
100
|
if (stackable(s)) {
|
|
91
|
-
const base =
|
|
101
|
+
const base = Array.from({ length: slots }, () => 0);
|
|
102
|
+
for (let i = 0; i < slots; i++) {
|
|
103
|
+
const v = s.values[i] ?? 0;
|
|
104
|
+
if (v < 0) {
|
|
105
|
+
base[i] = bottoms[i];
|
|
106
|
+
bottoms[i] += v;
|
|
107
|
+
} else {
|
|
108
|
+
base[i] = tops[i];
|
|
109
|
+
tops[i] += v;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
92
112
|
bases.set(s.key, base);
|
|
93
|
-
for (let i = 0; i < slots; i++) tops[i] += s.values[i] ?? 0;
|
|
94
113
|
} else {
|
|
95
|
-
for (let i = 0; i < slots; i++)
|
|
114
|
+
for (let i = 0; i < slots; i++) {
|
|
115
|
+
leftMax = Math.max(leftMax, s.values[i] ?? 0);
|
|
116
|
+
leftMin = Math.min(leftMin, s.values[i] ?? 0);
|
|
117
|
+
}
|
|
96
118
|
}
|
|
97
119
|
}
|
|
98
|
-
for (
|
|
120
|
+
for (let i = 0; i < slots; i++) {
|
|
121
|
+
leftMax = Math.max(leftMax, tops[i]);
|
|
122
|
+
leftMin = Math.min(leftMin, bottoms[i]);
|
|
123
|
+
}
|
|
99
124
|
let rightMax = 0;
|
|
100
|
-
|
|
125
|
+
let rightMin = 0;
|
|
126
|
+
for (const s of right) {
|
|
127
|
+
for (let i = 0; i < slots; i++) {
|
|
128
|
+
rightMax = Math.max(rightMax, s.values[i] ?? 0);
|
|
129
|
+
rightMin = Math.min(rightMin, s.values[i] ?? 0);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
101
132
|
for (const r of references ?? []) {
|
|
102
133
|
if (r.axis === "right") rightMax = Math.max(rightMax, r.value);
|
|
103
134
|
else leftMax = Math.max(leftMax, r.value);
|
|
104
135
|
}
|
|
105
136
|
|
|
106
|
-
const leftScale = niceScale(yMax ?? leftMax);
|
|
107
|
-
const rightScale = right.length > 0 ? niceScale(yMaxRight ?? rightMax) : null;
|
|
137
|
+
const leftScale = niceScale(yMax ?? leftMax, leftMin);
|
|
138
|
+
const rightScale = right.length > 0 ? niceScale(yMaxRight ?? rightMax, rightMin) : null;
|
|
108
139
|
const empty = visible.every(s => s.values.every(v => !v));
|
|
109
140
|
return { left, right, bases, stackable, leftScale, rightScale, empty };
|
|
110
141
|
}, [visible, slots, stacked, kind, yMax, yMaxRight, references]);
|
|
@@ -116,8 +147,10 @@ export function Chart({
|
|
|
116
147
|
const plotW = Math.max(0, width - padLeft - padRight);
|
|
117
148
|
const plotH = height - PAD_TOP - PAD_BOTTOM;
|
|
118
149
|
const slotW = slots > 0 ? plotW / slots : 0;
|
|
119
|
-
const
|
|
120
|
-
|
|
150
|
+
const project = (scale: AxisScale, v: number) =>
|
|
151
|
+
PAD_TOP + plotH - ((v - scale.min) / (scale.max - scale.min)) * plotH;
|
|
152
|
+
const yLeft = (v: number) => project(layout.leftScale, v);
|
|
153
|
+
const yRight = (v: number) => (layout.rightScale ? project(layout.rightScale, v) : yLeft(v));
|
|
121
154
|
const cx = (i: number) => padLeft + (i + 0.5) * slotW;
|
|
122
155
|
|
|
123
156
|
const barSeries = visible.filter(s => (s.kind ?? kind) === "bars");
|
|
@@ -212,7 +245,12 @@ export function Chart({
|
|
|
212
245
|
return (
|
|
213
246
|
<g key={s.key}>
|
|
214
247
|
<path d={`${line}${floor}Z`} fill={fill} fillOpacity={s.pattern ? 1 : 0.2} />
|
|
215
|
-
<path
|
|
248
|
+
<path
|
|
249
|
+
className="chart-line"
|
|
250
|
+
d={line}
|
|
251
|
+
stroke={s.color}
|
|
252
|
+
strokeDasharray={s.dashed ? "4 3" : undefined}
|
|
253
|
+
/>
|
|
216
254
|
</g>
|
|
217
255
|
);
|
|
218
256
|
})}
|
|
@@ -280,6 +318,38 @@ export function Chart({
|
|
|
280
318
|
);
|
|
281
319
|
})}
|
|
282
320
|
|
|
321
|
+
{visible
|
|
322
|
+
.filter(s => (s.kind ?? kind) === "dots")
|
|
323
|
+
.map(s => {
|
|
324
|
+
const y = s.axis === "right" ? yRight : yLeft;
|
|
325
|
+
return (
|
|
326
|
+
<g key={s.key} className="chart-dots" fill={s.color}>
|
|
327
|
+
{s.values.map((v, i) =>
|
|
328
|
+
v === null || v === undefined ? null : (
|
|
329
|
+
<circle key={i} cx={cx(i)} cy={y(v)} r={hover === i ? 3.5 : 2.5} />
|
|
330
|
+
),
|
|
331
|
+
)}
|
|
332
|
+
</g>
|
|
333
|
+
);
|
|
334
|
+
})}
|
|
335
|
+
|
|
336
|
+
{markers?.map(m =>
|
|
337
|
+
m.index >= 0 && m.index < slots ? (
|
|
338
|
+
<g key={`marker:${m.index}`} className="chart-marker">
|
|
339
|
+
<line x1={cx(m.index)} x2={cx(m.index)} y1={PAD_TOP} y2={PAD_TOP + plotH} />
|
|
340
|
+
{m.label && (
|
|
341
|
+
<text
|
|
342
|
+
x={cx(m.index) + (cx(m.index) > padLeft + plotW / 2 ? -5 : 5)}
|
|
343
|
+
y={PAD_TOP + 9}
|
|
344
|
+
textAnchor={cx(m.index) > padLeft + plotW / 2 ? "end" : "start"}
|
|
345
|
+
>
|
|
346
|
+
{m.label}
|
|
347
|
+
</text>
|
|
348
|
+
)}
|
|
349
|
+
</g>
|
|
350
|
+
) : null,
|
|
351
|
+
)}
|
|
352
|
+
|
|
283
353
|
{references?.map(r => {
|
|
284
354
|
const y = r.axis === "right" ? yRight(r.value) : yLeft(r.value);
|
|
285
355
|
return (
|
|
@@ -351,16 +421,29 @@ export function Chart({
|
|
|
351
421
|
);
|
|
352
422
|
}
|
|
353
423
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
424
|
+
export interface AxisScale {
|
|
425
|
+
min: number;
|
|
426
|
+
max: number;
|
|
427
|
+
ticks: number[];
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/**
|
|
431
|
+
* Round an axis span out to 1/2/2.5/5 × 10ⁿ steps and return evenly spaced
|
|
432
|
+
* ticks. The span always includes zero; `min` (default 0) extends it below.
|
|
433
|
+
*/
|
|
434
|
+
export function niceScale(max: number, min = 0): AxisScale {
|
|
435
|
+
const hi = Math.max(0, max);
|
|
436
|
+
const lo = Math.min(0, min);
|
|
437
|
+
if (!(hi - lo > 0)) return { min: 0, max: 1, ticks: [0] };
|
|
438
|
+
const rough = (hi - lo) / Y_TICKS;
|
|
358
439
|
const magnitude = 10 ** Math.floor(Math.log10(rough));
|
|
359
440
|
const step = [1, 2, 2.5, 5, 10].map(m => m * magnitude).find(s => s >= rough) ?? 10 * magnitude;
|
|
360
|
-
const top = Math.ceil(
|
|
441
|
+
const top = Math.ceil(hi / step) * step;
|
|
442
|
+
const bottom = Math.floor(lo / step) * step;
|
|
361
443
|
const ticks: number[] = [];
|
|
362
|
-
for (let t =
|
|
363
|
-
|
|
444
|
+
for (let t = bottom; t <= top + step / 2; t += step) ticks.push(Number(t.toPrecision(12)));
|
|
445
|
+
// An all-negative axis still needs a span above its lowest tick.
|
|
446
|
+
return { min: bottom, max: top > bottom ? top : bottom + step, ticks };
|
|
364
447
|
}
|
|
365
448
|
|
|
366
449
|
function labelWidth(labels: readonly string[]): number {
|