@oh-my-pi/omp-stats 18.16.0 → 18.17.0

package/src/shared-types.ts

@@ -321,6 +321,459 @@ export interface GainDashboardStats {
 	projects: string[];
 }
 
+/**
+ * A tool the Spotlight experiment shaped, by the name it was called with (`read`, `grep`, an MCP tool's name, …).
+ * Every tool's results are shaped (`spotlight.experiment.tools`, default all); read, bash and eval have their own
+ * paths in pi, every other tool goes through the generic one.
+ */
+export type SpotlightTool = string;
+
+/** One tool's totals, for lists ordered by the tool's saving. */
+export interface SpotlightToolRow extends SpotlightToolTotals {
+	tool: SpotlightTool;
+}
+
+/**
+ * How the model got back output a shortened result hid:
+ * - `artifact`: a later call read or searched an artifact the shortened result named (`artifact://N`);
+ * - `reread`: a later `read` of the same file whose selection overlaps the hidden lines, or the whole file again;
+ * - `rerun`: a later call of the same tool repeating the shortened one: a bash command with a pipe, redirect or
+ *   changed flags; any other tool with near-identical arguments (same tool, argument tokens' Jaccard ≥ 0.6).
+ */
+export type SpotlightRecoveryKind = "artifact" | "reread" | "rerun";
+
+/** A ratio over sessions with its bootstrap 90% interval (`lo`/`hi` null under 3 sessions; `value` null with none). */
+export interface SpotlightEstimate {
+	value: number | null;
+	lo: number | null;
+	hi: number | null;
+}
+
+/**
+ * Token totals of one tool's evaluated results. Savings are measured against the `plain` text pi would have shown
+ * unshaped for the same call (its inline cap and read page), not against the full output shaping reads.
+ */
+export interface SpotlightToolTotals {
+	/** Results the experiment evaluated. */
+	results: number;
+	/** Results it shortened. */
+	shaped: number;
+	/** The full output shaping read (a spilled bash output's whole artifact, a whole file up to 2 MiB). */
+	inTokens: number;
+	/**
+	 * What `plain` would have shown: the trace's `plainTokens`, else (read, bash and eval traces written before pi
+	 * recorded it) estimated from the inline cap.
+	 */
+	plainTokens: number;
+	outTokens: number;
+	/**
+	 * Σ (plainTokens − outTokens) over shortened results; a cut that showed more than `plain` would have counts
+	 * negative (see {@link SpotlightGross.widerThanPlain}).
+	 */
+	savedTokens: number;
+	/** Σ (inTokens − outTokens): the saving against the full output, which overstates what the model was spared. */
+	fullSavedTokens: number;
+}
+
+/** Strategy 1, gross savings: what the cuts removed. */
+export interface SpotlightGross extends SpotlightToolTotals {
+	/** savedTokens / (savedTokens + every tool-output token the model saw in the same sessions). */
+	ofToolOutput: number | null;
+	/** savedTokens / every token of the same sessions' requests (input + output + cache read + cache write). */
+	ofSessionTokens: number | null;
+	/** Every tool with an evaluated result in the range, the largest saving first. */
+	byTool: SpotlightToolRow[];
+	/** Shortened results whose `plainTokens` is estimated (journaled before pi recorded it). */
+	estimatedPlain: number;
+	/**
+	 * Shortened results whose cut was wider than what `plain` would have shown (out > plain): a loss where the cut
+	 * was shown, a saving of 0 where pi capped it at `plain`'s text (counted by the would-be cut).
+	 */
+	widerThanPlain: number;
+}
+
+/** Recovery totals of one kind. */
+export interface SpotlightRecoveryTotals {
+	events: number;
+	/** Tokens of the recovery calls' results, which re-enter the context. */
+	tokens: number;
+	/** Requests that issued nothing but recovery calls. */
+	turns: number;
+	/** Provider cost of those requests. */
+	turnsUsd: number;
+	/**
+	 * The recovery tokens carried through later requests, priced like {@link SpotlightCompounding.savedUsd}
+	 * (excluding `turnsUsd`); Σ over kinds + clawback.turnsUsd = compounding.recoveryUsd.
+	 */
+	usd: number;
+}
+
+/** Strategy 2, clawback: what getting hidden output back cost. */
+export interface SpotlightClawback extends SpotlightRecoveryTotals {
+	/** Shortened results with hidden lines: the ones a recovery can follow. */
+	withHidden: number;
+	/** Of those, the results at least one recovery followed. */
+	recovered: number;
+	byKind: Record<SpotlightRecoveryKind, SpotlightRecoveryTotals>;
+	/** gross.savedTokens − tokens. */
+	netTokens: number;
+}
+
+/** Strategy 3, compounding: savings as every later request of the session would have re-read them. */
+export interface SpotlightCompounding {
+	/** Σ over shortened results of saved tokens × later requests of the session until a compaction. */
+	avoidedTokens: number;
+	/** Recovery tokens carried the same way: they undo what the cut avoided. */
+	recoveredCarriedTokens: number;
+	/** Prompt tokens the same sessions' requests processed (input + cache read + cache write). */
+	promptTokens: number;
+	/** (avoided − recoveredCarried) / (promptTokens + avoided − recoveredCarried). */
+	avoidedShare: number | null;
+	/** Avoided tokens priced at each later request's own cache-read rate, plus one cache write. */
+	savedUsd: number;
+	/** recoveredCarriedTokens priced the same way, plus clawback.turnsUsd. */
+	recoveryUsd: number;
+	/** Spotlight's own bill (`spotlight:shape` costUsd). */
+	spotlightUsd: number;
+	/** savedUsd − recoveryUsd − spotlightUsd. */
+	netUsd: number;
+	/** Wall time shaping added to tool calls (Σ `spotlight:shape` ms). */
+	shapeMs: number;
+}
+
+/** Strategy 4: one bucket of the tool-output share of context, per session group. */
+export interface SpotlightSharePoint {
+	/** Bucket start (the range's bucket size, as every time series). */
+	timestamp: number;
+	/** `shaped`: an experiment session from its marker on; `outside`: everything else. */
+	group: "shaped" | "outside";
+	sessions: number;
+	/** Tool-result tokens the model saw. */
+	toolTokens: number;
+	/** Everything that entered the context: tool results, assistant output, user prompts and the first request's prompt. */
+	contextTokens: number;
+}
+
+/** One bucket of savings over time. */
+export interface SpotlightBucket {
+	timestamp: number;
+	results: number;
+	shaped: number;
+	savedTokens: number;
+	recoveries: number;
+	recoveredTokens: number;
+	netTokens: number;
+	savedUsd: number;
+	netUsd: number;
+}
+
+/** A recovery attributed to the latest shortened result it fetches from. */
+export interface SpotlightRecovery {
+	kind: SpotlightRecoveryKind;
+	toolCallId: string;
+	tool: string;
+	/** The call's command, path or pattern, clipped to 200 chars. */
+	target: string;
+	timestamp: number;
+	tokens: number;
+	/** Tool calls between the shortened result and this one. */
+	after: number;
+	/** Its request issued nothing but recovery calls. */
+	turn: boolean;
+}
+
+/** One evaluated result that changed the output, or would have (`holdout`). */
+export interface SpotlightResultRow {
+	sessionFile: string;
+	toolCallId: string;
+	timestamp: number;
+	tool: SpotlightTool;
+	/** Held out: the model saw the result unshaped; `savedTokens` is what the cut would have saved. */
+	holdout: boolean;
+	/** Command, path (with selector) or eval code head, clipped to 200 chars. */
+	target: string;
+	inTokens: number;
+	outTokens: number;
+	savedTokens: number;
+	hiddenLines: number;
+	steps: string[];
+	spotlightUsd: number;
+	shapeMs: number;
+	/** What `plain` would have shown (see {@link SpotlightToolTotals.plainTokens}). */
+	plainTokens: number;
+	/** Cut by the no-score fallback rules because the score-free coin skipped Spotlight. */
+	scoreFree: boolean;
+	recoveries: SpotlightRecovery[];
+	/** savedTokens − Σ recovery tokens. */
+	netTokens: number;
+}
+
+/**
+ * Who saw what for a result in the fetch-back comparison:
+ * - `applied`: Spotlight shortened it and the model saw the shortened text;
+ * - `holdout`: Spotlight would have shortened it, but the result was held out at random and the model saw it
+ *   unshaped (the experiment's true counterfactual);
+ * - `reference`: shaping left it unchanged (results of the same tool and sessions, standardized to the applied
+ *   cuts' original-size mix), the stand-in until held-out results accumulate;
+ * - `scoreFree`: shortened by the no-score fallback rules because a random coin skipped Spotlight
+ *   (`spotlight.experiment.scoreFree`), the model saw the cut: against `applied`, what Spotlight's scores add;
+ * - `gated`: a read pi's former edit gate (removed) kept whole, the model saw plain's page; only in transcripts
+ *   journaled while the gate ran.
+ * `applied` holds only cuts Spotlight scored (score-free cuts and cuts whose Spotlight calls failed are not in it).
+ */
+export type SpotlightCutGroup = "applied" | "holdout" | "reference" | "scoreFree" | "gated";
+
+/**
+ * Share of results the agent fetched more of within 12 calls, by one rule for every group: a later `read` of the
+ * same path overlapping the result's own window (the whole file without a selector), a rerun (see
+ * {@link SpotlightRecoveryKind}), or a read of an artifact the result named; not counted after an edit of that path
+ * (reads) or any edit (reruns), nor from the same request.
+ */
+export interface SpotlightFetchBackCell {
+	tool: SpotlightTool;
+	group: SpotlightCutGroup;
+	results: number;
+	sessions: number;
+	/** Session-clustered bootstrap over the group's results. */
+	rate: SpotlightEstimate;
+}
+
+/** What the cuts of one tool earned, in tokens and dollars (applied cuts only; held-out results save nothing). */
+export interface SpotlightCutEconomics {
+	tool: SpotlightTool;
+	cuts: number;
+	savedTokens: number;
+	recoveredTokens: number;
+	/** Requests that did nothing but fetch back output these cuts hid. */
+	extraTurns: number;
+	savedUsd: number;
+	/** Recovered tokens carried through later requests, priced like the savings. */
+	recoveryUsd: number;
+	/** Provider cost of the extra turns. */
+	turnUsd: number;
+	/** savedUsd − recoveryUsd − turnUsd − this cell's Spotlight bill. */
+	netUsd: number;
+	extraTurnsPer100Cuts: number;
+	netUsdPer100Cuts: number;
+}
+
+/** Calls between a cut (or held-out result) and the first fetch-back after it. */
+export type SpotlightGapBin = "1" | "2" | "3" | "4–6" | "7–12";
+
+/** Fetch-backs per gap bin and tool, for applied and held-out results: every bin of each tool with at least one. */
+export interface SpotlightGapCount {
+	tool: SpotlightTool;
+	gap: SpotlightGapBin;
+	applied: number;
+	holdout: number;
+}
+
+/** What shaping costs in time and reliability (all evaluated results, held-out ones included). */
+export interface SpotlightOverhead {
+	results: number;
+	/** Wall time shaping added to a tool call. */
+	shapeMsP50: number | null;
+	shapeMsP90: number | null;
+	/** Spotlight's own request time, over results whose calls all answered. */
+	spotlightMsP50: number | null;
+	spotlightMsP90: number | null;
+	/** Results whose Spotlight calls failed or timed out (shaped by the no-score fallback). */
+	failureRate: number | null;
+	/** Wall time tool calls spent shaping results whose Spotlight calls failed (mostly the 5 s timeout per call). */
+	failedWaitMs: number;
+	spotlightUsd: number;
+}
+
+/**
+ * The net once fetch-backs that would have happened anyway are taken out: held-out results (the model saw them
+ * unshaped) show how much of a would-be cut's output the agent reads again regardless, per tool; only recoveries
+ * beyond that baseline are the cut's cost. A tool's baseline is thin while it has fewer than `minHoldouts`.
+ */
+export interface SpotlightAdjustedTool {
+	tool: SpotlightTool;
+	/** Applied cuts (scored and score-free) and their recovered tokens. */
+	cuts: number;
+	recoveredTokens: number;
+	/** Held-out would-be cuts the baseline is measured on. */
+	holdouts: number;
+	/** Recovered tokens per held-out would-be cut, over all the tool's holdouts (session-clustered bootstrap). */
+	baselinePerCut: SpotlightEstimate;
+	/** max(0, recovered − baseline per cut × cuts). */
+	excessTokens: SpotlightEstimate;
+}
+
+/** Holdout-adjusted net savings (tokens, against the `plain` baseline). */
+export interface SpotlightAdjusted {
+	byTool: SpotlightAdjustedTool[];
+	/** Σ tools' excess (bootstrapped jointly over sessions). */
+	excessTokens: SpotlightEstimate;
+	/** gross.savedTokens − excessTokens. */
+	netTokens: SpotlightEstimate;
+	/** compounding.savedUsd − recoveryUsd × (excess / recovered) − spotlightUsd. */
+	netUsd: SpotlightEstimate;
+	/** Holdouts per tool below which its baseline is too thin to trust (the UI marks the tool). */
+	minHoldouts: number;
+}
+
+/**
+ * Lines of a shaped read's original the agent needed later: lines a later `read` of the same file selected, and lines
+ * a later `edit` of it changed (located by its Find text in the original), until the file's first edit or a later
+ * whole-file read.
+ */
+export type SpotlightNeedSource = "reread" | "edit";
+
+/**
+ * How well a group's per-line choices match what the agent needed later, over shaped reads with at least one needed
+ * line. Applied and score-free cuts are biased alike (the agent re-reads what it lacks), so compare them with each
+ * other; held-out would-be cuts are the unbiased measure.
+ */
+export interface SpotlightRankingCell {
+	group: "applied" | "holdout" | "scoreFree";
+	results: number;
+	neededLines: number;
+	/** P(heat of a needed line > heat of an unneeded line of the same result), ties half; 0.5 is chance. Null for `scoreFree` (no heat). */
+	auc: SpotlightEstimate;
+	/** Needed-line rate among each result's hottest 10% of lines over its rate among all lines; 1 is no better than chance. */
+	topLift: SpotlightEstimate;
+	/** Share of needed lines the cut kept over the share of all lines it kept; 1 is a random cut of the same size. */
+	keptLift: SpotlightEstimate;
+}
+
+/** Mean heat (0–9) of one line position over shaped results of at least 40 lines. */
+export interface SpotlightHeatPosition {
+	/** `line 1`, `lines 2–3`, then `0–10%` … `90–100%` of the output. */
+	label: string;
+	meanHeat: number;
+	lines: number;
+}
+
+/** Ranking quality: does Spotlight keep the lines the agent comes back for? */
+export interface SpotlightRanking {
+	cells: SpotlightRankingCell[];
+	position: SpotlightHeatPosition[];
+	/** Results with needed lines, by how the need was observed (a result can count under both). */
+	sources: Record<SpotlightNeedSource, number>;
+}
+
+/** Saved-token bins (against `plain`) the step economics split cuts by; a negative saving falls in `<250`. */
+export type SpotlightSizeBin = "<250" | "250–1k" | "1k–4k" | "≥4k";
+
+/** What one shaping step (or combination) earns at one cut size: applied cuts of one tool. */
+export interface SpotlightStepCell {
+	tool: SpotlightTool;
+	/** The cut's shaping steps joined with `+` (`outline+window`); combinations under 10 cuts in the range fold into `other`. */
+	steps: string;
+	size: SpotlightSizeBin;
+	cuts: number;
+	fetchBackRate: SpotlightEstimate;
+	/** Σ (plain − out), negative cuts (wider than `plain`) included. */
+	savedTokens: number;
+	recoveredTokens: number;
+	/** max(0, recovered − Σ each cut's holdout baseline (its tool's)). */
+	excessTokens: number;
+	/** (saved − excess) / saved: the share of the saving that stays saved; null when the cell saved nothing (≤ 0). */
+	keptShare: number | null;
+}
+
+/** What the experiment costs in agent time. */
+export interface SpotlightTimeCost {
+	/** Model time of the experiment sessions' requests (Σ request duration). */
+	agentMs: number;
+	/** Requests that did nothing but fetch back cut output, and their model time. */
+	fetchBackTurns: number;
+	fetchBackMs: number;
+	/** Wall time tool calls spent shaping (Spotlight requests included). */
+	shapeMs: number;
+	/** (fetchBackMs + shapeMs) / agentMs. */
+	share: number | null;
+}
+
+/** One tool's share of the tool output experiment sessions' models saw (chars ÷ 4). */
+export interface SpotlightCoverageRow {
+	tool: string;
+	/** The experiment evaluated results of it in the range. */
+	shaped: boolean;
+	calls: number;
+	tokens: number;
+	share: number;
+}
+
+/**
+ * Which transcripts the Spotlight dashboard measures (`GET /api/stats/spotlight?agent=`, default `all`): `main`
+ * agents only, task `subagent`s only, or everything (advisors appear only under `all`). Every metric, the share
+ * trend's `outside` group included, is filtered the same way.
+ */
+export type SpotlightAgentScope = "all" | "main" | "subagent";
+
+/** An experiment session, for the context chart's picker. */
+export interface SpotlightSessionSummary {
+	sessionFile: string;
+	title: string | null;
+	folder: string;
+	/** Which agent wrote the transcript (each subagent transcript is its own experiment session). */
+	agentType: AgentType;
+	/** The session's first Spotlight marker: activity before it (a resumed pre-experiment session) is not measured. */
+	startedAt: number;
+	requests: number;
+	savedTokens: number;
+	netUsd: number;
+}
+
+/**
+ * Spotlight dashboard payload (`GET /api/stats/spotlight?range=&agent=`). Every experiment session shapes tool
+ * output; held-out results within them are the counterfactual. Only activity at or after a session's Spotlight
+ * marker is the experiment's; earlier activity of the same transcript counts as `outside` in the share trend and is
+ * left out of every other metric.
+ */
+export interface SpotlightDashboardStats {
+	/** Experiment sessions with activity in the range. */
+	sessions: number;
+	/** First Spotlight marker ever seen (not range-bound), for the trend's marker; null before the experiment. */
+	experimentStart: number | null;
+	gross: SpotlightGross;
+	clawback: SpotlightClawback;
+	compounding: SpotlightCompounding;
+	series: SpotlightBucket[];
+	share: SpotlightSharePoint[];
+	/** Latest 60 shortened results, newest first. */
+	recent: SpotlightResultRow[];
+	/** Latest 30 experiment sessions in the range, newest first. */
+	sessionList: SpotlightSessionSummary[];
+	/** Held-out results in the range (would-be cuts the model saw unshaped). */
+	holdouts: number;
+	/** Score-free cuts in the range (shortened without Spotlight by the fallback rules). */
+	scoreFree: number;
+	adjusted: SpotlightAdjusted;
+	ranking: SpotlightRanking;
+	steps: SpotlightStepCell[];
+	timeCost: SpotlightTimeCost;
+	coverage: SpotlightCoverageRow[];
+	fetchBack: SpotlightFetchBackCell[];
+	economics: SpotlightCutEconomics[];
+	gaps: SpotlightGapCount[];
+	overhead: SpotlightOverhead;
+}
+
+/** One request of a session's context chart. */
+export interface SpotlightContextPoint {
+	timestamp: number;
+	/** input + cache read + cache write of this request. */
+	contextTokens: number;
+	/** The context without shaping: contextTokens + net tokens saved so far (reset at a compaction). */
+	withoutTokens: number;
+	/** The context shrank sharply before this request (a compaction). */
+	compacted: boolean;
+	/** Tokens saved and recovered by results that landed since the previous request. */
+	saved: number;
+	recovered: number;
+}
+
+/** A session's context with and without shaping (`GET /api/stats/spotlight/session?file=`). */
+export interface SpotlightSessionContext {
+	session: SpotlightSessionSummary;
+	points: SpotlightContextPoint[];
+}
+
 /**
  * Aggregated usage for a single tool over the active range.
  *