@oh-my-pi/omp-stats 18.15.0 → 18.17.0

package/src/client/spotlight/FetchBack.tsx

@@ -0,0 +1,222 @@
+import { useState } from "react";
+import { DotWhisker, type DotWhiskerMark, type DotWhiskerRow } from "../charts";
+import { formatInteger } from "../data/formatters";
+import type {
+	SpotlightCutGroup,
+	SpotlightDashboardStats,
+	SpotlightFetchBackCell,
+	SpotlightGapCount,
+	SpotlightTool,
+} from "../types";
+import { Card } from "../ui";
+import { GroupLegend, MoreTools } from "./chips";
+import { CUT_GROUP, CUT_GROUPS, foldTools, formatShare, GAP_BINS, toolIcon, toolsBySaving } from "./meta";
+
+/** Vertical spread of a row's marks, so overlapping intervals stay readable. */
+const MARK_OFFSET: Record<SpotlightCutGroup, number> = {
+	applied: -12,
+	scoreFree: -6,
+	holdout: 0,
+	gated: 6,
+	reference: 12,
+};
+
+/** What a scored cut's rate is read against, in the row's small print: the score-free cut, then the baselines. */
+const COMPARISONS: readonly SpotlightCutGroup[] = ["scoreFree", "holdout", "reference"];
+
+/** Groups the gap histogram counts (`SpotlightGapCount` fields). */
+type GapGroup = "applied" | "holdout";
+const GAP_GROUPS: readonly GapGroup[] = ["applied", "holdout"];
+
+const FETCH_BACK_RULE =
+	"One rule for every group: within 12 calls, a read of the same path overlapping the result's own window (the whole file without a selector), a rerun of the call (a similar bash command; for other tools, ≥ 60% shared argument tokens), or a read of an artifact it named; not after an edit of that path (reads) or any edit (reruns), nor from the same request.";
+
+export interface FetchBackProps {
+	stats: SpotlightDashboardStats;
+	stale: boolean;
+}
+
+/** Fetch-back rate per tool: the scored cut against the score-free one, the random holdout and the unchanged results. */
+export function FetchBack({ stats, stale }: FetchBackProps) {
+	const [expanded, setExpanded] = useState(false);
+	const cells = new Map<string, SpotlightFetchBackCell>();
+	for (const cell of stats.fetchBack) cells.set(`${cell.tool}/${cell.group}`, cell);
+	const measured = new Set(stats.fetchBack.filter(cell => cell.results > 0).map(cell => cell.tool));
+	const { listed, rest } = foldTools(toolsBySaving(stats).filter(tool => measured.has(tool)));
+	// Only reads were gated, and only while pi's former edit gate ran: the legend names the group where it has results.
+	const legend = CUT_GROUPS.filter(
+		group => group !== "gated" || stats.fetchBack.some(cell => cell.group === "gated" && cell.results > 0),
+	);
+
+	const rows: DotWhiskerRow[] = [];
+	for (const tool of expanded ? [...listed, ...rest] : listed) {
+		const Icon = toolIcon(tool);
+		const groups = CUT_GROUPS.flatMap(group => {
+			const cell = cells.get(`${tool}/${group}`);
+			return cell !== undefined && cell.results > 0 ? [cell] : [];
+		});
+		if (groups.length === 0) continue;
+		const byGroup = new Map(groups.map(cell => [cell.group, cell]));
+		const applied = byGroup.get("applied");
+		const marks: DotWhiskerMark[] = groups.map(cell => ({
+			key: cell.group,
+			...cell.rate,
+			color: CUT_GROUP[cell.group].color,
+			shape: CUT_GROUP[cell.group].shape,
+			offset: MARK_OFFSET[cell.group],
+			title: `${CUT_GROUP[cell.group].label}: ${formatShare(cell.rate.value)} of ${formatInteger(cell.results)} results in ${formatInteger(cell.sessions)} sessions`,
+		}));
+		rows.push({
+			key: tool,
+			label: (
+				<span className="spot-fb-tool" title={tool}>
+					<Icon size={13} aria-hidden="true" />
+					<span className="spot-fb-tool-name">{tool}</span>
+				</span>
+			),
+			marks,
+			figure: (
+				<span style={{ color: CUT_GROUP.applied.color }} title={CUT_GROUP.applied.rule}>
+					{applied === undefined ? "–" : formatShare(applied.rate.value)}
+				</span>
+			),
+			detail: (
+				<>
+					{COMPARISONS.map((group, i) => {
+						const cell = byGroup.get(group);
+						return (
+							<span key={group} title={`${CUT_GROUP[group].label}: ${CUT_GROUP[group].rule}`}>
+								{i > 0 && " · "}
+								<span className="spot-fb-detail" data-group={group}>
+									{cell === undefined ? "–" : formatShare(cell.rate.value)}
+								</span>
+							</span>
+						);
+					})}
+				</>
+			),
+		});
+	}
+
+	return (
+		<Card
+			index={3}
+			title="Does the agent come back for it?"
+			description="Share of results the agent fetched more of within 12 calls: cut with Spotlight's scores, cut without them, held out at random, kept whole by pi's former edit gate, and left unchanged"
+			stale={stale}
+			footer={`${FETCH_BACK_RULE} Held-out results are the true counterfactual: drawn at random from the would-be cuts and shown whole; until they accumulate, unchanged results stand in, re-weighted to the cuts' size mix. Score-free cuts skip Spotlight by a second coin and take the no-score fallback rules: if they come back as often as scored cuts, the scores add nothing. Gated reads are reads pi's former edit gate (removed) kept whole, listed only where the range has any. Small print: score-free · held out · unchanged. Whiskers are 90% session bootstrap intervals, from 3 sessions.`}
+		>
+			{rows.length === 0 ? (
+				<div className="empty">No evaluated results in this range.</div>
+			) : (
+				<div className="spot-fb">
+					<GroupLegend groups={legend} />
+					<DotWhisker rows={rows} format={formatShare} formatTick={v => `${Math.round(v * 100)}%`} />
+					{rest.length > 0 && (
+						<MoreTools hidden={rest} expanded={expanded} onToggle={() => setExpanded(value => !value)} />
+					)}
+				</div>
+			)}
+		</Card>
+	);
+}
+
+/** Fetch-backs per gap bin as each group's share of its own fetch-backs (the groups differ in size). */
+function GapPanel({ tool, counts }: { tool: SpotlightTool; counts: readonly SpotlightGapCount[] }) {
+	const Icon = toolIcon(tool);
+	const byBin = new Map(counts.map(c => [c.gap, c]));
+	let applied = 0;
+	let holdout = 0;
+	for (const c of counts) {
+		applied += c.applied;
+		holdout += c.holdout;
+	}
+	const totals: Record<GapGroup, number> = { applied, holdout };
+	let top = 0;
+	for (const c of counts) {
+		if (applied > 0) top = Math.max(top, c.applied / applied);
+		if (holdout > 0) top = Math.max(top, c.holdout / holdout);
+	}
+
+	return (
+		<section className="spot-gap" aria-label={`${tool} fetch-back gaps`}>
+			<h3 className="spot-panel-title row spot-gap-title">
+				<Icon size={13} aria-hidden="true" />
+				{tool}
+				<span className="dim">
+					<span className="num">{formatInteger(applied)}</span> shortened ·{" "}
+					<span className="num">{formatInteger(holdout)}</span> held out
+				</span>
+			</h3>
+			{applied + holdout === 0 ? (
+				<div className="spot-gap-empty dim">No fetch-backs</div>
+			) : (
+				<div className="spot-gap-bins">
+					{GAP_BINS.map(bin => {
+						const count = byBin.get(bin);
+						return (
+							<div key={bin} className="spot-gap-bin">
+								<div className="spot-gap-bars">
+									{GAP_GROUPS.map(group => {
+										const n = count?.[group] ?? 0;
+										const total = totals[group];
+										const share = total > 0 ? n / total : 0;
+										const noun = bin === "1" ? "call" : "calls";
+										return (
+											<span
+												key={group}
+												className="spot-gap-bar"
+												data-group={group}
+												data-empty={total === 0}
+												title={`${bin} ${noun} later: ${formatInteger(n)} of ${formatInteger(total)} ${CUT_GROUP[group].label.toLowerCase()} fetch-backs`}
+											>
+												{total > 0 && <span className="spot-gap-value num">{formatShare(share)}</span>}
+												<span
+													className="spot-gap-fill"
+													style={{ height: `${top > 0 ? (share / top) * 100 : 0}%` }}
+												/>
+											</span>
+										);
+									})}
+								</div>
+								<span className="spot-gap-label num">{bin}</span>
+							</div>
+						);
+					})}
+				</div>
+			)}
+		</section>
+	);
+}
+
+/** How many calls after the cut the agent fetched back, per tool, shortened against held out. */
+export function FetchBackGaps({ stats, stale }: FetchBackProps) {
+	const [expanded, setExpanded] = useState(false);
+	const counted = new Set(stats.gaps.map(count => count.tool));
+	const { listed, rest } = foldTools(toolsBySaving(stats).filter(tool => counted.has(tool)));
+	return (
+		<Card
+			index={3}
+			title="How soon it comes back"
+			description="Calls between a result and the first fetch-back after it"
+			stale={stale}
+			footer="Bars are each group's share of its own fetch-backs, so shortened and held-out results compare despite their different counts (next to each tool). Same rule as above; nothing after 12 calls counts."
+		>
+			{listed.length === 0 ? (
+				<div className="empty">No fetch-backs in this range.</div>
+			) : (
+				<>
+					<GroupLegend groups={GAP_GROUPS} />
+					<div className="spot-gaps">
+						{(expanded ? [...listed, ...rest] : listed).map(tool => (
+							<GapPanel key={tool} tool={tool} counts={stats.gaps.filter(c => c.tool === tool)} />
+						))}
+					</div>
+					{rest.length > 0 && (
+						<MoreTools hidden={rest} expanded={expanded} onToggle={() => setExpanded(value => !value)} />
+					)}
+				</>
+			)}
+		</Card>
+	);
+}

package/src/client/spotlight/Hero.tsx

@@ -0,0 +1,177 @@
+import { TriangleAlert } from "lucide-react";
+import { formatCompact, formatInteger } from "../data/formatters";
+import type { SpotlightDashboardStats } from "../types";
+import { Stat } from "../ui";
+import { foldTools, formatInterval, formatShare, formatSignedCompact, formatSignedCost, OTHER_TOOLS } from "./meta";
+
+export interface SpotlightHeroProps {
+	stats: SpotlightDashboardStats;
+	/** Prose window ("the last 7 days"). */
+	windowLabel: string;
+	stale: boolean;
+}
+
+/**
+ * Headline: the holdout-adjusted net (tokens and $, with intervals) against what `plain` would have shown, the old
+ * full-output figure struck through as the baseline note, then gross, the excess fetch-back and the sample.
+ */
+export function SpotlightHero({ stats, windowLabel, stale }: SpotlightHeroProps) {
+	const { gross, clawback, adjusted } = stats;
+	const net = adjusted.netTokens;
+	const netUsd = adjusted.netUsd;
+	const tokenInterval = formatInterval(net, formatSignedCompact);
+	const usdInterval = formatInterval(netUsd, formatSignedCost);
+	// A tool's baseline only matters when it has cuts to adjust; `adjusted.byTool` runs the largest saving first.
+	const thin = adjusted.byTool.filter(row => row.cuts > 0 && row.holdouts < adjusted.minHoldouts).map(row => row.tool);
+	// Prose list: `read`, `read and bash`, `read, bash and eval`, `read, bash, … and 4 more tools`.
+	const thinFold = foldTools(thin);
+	const thinNames = thinFold.rest.length > 0 ? [...thinFold.listed, `${thinFold.rest.length} more tools`] : thin;
+	const thinProse =
+		thinNames.length < 2 ? (thinNames[0] ?? "") : `${thinNames.slice(0, -1).join(", ")} and ${thinNames.at(-1)}`;
+	const heldFold = foldTools(adjusted.byTool);
+	const heldRows = heldFold.listed.map(row => ({
+		tool: row.tool,
+		holdouts: row.holdouts,
+		thin: thin.includes(row.tool),
+	}));
+	if (heldFold.rest.length > 0) {
+		heldRows.push({
+			tool: OTHER_TOOLS,
+			holdouts: heldFold.rest.reduce((sum, row) => sum + row.holdouts, 0),
+			thin: heldFold.rest.some(row => thin.includes(row.tool)),
+		});
+	}
+	const overstated = gross.fullSavedTokens > 0 ? 1 - gross.savedTokens / gross.fullSavedTokens : null;
+
+	return (
+		<section className="card rise spot-hero" data-stale={stale} aria-label="Spotlight savings summary">
+			<div className="spot-hero-main">
+				<div className="spot-hero-headline">
+					<div className="spot-eyebrow">Net saved in {windowLabel}, after fetch-backs a cut caused</div>
+					<div className="spot-hero-figures">
+						<span
+							className="spot-hero-value gradient-text"
+							title={net.value === null ? undefined : `${formatInteger(net.value)} tokens`}
+						>
+							{net.value === null ? "–" : formatSignedCompact(net.value)}
+						</span>
+						<span className="spot-hero-unit">tokens</span>
+						{netUsd.value !== null && (
+							<span
+								className="spot-hero-usd"
+								data-negative={netUsd.value < 0}
+								title="Avoided context priced at each later request's cache rates, minus the fetch-backs beyond the holdout baseline (priced the same way) and Spotlight's own bill"
+							>
+								{formatSignedCost(netUsd.value)}
+							</span>
+						)}
+					</div>
+					{(tokenInterval !== "" || usdInterval !== "") && (
+						<div className="spot-hero-interval">
+							90% interval{" "}
+							<span className="num">{[tokenInterval, usdInterval].filter(Boolean).join(" · ")}</span>
+						</div>
+					)}
+				</div>
+				<div className="spot-hero-aside">
+					<p className="spot-hero-note">
+						Counted against what pi would have shown unshaped (<code>plain</code>) for the same call (its inline
+						cap and read page), with fetch-backs that held-out results show happen anyway added back.{" "}
+						{gross.fullSavedTokens > gross.savedTokens && (
+							<>
+								Against the full output the shaping read, the cuts would claim{" "}
+								<s className="spot-struck num">{formatCompact(gross.fullSavedTokens)}</s>, overstating the gross
+								by {formatShare(overstated)}.{" "}
+							</>
+						)}
+						{gross.widerThanPlain > 0 &&
+							`${formatInteger(gross.widerThanPlain)} ${gross.widerThanPlain === 1 ? "result was" : "results were"} wider than plain would have shown: cuts shown that way count as losses, results pi capped at plain's text save nothing. `}
+						{gross.estimatedPlain > 0 &&
+							`${formatInteger(gross.estimatedPlain)} older ${gross.estimatedPlain === 1 ? "result estimates" : "results estimate"} plain from the inline cap.`}
+					</p>
+					{thin.length > 0 && (
+						<p className="spot-hero-warn" role="note">
+							<TriangleAlert size={14} aria-hidden="true" />
+							<span>
+								{thinProse} {thin.length === 1 ? "has" : "have"} fewer than{" "}
+								{formatInteger(adjusted.minHoldouts)} held-out results: {thin.length === 1 ? "its" : "their"}{" "}
+								part of the net rests on a thin baseline.
+							</span>
+						</p>
+					)}
+				</div>
+			</div>
+
+			<div className="spot-hero-stats">
+				<Stat
+					size="sm"
+					label="Gross vs plain"
+					title="Σ (plain − shortened) tokens over shortened results; a cut that showed more than plain counts negative"
+					value={formatSignedCompact(gross.savedTokens)}
+					hint={`${formatShare(gross.ofToolOutput)} of tool output · ${formatInteger(gross.shaped)} of ${formatInteger(gross.results)} results shortened`}
+				/>
+				<Stat
+					size="sm"
+					label="Fetched back beyond baseline"
+					title="Fetch-back tokens beyond what held-out results (shown whole) fetch anyway, per tool (reads split by whether the agent was likely to edit them); 90% interval below"
+					value={adjusted.excessTokens.value === null ? "–" : formatSignedCompact(-adjusted.excessTokens.value)}
+					hint={
+						<>
+							of {formatCompact(clawback.tokens)} fetched back
+							{adjusted.excessTokens.lo !== null && adjusted.excessTokens.hi !== null && (
+								<>
+									{" "}
+									<span className="dim">
+										·{" "}
+										<span className="spot-nowrap">
+											90% {formatInterval(adjusted.excessTokens, formatCompact)}
+										</span>
+									</span>
+								</>
+							)}
+						</>
+					}
+				/>
+				<Stat
+					size="sm"
+					label="Held out"
+					title={`Would-be cuts shown whole at random: the baseline per tool (under ${adjusted.minHoldouts} is thin)`}
+					value={formatInteger(stats.holdouts)}
+					hint={
+						<span className="spot-hero-tools">
+							{heldRows.map(row => (
+								<span
+									key={row.tool}
+									data-thin={row.thin}
+									title={
+										row.tool === OTHER_TOOLS
+											? heldFold.rest.map(rest => `${rest.tool} ${formatInteger(rest.holdouts)}`).join(", ")
+											: row.thin
+												? `${row.tool}: under ${adjusted.minHoldouts} held out`
+												: undefined
+									}
+								>
+									{row.tool} <span className="num">{formatInteger(row.holdouts)}</span>
+									{row.thin && <TriangleAlert size={11} aria-label="thin baseline" />}
+								</span>
+							))}
+						</span>
+					}
+				/>
+				<Stat
+					size="sm"
+					label="Score-free cuts"
+					title="Cuts a random coin made without Spotlight (the no-score fallback rules): what Spotlight's scores add is the difference"
+					value={formatInteger(stats.scoreFree)}
+					hint="cut without Spotlight, for comparison"
+				/>
+				<Stat
+					size="sm"
+					label="Sessions"
+					value={formatInteger(stats.sessions)}
+					hint="experiment sessions, all shaped"
+				/>
+			</div>
+		</section>
+	);
+}

package/src/client/spotlight/Overhead.tsx

@@ -0,0 +1,135 @@
+import { TriangleAlert } from "lucide-react";
+import { formatCost, formatDurationMs, formatInteger } from "../data/formatters";
+import type { SpotlightDashboardStats, SpotlightOverhead } from "../types";
+import { Badge, Card, type Column, errorRateTone, Table } from "../ui";
+import { formatShare } from "./meta";
+
+/** p50 with p90 beside it, dim. */
+function Percentiles({ p50, p90 }: { p50: number | null; p90: number | null }) {
+	if (p50 === null) return <span className="dim">–</span>;
+	return (
+		<span className="num">
+			{formatDurationMs(p50)}
+			<span className="dim"> / {formatDurationMs(p90)}</span>
+		</span>
+	);
+}
+
+const COLUMNS: Column<SpotlightOverhead>[] = [
+	{
+		key: "failures",
+		header: "Spotlight failures",
+		align: "right",
+		sort: row => row.failureRate ?? -1,
+		title: "Results whose Spotlight calls failed or timed out, of those that called it: they were cut by the no-score fallback",
+		render: row =>
+			row.failureRate === null ? (
+				<span className="dim">–</span>
+			) : (
+				<Badge tone={errorRateTone(row.failureRate)} mono>
+					{formatShare(row.failureRate)}
+				</Badge>
+			),
+	},
+	{
+		key: "failedWait",
+		header: "Waited on failures",
+		align: "right",
+		sort: row => row.failedWaitMs,
+		title: "Wall time tool calls spent on Spotlight calls that failed or timed out",
+		render: row =>
+			row.failedWaitMs > 0 ? (
+				<span className="num">{formatDurationMs(row.failedWaitMs)}</span>
+			) : (
+				<span className="dim">–</span>
+			),
+	},
+	{
+		key: "results",
+		header: "Results",
+		align: "right",
+		sort: row => row.results,
+		title: "Results Spotlight evaluated, held-out ones included",
+		render: row => <span className="num">{formatInteger(row.results)}</span>,
+	},
+	{
+		key: "shape",
+		header: "Shaping p50 / p90",
+		align: "right",
+		sort: row => row.shapeMsP50 ?? -1,
+		title: "Wall time shaping added to a tool call",
+		render: row => <Percentiles p50={row.shapeMsP50} p90={row.shapeMsP90} />,
+	},
+	{
+		key: "spotlight",
+		header: "Spotlight p50 / p90",
+		align: "right",
+		sort: row => row.spotlightMsP50 ?? -1,
+		title: "Spotlight's own request time, over results whose calls all answered",
+		render: row => <Percentiles p50={row.spotlightMsP50} p90={row.spotlightMsP90} />,
+	},
+	{
+		key: "usd",
+		header: "Spotlight $",
+		align: "right",
+		sort: row => row.spotlightUsd,
+		render: row => <span className="num">{formatCost(row.spotlightUsd)}</span>,
+	},
+];
+
+/**
+ * Failed Spotlight calls mean the cut came from the no-score fallback, not Spotlight's scorer: every comparison then
+ * partly measures the fallback. Called out above the table once the failure rate reaches the warning rate.
+ */
+function FailureCallout({ overhead }: { overhead: SpotlightOverhead }) {
+	const rate = overhead.failureRate;
+	if (rate === null) return null;
+	const tone = errorRateTone(rate);
+	if (tone === "ok") return null;
+	const waitedMs = overhead.failedWaitMs;
+	return (
+		<div className="spot-callout" data-tone={tone} role="note">
+			<TriangleAlert size={16} aria-hidden="true" />
+			<div>
+				<div className="spot-callout-head">
+					<span className="num">{formatShare(rate)}</span> of results that called Spotlight failed
+				</div>
+				<p className="spot-callout-copy">
+					A failed or timed-out call leaves the cut to the no-score fallback, so those results measure the fallback
+					rather than Spotlight's scorer. Read every comparison on this page with that share in mind.
+					{waitedMs > 0 && (
+						<>
+							{" "}
+							Tool calls waited <span className="num">{formatDurationMs(waitedMs)}</span> on those failures.
+						</>
+					)}
+				</p>
+			</div>
+		</div>
+	);
+}
+
+/** What shaping costs in latency, reliability and Spotlight's bill. */
+export function Overhead({ stats, stale }: { stats: SpotlightDashboardStats; stale: boolean }) {
+	const { overhead } = stats;
+	const rows = overhead.results > 0 ? [overhead] : [];
+	return (
+		<Card
+			index={3}
+			title="Overhead"
+			description="What shaping costs: Spotlight failures, added latency and its bill"
+			flush
+			stale={stale}
+			footer="Every evaluated result counts, held-out ones included: they were shaped too, just not shown."
+		>
+			{overhead.results > 0 && <FailureCallout overhead={overhead} />}
+			<Table
+				rows={rows}
+				rowKey={() => "overhead"}
+				columns={COLUMNS}
+				dense
+				empty={<div className="empty">No evaluated results in this range.</div>}
+			/>
+		</Card>
+	);
+}