@oh-my-pi/omp-stats 18.15.0 → 18.17.0

package/src/client/spotlight/Ranking.tsx

@@ -0,0 +1,248 @@
+import { DotWhisker, type DotWhiskerMark, type DotWhiskerRow } from "../charts";
+import { formatCompact, formatInteger } from "../data/formatters";
+import type { SpotlightDashboardStats, SpotlightHeatPosition, SpotlightRankingCell } from "../types";
+import { Card } from "../ui";
+import { GroupLegend } from "./chips";
+import { CUT_GROUP, formatInterval } from "./meta";
+
+type RankingGroup = SpotlightRankingCell["group"];
+
+/** Scored cuts first (the figure), then what they are read against. */
+const GROUPS: readonly RankingGroup[] = ["applied", "scoreFree", "holdout"];
+
+const MARK_OFFSET: Record<RankingGroup, number> = { applied: -7, scoreFree: 0, holdout: 7 };
+
+interface Measure {
+	key: "auc" | "topLift" | "keptLift";
+	title: string;
+	/** What the chance line means for this measure. */
+	chanceLabel: string;
+	chance: number;
+	rule: string;
+	format: (value: number) => string;
+}
+
+const MEASURES: readonly Measure[] = [
+	{
+		key: "auc",
+		title: "Heat ranks needed lines higher",
+		chanceLabel: "AUC · chance 0.5",
+		chance: 0.5,
+		rule: "P(a needed line is hotter than an unneeded line of the same result), ties half",
+		format: v => v.toFixed(2),
+	},
+	{
+		key: "topLift",
+		title: "Hottest tenth holds them",
+		chanceLabel: "lift · chance 1.0",
+		chance: 1,
+		rule: "Needed-line rate among each result's hottest 10% of lines over its rate among all lines",
+		format: v => `${v.toFixed(2)}×`,
+	},
+	{
+		key: "keptLift",
+		title: "The cut keeps them",
+		chanceLabel: "lift · random cut 1.0",
+		chance: 1,
+		rule: "Share of needed lines the cut kept over the share of all lines it kept",
+		format: v => `${v.toFixed(2)}×`,
+	},
+];
+
+const NEED_SOURCES = [
+	{ key: "reread", label: "Read again", rule: "A later read of the same file selected the lines" },
+	{ key: "edit", label: "Edited", rule: "A later edit's Find text matched the lines" },
+] as const;
+
+/** Per-line ranking quality: Spotlight's heat and the cut against the lines the agent needed later. */
+export function Ranking({ stats, stale }: { stats: SpotlightDashboardStats; stale: boolean }) {
+	const { ranking } = stats;
+	const cells: ReadonlyMap<RankingGroup, SpotlightRankingCell> = new Map(
+		ranking.cells.map(cell => [cell.group, cell]),
+	);
+	const measured = GROUPS.some(group => (cells.get(group)?.results ?? 0) > 0);
+
+	return (
+		<Card
+			index={4}
+			title="Does Spotlight keep what the agent needs?"
+			description="Lines of a shaped read the agent read again or edited later, against Spotlight's heat and the cut"
+			actions={<GroupLegend groups={GROUPS} />}
+			stale={stale}
+			footer="Shaped reads with at least one needed line, until the file's first edit or a later whole-file read. Scored and score-free cuts are biased alike (the agent re-reads what a cut hid), so compare them with each other; held-out would-be cuts, shown whole, are the unbiased measure. Score-free cuts have no heat, so only the cut's lift exists for them. Dashed lines are chance; whiskers are 90% session bootstrap intervals, from 3 sessions."
+		>
+			{!measured ? (
+				<div className="empty">No shaped read was needed again in this range.</div>
+			) : (
+				<div className="spot-rank">
+					<div className="spot-rank-panels">
+						{MEASURES.map(measure => (
+							<RankPanel key={measure.key} measure={measure} cells={cells} />
+						))}
+					</div>
+					<div className="spot-rank-foot">
+						<SampleTable cells={cells} />
+						<PositionHeat positions={ranking.position} />
+						<section className="spot-rank-sources" aria-label="How the need was seen">
+							<h3 className="spot-panel-title">How the need was seen</h3>
+							<dl className="spot-facts spot-rank-facts">
+								{NEED_SOURCES.map(source => (
+									<div key={source.key} title={source.rule}>
+										<dt>{source.label}</dt>
+										<dd className="num">
+											{formatInteger(ranking.sources[source.key])}
+											<span className="dim"> results</span>
+										</dd>
+									</div>
+								))}
+							</dl>
+						</section>
+					</div>
+				</div>
+			)}
+		</Card>
+	);
+}
+
+function RankPanel({ measure, cells }: { measure: Measure; cells: ReadonlyMap<RankingGroup, SpotlightRankingCell> }) {
+	const marks: DotWhiskerMark[] = GROUPS.flatMap(group => {
+		const cell = cells.get(group);
+		if (cell === undefined || cell.results === 0) return [];
+		const estimate = cell[measure.key];
+		const interval = formatInterval(estimate, measure.format);
+		return [
+			{
+				key: group,
+				...estimate,
+				color: CUT_GROUP[group].color,
+				shape: CUT_GROUP[group].shape,
+				offset: MARK_OFFSET[group],
+				title: `${CUT_GROUP[group].label}: ${estimate.value === null ? "–" : measure.format(estimate.value)}${interval === "" ? "" : ` (${interval})`} over ${formatInteger(cell.results)} results, ${formatInteger(cell.neededLines)} needed lines`,
+			},
+		];
+	});
+	const applied = cells.get("applied")?.[measure.key].value ?? null;
+	const rows: DotWhiskerRow[] = [
+		{
+			key: "reads",
+			label: "Shaped reads",
+			marks,
+			figure: (
+				<span style={{ color: CUT_GROUP.applied.color }}>{applied === null ? "–" : measure.format(applied)}</span>
+			),
+			detail: (
+				<>
+					{(["scoreFree", "holdout"] as const).map((group, i) => {
+						const value = cells.get(group)?.[measure.key].value ?? null;
+						return (
+							<span key={group} title={CUT_GROUP[group].label}>
+								{i > 0 && " · "}
+								<span className="spot-fb-detail" data-group={group}>
+									{value === null ? "–" : measure.format(value)}
+								</span>
+							</span>
+						);
+					})}
+				</>
+			),
+		},
+	];
+	return (
+		<section className="spot-rank-panel" aria-label={measure.title}>
+			<h3 className="spot-panel-title" title={measure.rule}>
+				{measure.title}
+				<span className="spot-rank-chance">{measure.chanceLabel}</span>
+			</h3>
+			<DotWhisker
+				rows={rows}
+				guides={[{ key: "chance", value: measure.chance, color: "var(--ink-3)" }]}
+				format={measure.format}
+				formatTick={v => String(Number(v.toFixed(2)))}
+			/>
+		</section>
+	);
+}
+
+/** Results and needed lines behind every cell of the panels. */
+function SampleTable({ cells }: { cells: ReadonlyMap<RankingGroup, SpotlightRankingCell> }) {
+	return (
+		<section aria-label="Sample per cell">
+			<h3 className="spot-panel-title">
+				Sample per cell
+				<span className="spot-rank-chance">results · needed lines</span>
+			</h3>
+			<table className="spot-rank-n">
+				<thead>
+					<tr>
+						{GROUPS.map(group => (
+							<th key={group} scope="col" title={CUT_GROUP[group].rule}>
+								{CUT_GROUP[group].label}
+							</th>
+						))}
+					</tr>
+				</thead>
+				<tbody>
+					<tr>
+						{GROUPS.map(group => {
+							const cell = cells.get(group);
+							return (
+								<td key={group} className="num" data-thin={cell !== undefined && cell.results < 20}>
+									{cell === undefined || cell.results === 0 ? (
+										<span className="dim">–</span>
+									) : (
+										<span title={`${formatInteger(cell.neededLines)} needed lines`}>
+											{formatInteger(cell.results)}
+											<span className="dim"> · {formatCompact(cell.neededLines)}</span>
+										</span>
+									)}
+								</td>
+							);
+						})}
+					</tr>
+				</tbody>
+			</table>
+		</section>
+	);
+}
+
+/** Mean heat by line position. */
+function PositionHeat({ positions }: { positions: readonly SpotlightHeatPosition[] }) {
+	const top = Math.max(0, ...positions.map(p => p.meanHeat));
+	return (
+		<section className="spot-rank-pos" aria-label="Mean heat by position">
+			<h3 className="spot-panel-title">
+				Mean heat by position
+				<span className="spot-rank-chance">0–9, outputs ≥ 40 lines</span>
+			</h3>
+			{positions.length === 0 || top === 0 ? (
+				<div className="dim spot-gap-empty">No heat recorded</div>
+			) : (
+				<>
+					<div className="spot-rank-bars">
+						{positions.map(p => (
+							<div
+								key={p.label}
+								className="spot-rank-bar"
+								title={`${p.label}: mean heat ${p.meanHeat.toFixed(2)} over ${formatInteger(p.lines)} lines`}
+							>
+								<span className="spot-rank-bar-track">
+									<span className="spot-rank-bar-value num">{p.meanHeat.toFixed(1)}</span>
+									{/* The value's line (14px) stays clear above the tallest bar. */}
+									<span
+										className="spot-rank-bar-fill"
+										style={{ height: `calc((100% - 14px) * ${p.meanHeat / top})` }}
+									/>
+								</span>
+								{/* `line 1` → `L1`, `lines 2–3` → `L2–3`, `0–10%` → `0–10`: short enough to sit under a bar. */}
+								<span className="spot-rank-bar-label num">
+									{p.label.replace(/^lines? /, "L").replace(/%$/, "")}
+								</span>
+							</div>
+						))}
+					</div>
+					<div className="micro spot-rank-axis">L = line of the output, then each tenth of it (% into it)</div>
+				</>
+			)}
+		</section>
+	);
+}

package/src/client/spotlight/ResultsFeed.tsx

@@ -0,0 +1,354 @@
+import { CornerDownRight, Scissors } from "lucide-react";
+import { useCallback, useMemo, useState } from "react";
+import {
+	formatCompact,
+	formatCost,
+	formatDurationMs,
+	formatElapsed,
+	formatInteger,
+	formatRelativeTime,
+	formatTimestamp,
+} from "../data/formatters";
+import type { SpotlightRecoveryKind, SpotlightResultRow } from "../types";
+import { Badge, Card, type Column, Drawer, KeyValues, Table } from "../ui";
+import { KindBadge } from "./chips";
+import { formatSignedCompact, RECOVERY_KIND, RECOVERY_KINDS, toolIcon } from "./meta";
+
+function rowKey(row: SpotlightResultRow): string {
+	return `${row.sessionFile}\u0000${row.toolCallId}`;
+}
+
+function NetCell({ value }: { value: number }) {
+	return (
+		<span className={value < 0 ? "num tone-bad" : value > 0 ? "num tone-ok" : "num"}>
+			{formatSignedCompact(value)}
+		</span>
+	);
+}
+
+/** Plain → out with a bar of what survived the cut (would have, when held out). */
+function ShrinkCell({ row }: { row: SpotlightResultRow }) {
+	const { plainTokens, outTokens } = row;
+	const kept = plainTokens > 0 ? Math.min(1, outTokens / plainTokens) : 1;
+	return (
+		<div
+			className="spot-shrink"
+			data-holdout={row.holdout}
+			title={`plain would show ${formatInteger(plainTokens)} tokens, the cut ${row.holdout ? "would show" : "showed"} ${formatInteger(outTokens)} (shaping read ${formatInteger(row.inTokens)})`}
+		>
+			<span className="num">
+				{formatCompact(plainTokens)}
+				<span className="dim"> → </span>
+				{formatCompact(outTokens)}
+			</span>
+			<span className="spot-shrink-bar" aria-hidden="true">
+				<span className="spot-shrink-kept" style={{ width: `${kept * 100}%` }} />
+			</span>
+		</div>
+	);
+}
+
+function kindCounts(row: SpotlightResultRow): [SpotlightRecoveryKind, number][] {
+	const counts = new Map<SpotlightRecoveryKind, number>();
+	for (const r of row.recoveries) counts.set(r.kind, (counts.get(r.kind) ?? 0) + 1);
+	return RECOVERY_KINDS.flatMap(kind => {
+		const count = counts.get(kind);
+		return count ? [[kind, count]] : [];
+	});
+}
+
+const COLUMNS: Column<SpotlightResultRow>[] = [
+	{
+		key: "target",
+		header: "Result",
+		width: "40%",
+		sort: row => row.timestamp,
+		render: row => {
+			const Icon = toolIcon(row.tool);
+			return (
+				<div className="spot-result">
+					<span className="spot-tool" title={row.tool}>
+						<Icon size={14} aria-hidden="true" />
+						<span className="sr-only">{row.tool}</span>
+					</span>
+					<div className="stack spot-result-text">
+						<span className="spot-target truncate" title={row.target}>
+							{row.target}
+						</span>
+						<span className="row spot-result-meta">
+							{row.holdout && <HoldoutBadge />}
+							{row.scoreFree && <ScoreFreeBadge />}
+							<span className="dim" title={formatTimestamp(row.timestamp)}>
+								{formatRelativeTime(row.timestamp)}
+							</span>
+						</span>
+					</div>
+				</div>
+			);
+		},
+	},
+	{
+		key: "shrink",
+		header: "Plain → out",
+		align: "right",
+		sort: row => (row.plainTokens > 0 ? row.outTokens / row.plainTokens : 1),
+		title: "What pi would have shown unshaped (plain) for the call, and what the cut showed; the bar is the share that survived",
+		render: row => <ShrinkCell row={row} />,
+	},
+	{
+		key: "saved",
+		header: "Saved",
+		align: "right",
+		sort: row => row.savedTokens,
+		render: row =>
+			row.holdout ? (
+				<span className="spot-would-save" title="Held out: what the cut would have saved">
+					<span className="dim">would save </span>
+					<span className="num">{formatSignedCompact(row.savedTokens)}</span>
+				</span>
+			) : (
+				<span className="num">{formatSignedCompact(row.savedTokens)}</span>
+			),
+	},
+	{
+		key: "hidden",
+		header: "Hidden lines",
+		align: "right",
+		sort: row => row.hiddenLines,
+		render: row =>
+			row.hiddenLines > 0 ? (
+				<span className="num">{formatInteger(row.hiddenLines)}</span>
+			) : (
+				<span className="dim">–</span>
+			),
+	},
+	{
+		key: "recoveries",
+		header: "Fetched back",
+		sort: row => row.recoveries.length,
+		render: row => {
+			const counts = kindCounts(row);
+			return counts.length === 0 ? (
+				<span className="dim" title="Nothing fetched back">
+					–
+				</span>
+			) : (
+				<span className="row spot-kinds">
+					{counts.map(([kind, count]) => (
+						<KindBadge key={kind} kind={kind} count={count} />
+					))}
+				</span>
+			);
+		},
+	},
+	{
+		key: "net",
+		header: "Net",
+		align: "right",
+		sort: row => row.netTokens,
+		title: "Saved tokens minus the tokens of the calls that fetched hidden output back",
+		render: row =>
+			row.holdout ? (
+				<span className="dim" title="Held out: the model saw the whole result, so nothing was saved">
+					–
+				</span>
+			) : (
+				<NetCell value={row.netTokens} />
+			),
+	},
+];
+
+function HoldoutBadge() {
+	return (
+		<span title="Held out at random: the model saw the original, unshortened result">
+			<Badge>held out</Badge>
+		</span>
+	);
+}
+
+function ScoreFreeBadge() {
+	return (
+		<span title="A random coin skipped Spotlight: the no-score fallback rules made this cut">
+			<Badge>score-free</Badge>
+		</span>
+	);
+}
+
+/** The 60 newest shortened results; a row opens its recovery timeline. */
+export function ResultsFeed({ rows, stale }: { rows: readonly SpotlightResultRow[]; stale: boolean }) {
+	const [selected, setSelected] = useState<string | null>(null);
+	const close = useCallback(() => setSelected(null), []);
+	const current = useMemo(() => rows.find(row => rowKey(row) === selected) ?? null, [rows, selected]);
+
+	return (
+		<>
+			<Card
+				index={6}
+				title="Recent results"
+				description="The newest shortened and held-out tool results and what the agent fetched back"
+				flush
+				stale={stale}
+				footer="Saved is what plain would have shown minus what the cut showed (the would-be cut for held-out ones, which save nothing); recovery tokens are the recovery calls' results (chars ÷ 4 when no exact count exists)."
+			>
+				<Table
+					rows={rows}
+					rowKey={rowKey}
+					columns={COLUMNS}
+					onRowClick={row => setSelected(rowKey(row))}
+					selectedKey={selected}
+					limit={15}
+					empty={<div className="empty">No shortened or held-out results in this range.</div>}
+				/>
+			</Card>
+			<RecoveryDrawer row={current} onClose={close} />
+		</>
+	);
+}
+
+function RecoveryDrawer({ row, onClose }: { row: SpotlightResultRow | null; onClose: () => void }) {
+	if (row === null) return null;
+	const Icon = toolIcon(row.tool);
+	const recovered = row.savedTokens - row.netTokens;
+	return (
+		<Drawer
+			open
+			onClose={onClose}
+			width={600}
+			title={
+				<span className="spot-drawer-title" title={row.target}>
+					{row.target}
+				</span>
+			}
+			subtitle={
+				<span className="row spot-drawer-subtitle">
+					<Icon size={13} aria-hidden="true" />
+					{row.tool}
+					{row.holdout && <HoldoutBadge />}
+					{row.scoreFree && <ScoreFreeBadge />}
+					<span>{formatTimestamp(row.timestamp)}</span>
+				</span>
+			}
+		>
+			<div className="spot-drawer-summary">
+				<div>
+					<div className="spot-eyebrow">{row.holdout ? "Would save" : "Saved"}</div>
+					<div className="spot-drawer-figure num">{formatSignedCompact(row.savedTokens)}</div>
+				</div>
+				<div>
+					<div className="spot-eyebrow">Fetched back</div>
+					<div className={recovered > 0 ? "spot-drawer-figure num tone-warn" : "spot-drawer-figure num"}>
+						{formatCompact(recovered)}
+					</div>
+				</div>
+				<div>
+					<div className="spot-eyebrow">Net</div>
+					{row.holdout ? (
+						<div className="spot-drawer-figure dim" title="Held out: the model saw the whole result">
+							–
+						</div>
+					) : (
+						<div
+							className={
+								row.netTokens < 0 ? "spot-drawer-figure num tone-bad" : "spot-drawer-figure num tone-ok"
+							}
+						>
+							{formatSignedCompact(row.netTokens)}
+						</div>
+					)}
+				</div>
+			</div>
+
+			<KeyValues
+				items={[
+					{ key: "plain", label: "Plain would show", value: formatInteger(row.plainTokens) },
+					{ key: "out", label: "Tokens out", value: formatInteger(row.outTokens) },
+					{ key: "in", label: "Full output read", value: formatInteger(row.inTokens) },
+					{
+						key: "hidden",
+						label: row.holdout ? "Lines it would hide" : "Hidden lines",
+						value: formatInteger(row.hiddenLines),
+					},
+					{ key: "steps", label: "Steps", value: row.steps.length > 0 ? row.steps.join(" → ") : "–" },
+					{ key: "cost", label: "Spotlight cost", value: formatCost(row.spotlightUsd) },
+					{ key: "ms", label: "Shaping time", value: formatDurationMs(row.shapeMs) },
+				]}
+			/>
+
+			<div>
+				<div className="section-label">Call</div>
+				<pre className="code-block spot-drawer-call">{row.target}</pre>
+			</div>
+
+			<div>
+				<div className="section-label">Recovery timeline</div>
+				<ol className="spot-timeline">
+					<li className="spot-timeline-item" data-kind={row.holdout ? "holdout" : "cut"}>
+						<span className="spot-timeline-node" aria-hidden="true">
+							<Scissors size={12} />
+						</span>
+						<div className="spot-timeline-body">
+							<div className="row spot-timeline-head">
+								<span className="cell-primary">
+									{row.holdout ? "Held out: the model saw the original" : "Shortened"}
+								</span>
+								<span className="dim num">
+									{row.holdout && "would be "}
+									{formatCompact(row.plainTokens)} → {formatCompact(row.outTokens)} tokens
+									{row.hiddenLines > 0 && `, ${formatInteger(row.hiddenLines)} lines hidden`}
+								</span>
+							</div>
+						</div>
+					</li>
+					{row.recoveries.map(r => (
+						<li
+							key={r.toolCallId}
+							className="spot-timeline-item"
+							data-kind={r.kind}
+							style={{ "--kind": RECOVERY_KIND[r.kind].color }}
+						>
+							<span className="spot-timeline-node" aria-hidden="true">
+								<CornerDownRight size={12} />
+							</span>
+							<div className="spot-timeline-body">
+								<div className="row spot-timeline-head">
+									<KindBadge kind={r.kind} />
+									{r.turn && <Badge tone="warn">extra turn</Badge>}
+									<span className="spot-timeline-spacer" />
+									<span className="num tone-warn">+{formatCompact(r.tokens)}</span>
+								</div>
+								<code className="spot-timeline-target" title={r.target}>
+									<span className="dim">{r.tool} </span>
+									{r.target}
+								</code>
+								<div className="micro">
+									{r.after === 0
+										? "The next call"
+										: `${formatInteger(r.after)} ${r.after === 1 ? "call" : "calls"} later`}
+									{` · ${formatElapsed(Math.max(0, r.timestamp - row.timestamp))} after the cut`}
+								</div>
+							</div>
+						</li>
+					))}
+					<li className="spot-timeline-item" data-kind="net">
+						<span className="spot-timeline-node" aria-hidden="true" />
+						<div className="spot-timeline-body">
+							<div className="row spot-timeline-head">
+								<span className="cell-primary">
+									{row.holdout
+										? row.recoveries.length === 0
+											? "Nothing fetched back"
+											: "Fetched more although it saw everything"
+										: row.recoveries.length === 0
+											? "Nothing fetched back: the full saving stands"
+											: "Net"}
+								</span>
+								<span className="spot-timeline-spacer" />
+								{row.holdout ? <span className="dim">saves nothing</span> : <NetCell value={row.netTokens} />}
+							</div>
+						</div>
+					</li>
+				</ol>
+			</div>
+		</Drawer>
+	);
+}