@oh-my-pi/omp-stats 18.15.0 → 18.17.0

package/src/client/spotlight/Compounding.tsx

@@ -0,0 +1,222 @@
+import { format } from "@oh-my-pi/pi-utils/dates";
+import { useMemo, useState } from "react";
+import { getSpotlightSessionContext } from "../api";
+import { Chart, type ChartSeries, Legend, type SlotMarker } from "../charts";
+import { formatCompact, formatFolder, formatInteger } from "../data/formatters";
+import { useQuery } from "../data/query";
+import type { SpotlightDashboardStats, SpotlightSessionContext, SpotlightSessionSummary } from "../types";
+import { Badge, Card, ChartSkeleton, EmptyState, ErrorState } from "../ui";
+import { formatShare, formatSignedCompact, formatSignedCost } from "./meta";
+
+export interface CompoundingProps {
+	stats: SpotlightDashboardStats;
+	active: boolean;
+	stale: boolean;
+}
+
+/**
+ * Title, else a subagent's file name (`Consumers.jsonl`), else the project
+ * folder. Subagent and advisor sessions are indented under an arrow and tagged,
+ * since they sit in the newest-first list beside the session that spawned them.
+ */
+function sessionLabel(session: SpotlightSessionSummary): string {
+	const main = session.agentType === "main";
+	const base = (session.sessionFile.split("/").pop() ?? "").replace(/\.jsonl$/, "");
+	const name = session.title?.trim() || (main ? formatFolder(session.folder) : base);
+	const head = main ? name : `↳ ${name} (${session.agentType})`;
+	return `${head} · ${formatCompact(session.savedTokens)} saved`;
+}
+
+/** A cut keeps paying: every later request would have re-read it. One session's context with and without shaping. */
+export function Compounding({ stats, active, stale }: CompoundingProps) {
+	const { compounding } = stats;
+	const [picked, setPicked] = useState<string | null>(null);
+	const sessions = stats.sessionList;
+	// Default: the newest session that saved anything (the list is newest first).
+	const fallback = sessions.find(s => s.savedTokens > 0) ?? sessions[0];
+	const file = picked !== null && sessions.some(s => s.sessionFile === picked) ? picked : fallback?.sessionFile;
+
+	const context = useQuery(
+		["spotlight-session", file ?? null],
+		({ signal }) => getSpotlightSessionContext(file ?? "", signal),
+		{ enabled: active && file !== undefined },
+	);
+	const avoided = compounding.avoidedTokens - compounding.recoveredCarriedTokens;
+	// Saving-weighted mean of the requests each cut stayed out of.
+	const carry = stats.gross.savedTokens > 0 ? compounding.avoidedTokens / stats.gross.savedTokens : null;
+	const points = context.data?.points;
+	// The selected session's own share: what its requests would have carried without shaping, and didn't.
+	const sessionShare = useMemo(() => {
+		if (!points) return null;
+		let gap = 0;
+		let without = 0;
+		for (const p of points) {
+			gap += p.withoutTokens - p.contextTokens;
+			without += p.withoutTokens;
+		}
+		return without > 0 ? { share: gap / without, tokens: gap } : null;
+	}, [points]);
+
+	return (
+		<Card
+			index={3}
+			title="Compounding"
+			description="A cut is never re-read: each later request of the session carries a smaller context"
+			actions={
+				sessions.length > 0 && (
+					<select
+						className="input spot-session-select"
+						aria-label="Session"
+						value={file ?? ""}
+						onChange={e => setPicked(e.target.value)}
+					>
+						{sessions.map(s => (
+							<option key={s.sessionFile} value={s.sessionFile}>
+								{sessionLabel(s)}
+							</option>
+						))}
+					</select>
+				)
+			}
+			stale={stale}
+			footer="Context = input + cache read + cache write of each request. A compaction is a request whose context is under 60% of the previous one's; it ends the carry."
+		>
+			<div className="spot-compounding">
+				<div className="spot-compounding-chart">
+					{file === undefined ? (
+						<EmptyState title="No experiment sessions in this range" />
+					) : context.data === null ? (
+						context.error ? (
+							<ErrorState error={context.error} onRetry={context.refetch} />
+						) : (
+							<ChartSkeleton height={280} />
+						)
+					) : (
+						<ContextChart context={context.data} stale={context.stale} />
+					)}
+				</div>
+				<aside className="spot-compounding-side">
+					<div className="spot-eyebrow">Context avoided in this session</div>
+					<div className="spot-side-value">{formatShare(sessionShare?.share ?? null)}</div>
+					{points !== undefined && (
+						<p className="spot-side-copy">
+							{sessionShare === null
+								? "This session has no recorded requests."
+								: `of what its requests would have carried without Spotlight: ${formatSignedCompact(sessionShare.tokens)} prompt tokens, net of recoveries.`}
+						</p>
+					)}
+					{context.data && <SessionFacts session={context.data.session} />}
+					<p className="spot-side-range">
+						<span className="spot-side-range-value">{formatShare(compounding.avoidedShare)}</span> across all the
+						sessions shown: cuts kept {formatCompact(Math.max(0, avoided))} tokens out of later requests
+						{carry !== null && <>, each saved token re-sent {carry.toFixed(1)} times on average</>}.
+					</p>
+				</aside>
+			</div>
+		</Card>
+	);
+}
+
+function SessionFacts({ session }: { session: SpotlightSessionSummary }) {
+	return (
+		<dl className="spot-facts">
+			<div>
+				<dt>Agent</dt>
+				<dd className="row">
+					<Badge>{session.agentType}</Badge>
+				</dd>
+			</div>
+			<div>
+				<dt>Requests</dt>
+				<dd className="num">{formatInteger(session.requests)}</dd>
+			</div>
+			<div>
+				<dt>Saved</dt>
+				<dd className="num">{formatSignedCompact(session.savedTokens)}</dd>
+			</div>
+			<div>
+				<dt>Net $</dt>
+				<dd className="num">{formatSignedCost(session.netUsd)}</dd>
+			</div>
+		</dl>
+	);
+}
+
+const CONTEXT_LEGEND = [
+	{ key: "actual", label: "Context", color: "var(--spot-gross)" },
+	{ key: "ghost", label: "Without Spotlight", color: "var(--spot-ghost)" },
+	{ key: "compaction", label: "Compaction", color: "var(--line-4)" },
+];
+
+/** One session's context per request. The x axis is the request number (several can share a minute); the tooltip carries the time. */
+function ContextChart({ context, stale }: { context: SpotlightSessionContext; stale: boolean }) {
+	const { points } = context;
+	const chart = useMemo(() => {
+		const series: ChartSeries[] = [
+			{
+				key: "actual",
+				label: "Context",
+				color: "var(--spot-gross)",
+				kind: "area",
+				values: points.map(p => p.contextTokens),
+			},
+			{
+				key: "ghost",
+				label: "Avoided",
+				color: "var(--spot-ghost)",
+				kind: "area",
+				pattern: "hatch",
+				dashed: true,
+				values: points.map(p => Math.max(0, p.withoutTokens - p.contextTokens)),
+			},
+		];
+		const markers: SlotMarker[] = [];
+		points.forEach((p, i) => {
+			if (p.compacted) markers.push({ index: i });
+		});
+		return { series, markers };
+	}, [points]);
+
+	if (points.length === 0) {
+		return <EmptyState title="No requests recorded for this session" />;
+	}
+	return (
+		<div className="spot-context" data-stale={stale}>
+			<Legend items={CONTEXT_LEGEND} />
+			<Chart
+				slots={points.length}
+				series={chart.series}
+				markers={chart.markers}
+				height={280}
+				tickLabel={i => `#${i + 1}`}
+				tooltipTitle={i => `Request ${i + 1} · ${format(new Date(points[i].timestamp), "MMM d HH:mm:ss")}`}
+				formatTooltip={formatInteger}
+				showTotal={false}
+				tooltipExtra={i => {
+					const p = points[i];
+					return (
+						<div className="spot-tooltip-extra">
+							<div className="chart-tooltip-row chart-tooltip-total">
+								<span className="chart-tooltip-label">Without Spotlight</span>
+								<span className="chart-tooltip-value">{formatInteger(p.withoutTokens)}</span>
+							</div>
+							{p.saved > 0 && (
+								<div className="chart-tooltip-row">
+									<span className="chart-tooltip-label">Cut since last request</span>
+									<span className="chart-tooltip-value">{formatInteger(p.saved)}</span>
+								</div>
+							)}
+							{p.recovered > 0 && (
+								<div className="chart-tooltip-row">
+									<span className="chart-tooltip-label">Fetched back</span>
+									<span className="chart-tooltip-value">{formatInteger(p.recovered)}</span>
+								</div>
+							)}
+							{p.compacted && <div className="chart-tooltip-row dim">Compaction before this request</div>}
+						</div>
+					);
+				}}
+			/>
+		</div>
+	);
+}

package/src/client/spotlight/Coverage.tsx

@@ -0,0 +1,96 @@
+import { BarList, Legend, ShareBar } from "../charts";
+import { formatCompact, formatInteger } from "../data/formatters";
+import type { SpotlightDashboardStats } from "../types";
+import { Badge, Card } from "../ui";
+import { foldTools, formatShare } from "./meta";
+
+const SHAPED_COLOR = "var(--spot-applied)";
+const UNSHAPED_COLOR = "var(--spot-unshaped)";
+
+/** Which tools' output the models of experiment sessions saw, shaped and not: where shaping could reach next. */
+export function Coverage({ stats, stale }: { stats: SpotlightDashboardStats; stale: boolean }) {
+	const rows = [...stats.coverage].sort((a, b) => b.tokens - a.tokens);
+	const { listed, rest } = foldTools(rows);
+	let shaped = 0;
+	let unshaped = 0;
+	for (const row of rows) {
+		if (row.shaped) shaped += row.share;
+		else unshaped += row.share;
+	}
+	const items = listed.map(row => ({
+		key: row.tool,
+		label: (
+			<>
+				<span className="spot-coverage-tool">{row.tool}</span>
+				{row.shaped && <Badge>shaped</Badge>}
+				<span className="dim num">{formatInteger(row.calls)} calls</span>
+			</>
+		),
+		value: row.share,
+		display: `${formatShare(row.share)} · ${formatCompact(row.tokens)}`,
+		color: row.shaped ? SHAPED_COLOR : UNSHAPED_COLOR,
+	}));
+	if (rest.length > 0) {
+		let share = 0;
+		let tokens = 0;
+		let calls = 0;
+		for (const row of rest) {
+			share += row.share;
+			tokens += row.tokens;
+			calls += row.calls;
+		}
+		items.push({
+			key: "\u0000rest",
+			label: (
+				<>
+					<span className="spot-coverage-tool">
+						{formatInteger(rest.length)} more {rest.length === 1 ? "tool" : "tools"}
+					</span>
+					<span className="dim num">{formatInteger(calls)} calls</span>
+				</>
+			),
+			value: share,
+			display: `${formatShare(share)} · ${formatCompact(tokens)}`,
+			color: rest.some(row => row.shaped) ? SHAPED_COLOR : UNSHAPED_COLOR,
+		});
+	}
+
+	return (
+		<Card
+			index={5}
+			title="Not shaped yet"
+			description="Share of the tool output experiment sessions' models saw, by tool: the unshaped ones are where shaping could reach next"
+			actions={
+				<Legend
+					items={[
+						{ key: "shaped", label: "Shaped", color: SHAPED_COLOR },
+						{ key: "unshaped", label: "Not shaped", color: UNSHAPED_COLOR },
+					]}
+				/>
+			}
+			stale={stale}
+			footer="Tokens are tool-result chars ÷ 4 over the experiment's sessions in the range; a tool counts as shaped when the experiment evaluated any of its results in the range."
+		>
+			{rows.length === 0 ? (
+				<div className="empty">No tool output in this range.</div>
+			) : (
+				<div className="spot-coverage">
+					<div className="spot-coverage-summary">
+						<div className="spot-coverage-headline">
+							<span className="num">{formatShare(unshaped)}</span> of tool output comes from tools the experiment
+							did not shape in this range
+						</div>
+						<ShareBar
+							height={8}
+							segments={[
+								{ key: "shaped", label: "Shaped", value: shaped, color: SHAPED_COLOR },
+								{ key: "unshaped", label: "Not shaped", value: unshaped, color: UNSHAPED_COLOR },
+							]}
+						/>
+					</div>
+					<BarList items={items} />
+				</div>
+			)}
+		</Card>
+	);
+}

package/src/client/spotlight/Economics.tsx

@@ -0,0 +1,167 @@
+import { formatCompact, formatCost, formatInteger } from "../data/formatters";
+import type { SpotlightCutEconomics, SpotlightDashboardStats } from "../types";
+import { Card } from "../ui";
+import { foldTools, formatSignedCost, OTHER_TOOLS, toolIcon, toolsBySaving } from "./meta";
+
+export interface EconomicsProps {
+	stats: SpotlightDashboardStats;
+	stale: boolean;
+}
+
+/**
+ * Tint strength of the strongest cell; the rest scale with √|net| so the sign reads first, size second, and one
+ * outlier does not wash out every other cell.
+ */
+const MAX_TINT = 30;
+const MIN_TINT = 5;
+
+/** Narrowest tool column before the table scrolls sideways instead of squeezing its cells. */
+const TOOL_COLUMN_PX = 120;
+
+function cellTitle(cell: SpotlightCutEconomics): string {
+	return [
+		`${cell.tool}: ${formatInteger(cell.cuts)} cuts`,
+		`Saved ${formatCompact(cell.savedTokens)} tokens (${formatCost(cell.savedUsd)})`,
+		`Fetched back ${formatCompact(cell.recoveredTokens)} tokens (${formatCost(cell.recoveryUsd)})`,
+		`${formatInteger(cell.extraTurns)} extra turns (${formatCost(cell.turnUsd)})`,
+		`Net ${formatSignedCost(cell.netUsd)}`,
+	].join("\n");
+}
+
+/** The folded tools' cells summed into an `other` cell. */
+function otherCell(cells: readonly SpotlightCutEconomics[]): SpotlightCutEconomics | undefined {
+	if (cells.length === 0) return undefined;
+	const sum: SpotlightCutEconomics = {
+		tool: OTHER_TOOLS,
+		cuts: 0,
+		savedTokens: 0,
+		recoveredTokens: 0,
+		extraTurns: 0,
+		savedUsd: 0,
+		recoveryUsd: 0,
+		turnUsd: 0,
+		netUsd: 0,
+		extraTurnsPer100Cuts: 0,
+		netUsdPer100Cuts: 0,
+	};
+	for (const cell of cells) {
+		sum.cuts += cell.cuts;
+		sum.savedTokens += cell.savedTokens;
+		sum.recoveredTokens += cell.recoveredTokens;
+		sum.extraTurns += cell.extraTurns;
+		sum.savedUsd += cell.savedUsd;
+		sum.recoveryUsd += cell.recoveryUsd;
+		sum.turnUsd += cell.turnUsd;
+		sum.netUsd += cell.netUsd;
+	}
+	if (sum.cuts > 0) {
+		sum.extraTurnsPer100Cuts = (sum.extraTurns * 100) / sum.cuts;
+		sum.netUsdPer100Cuts = (sum.netUsd * 100) / sum.cuts;
+	}
+	return sum;
+}
+
+/**
+ * Per tool: what a hundred cuts earn after fetch-backs, extra turns and Spotlight's bill. Tools with cuts, the
+ * largest saving first; past the top ones the rest sum into one `other` column.
+ */
+export function Economics({ stats, stale }: EconomicsProps) {
+	const cut = new Set(stats.economics.filter(cell => cell.cuts > 0).map(cell => cell.tool));
+	const { listed, rest } = foldTools(toolsBySaving(stats).filter(tool => cut.has(tool)));
+	const folded = new Set(rest);
+	const cells = new Map(stats.economics.filter(cell => !folded.has(cell.tool)).map(cell => [cell.tool, cell]));
+	const other = otherCell(stats.economics.filter(cell => folded.has(cell.tool)));
+	if (other) cells.set(OTHER_TOOLS, other);
+	const tools = rest.length > 0 ? [...listed, OTHER_TOOLS] : listed;
+	let maxAbs = 0;
+	for (const cell of cells.values()) if (cell.cuts > 0) maxAbs = Math.max(maxAbs, Math.abs(cell.netUsdPer100Cuts));
+
+	return (
+		<Card
+			index={3}
+			title="What cuts earn, by tool"
+			description="Net dollars per 100 cuts after fetch-backs, extra turns and Spotlight's own bill"
+			actions={
+				maxAbs > 0 && (
+					<span className="spot-econ-scale" aria-hidden="true">
+						<span className="num">{formatSignedCost(-maxAbs)}</span>
+						<span className="spot-econ-ramp" />
+						<span className="num">{formatSignedCost(maxAbs)}</span>
+					</span>
+				)
+			}
+			stale={stale}
+			footer="Shortened results only; held-out ones save nothing. Every fetch-back counts against the cut here, the share held-out results show comes back anyway included, so this is the cautious view. An extra turn is a request whose every tool call fetched back output these cuts hid. Dollars price saved and fetched-back tokens as carried context, like Compounding."
+		>
+			{tools.length === 0 ? (
+				<div className="empty">No shortened results in this range.</div>
+			) : (
+				<div className="spot-econ-scroll">
+					<table className="spot-econ" style={{ minWidth: `${tools.length * TOOL_COLUMN_PX}px` }}>
+						<thead>
+							<tr>
+								{tools.map(tool => {
+									const Icon = toolIcon(tool);
+									return (
+										<th key={tool} scope="col" title={tool === OTHER_TOOLS ? rest.join(", ") : tool}>
+											<span className="row spot-econ-tool">
+												<Icon size={13} aria-hidden="true" />
+												<span className="spot-econ-tool-name">{tool}</span>
+											</span>
+										</th>
+									);
+								})}
+							</tr>
+						</thead>
+						<tbody>
+							<tr>
+								{tools.map(tool => {
+									const cell = cells.get(tool);
+									if (cell === undefined || cell.cuts === 0) {
+										return (
+											<td key={tool}>
+												<div className="spot-econ-cell" data-empty="true">
+													<span className="dim">No cuts</span>
+												</div>
+											</td>
+										);
+									}
+									const net = cell.netUsdPer100Cuts;
+									const tint =
+										maxAbs > 0 ? MIN_TINT + Math.sqrt(Math.abs(net) / maxAbs) * (MAX_TINT - MIN_TINT) : 0;
+									const tone = net < 0 ? "bad" : net > 0 ? "ok" : "neutral";
+									return (
+										<td key={tool}>
+											<div
+												className="spot-econ-cell"
+												data-tone={tone}
+												style={{ "--tint": `${tint}%` }}
+												title={cellTitle(cell)}
+											>
+												<span className="spot-econ-net num">{formatSignedCost(net)}</span>
+												<dl className="spot-econ-facts">
+													<div>
+														<dt>cuts</dt>
+														<dd className="num">{formatInteger(cell.cuts)}</dd>
+													</div>
+													<div>
+														<dt>saved</dt>
+														<dd className="num">{formatCompact(cell.savedTokens)}</dd>
+													</div>
+													<div>
+														<dt>extra turns /100</dt>
+														<dd className="num">{cell.extraTurnsPer100Cuts.toFixed(1)}</dd>
+													</div>
+												</dl>
+											</div>
+										</td>
+									);
+								})}
+							</tr>
+						</tbody>
+					</table>
+				</div>
+			)}
+		</Card>
+	);
+}