@oh-my-pi/pi-coding-agent 17.0.9 → 17.1.0

package/src/cli/bench-cli.ts

@@ -16,7 +16,7 @@ import type {
 import { resolveModelServiceTier, streamSimple } from "@oh-my-pi/pi-ai";
 import { buildModelProviderPriorityRank } from "@oh-my-pi/pi-catalog/identity";
 import { replaceTabs, truncateToWidth } from "@oh-my-pi/pi-tui";
-import { formatDuration, getProjectDir } from "@oh-my-pi/pi-utils";
+import { formatDuration, getProjectDir, prompt } from "@oh-my-pi/pi-utils";
 import chalk from "chalk";
 import type { ApiKeyResolverModel } from "../config/api-key-resolver";
 import { ModelRegistry } from "../config/model-registry";
@@ -28,6 +28,9 @@ import {
 } from "../config/model-resolver";
 import { buildServiceTierByFamily, serviceTierForAllFamilies, serviceTierSettingToTier } from "../config/service-tier";
 import { Settings } from "../config/settings";
+import cachePrefixTemplate from "../prompts/bench/cache-prefix.md" with { type: "text" };
+import cachePrefixChunk from "../prompts/bench/cache-prefix-chunk.md" with { type: "text" };
+import cacheSuffixTemplate from "../prompts/bench/cache-suffix.md" with { type: "text" };
 import benchPrompt from "../prompts/bench.md" with { type: "text" };
 import { discoverAuthStorage, loadCliExtensionProviders } from "../sdk";
 import {
@@ -40,8 +43,18 @@ import {
 const DEFAULT_RUNS = 10;
 const DEFAULT_PAR = 4;
 const DEFAULT_MAX_TOKENS = 512;
+const DEFAULT_CACHE_MAX_TOKENS = 64;
+const DEFAULT_CACHE_PREFIX_BYTES = 8_192;
+const DEFAULT_CACHE_PAIRS = 1;
+const DEFAULT_CACHE_CONCURRENCY = 1;
 const ERROR_WIDTH = 110;
 const BENCH_PROMPT = benchPrompt.trim();
+const UTF8_ENCODER = new TextEncoder();
+const UTF8_DECODER = new TextDecoder();
+const CACHE_PREFIX_CHUNK = cachePrefixChunk;
+const CACHE_PREFIX_PLACEHOLDER = "__OMP_CACHE_BENCH_RAW_PREFIX__";
+const CACHE_PREFIX_CHUNK_BYTES = UTF8_ENCODER.encode(CACHE_PREFIX_CHUNK).byteLength;
+const RESPONSE_CACHE_STATUS_HEADERS = ["cf-aig-cache-status"] as const;
 
 export interface BenchCommandArgs {
 	models: string[];
@@ -53,6 +66,11 @@ export interface BenchCommandArgs {
 		serviceTier?: string;
 		json?: boolean;
 		par?: number;
+		cache?: boolean;
+		cachePrefixFile?: string;
+		cachePrefixBytes?: number;
+		cachePairs?: number;
+		cacheConcurrency?: number;
 	};
 }
 
@@ -85,6 +103,44 @@ export interface BenchRunFailure {
 
 export type BenchRunResult = BenchRunSuccess | BenchRunFailure;
 
+export type CacheObservation =
+	| "prompt_cache_read_observed"
+	| "prompt_cache_write_observed"
+	| "response_cache_hit_observed"
+	| "no_provider_proof";
+
+export interface BenchCacheUsage {
+	inputTokens: number;
+	outputTokens: number;
+	cacheReadTokens: number;
+	cacheWriteTokens: number;
+	totalTokens: number;
+	cost: number;
+}
+
+export interface BenchCacheRunReport {
+	phase: "cold" | "warm";
+	result: BenchRunResult;
+	usage?: BenchCacheUsage;
+	requestIdObserved: boolean;
+	observations: CacheObservation[];
+}
+
+export interface BenchCachePairReport {
+	cold: BenchCacheRunReport;
+	warm: BenchCacheRunReport;
+	/** The nominal cold request showed cache reuse, so it is not a true cold baseline. */
+	coldAlreadyWarm: boolean;
+	/** Structural-only comparisons: prompt text and cache keys are never emitted. */
+	stablePrefix: true;
+	suffixChanged: true;
+	promptCacheKeyStable: true;
+	statefulResponsesDisabled: true;
+	freshProviderSessionState: true;
+	/** "unavailable" when a transport does not expose the provider payload locally. */
+	payloadStructureStable: boolean | "unavailable";
+}
+
 export interface BenchAverages {
 	ttftMs: number;
 	durationMs: number;
@@ -102,6 +158,7 @@ export interface BenchModelReport {
 	results: BenchRunResult[];
 	/** Averages over successful runs; null when every run failed. */
 	average: BenchAverages | null;
+	cachePairs?: BenchCachePairReport[];
 }
 
 export interface BenchSummary {
@@ -111,6 +168,10 @@ export interface BenchSummary {
 	failures: number;
 	/** Requested per-family service tiers, resolved per model before reaching the wire. */
 	serviceTierByFamily?: ServiceTierByFamily;
+	cache?: {
+		pairs: number;
+		concurrency: number;
+	};
 }
 
 type BenchStreamSimple = (
@@ -127,6 +188,7 @@ export interface BenchDependencies {
 	setExitCode?: (code: number) => void;
 	streamSimple?: BenchStreamSimple;
 	now?: () => number;
+	readTextFile?: (path: string, maxBytes: number) => Promise<string>;
 	stdoutIsTTY?: boolean;
 }
 
@@ -184,10 +246,176 @@ function hasVisibleFinalContent(message: AssistantMessage): boolean {
 	});
 }
 
+interface CacheRequestCapture {
+	payloadStructure?: string;
+	requestIdObserved: boolean;
+	responseCacheHit: boolean;
+	usage?: BenchCacheUsage;
+}
+
+function payloadStructure(payload: unknown): string {
+	if (payload === null) return "null";
+	if (Array.isArray(payload)) return `[${payload.map(payloadStructure).join(",")}]`;
+	if (typeof payload === "object") {
+		const record = payload as Record<string, unknown>;
+		return `{${Object.keys(record)
+			.sort()
+			.map(key => `${key}:${payloadStructure(record[key])}`)
+			.join(",")}}`;
+	}
+	return typeof payload;
+}
+
+function asNonNegativeNumber(value: unknown): number {
+	return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
+}
+
+function captureUsage(message: AssistantMessage): BenchCacheUsage {
+	const usage = message.usage;
+	return {
+		inputTokens: asNonNegativeNumber(usage.input),
+		outputTokens: asNonNegativeNumber(usage.output),
+		cacheReadTokens: asNonNegativeNumber(usage.cacheRead),
+		cacheWriteTokens: asNonNegativeNumber(usage.cacheWrite),
+		totalTokens: asNonNegativeNumber(usage.totalTokens),
+		cost: asNonNegativeNumber(usage.cost?.total),
+	};
+}
+
+function cacheObservations(capture: CacheRequestCapture): CacheObservation[] {
+	const observations: CacheObservation[] = [];
+	if ((capture.usage?.cacheReadTokens ?? 0) > 0) observations.push("prompt_cache_read_observed");
+	if ((capture.usage?.cacheWriteTokens ?? 0) > 0) observations.push("prompt_cache_write_observed");
+	if (capture.responseCacheHit) observations.push("response_cache_hit_observed");
+	return observations.length > 0 ? observations : ["no_provider_proof"];
+}
+
+function cacheRunReport(
+	phase: BenchCacheRunReport["phase"],
+	result: BenchRunResult,
+	capture: CacheRequestCapture,
+): BenchCacheRunReport {
+	return {
+		phase,
+		result,
+		usage: capture.usage,
+		requestIdObserved: capture.requestIdObserved,
+		observations: cacheObservations(capture),
+	};
+}
+
+function truncateUtf8ByteLength(bytes: Uint8Array, maxBytes: number): number {
+	const end = Math.min(bytes.byteLength, maxBytes);
+	if (end === 0) return 0;
+
+	let sequenceStart = end - 1;
+	while (sequenceStart > 0 && (bytes[sequenceStart]! & 0b1100_0000) === 0b1000_0000) sequenceStart--;
+
+	const leadingByte = bytes[sequenceStart]!;
+	const sequenceLength =
+		leadingByte <= 0b0111_1111
+			? 1
+			: leadingByte >= 0b1100_0010 && leadingByte <= 0b1101_1111
+				? 2
+				: leadingByte >= 0b1110_0000 && leadingByte <= 0b1110_1111
+					? 3
+					: leadingByte >= 0b1111_0000 && leadingByte <= 0b1111_0100
+						? 4
+						: 1;
+	return sequenceLength > end - sequenceStart ? sequenceStart : end;
+}
+
+async function readBoundedUtf8File(path: string, maxBytes: number): Promise<string> {
+	const file = Bun.file(path);
+	const bytes = new Uint8Array(await file.slice(0, maxBytes).arrayBuffer());
+	const end = truncateUtf8ByteLength(bytes, maxBytes);
+	return UTF8_DECODER.decode(bytes.subarray(0, end));
+}
+
+function truncateUtf8(text: string, maxBytes: number): string {
+	const bytes = UTF8_ENCODER.encode(text);
+	const end = truncateUtf8ByteLength(bytes, maxBytes);
+	return end === bytes.byteLength ? text : UTF8_DECODER.decode(bytes.subarray(0, end));
+}
+
+function generatedCachePrefix(bytes: number): string {
+	return truncateUtf8(CACHE_PREFIX_CHUNK.repeat(Math.ceil(bytes / CACHE_PREFIX_CHUNK_BYTES)), bytes);
+}
+
+function renderCacheBenchmarkPrefix(prefix: string, namespace: string): string {
+	const rendered = prompt.render(cachePrefixTemplate, {
+		prefix: CACHE_PREFIX_PLACEHOLDER,
+		namespace,
+	});
+	if (!rendered.includes(CACHE_PREFIX_PLACEHOLDER)) {
+		throw new Error("Cache benchmark prefix template is missing its raw prefix placeholder");
+	}
+	// Render the static wrapper first, then inject caller bytes so prompt
+	// normalization cannot trim spaces or collapse blank lines in prefix files.
+	// Function replacer: a prefix containing `$&`/`$'`/`` $` `` must not be
+	// expanded as a string-replacement pattern.
+	return rendered.replace(CACHE_PREFIX_PLACEHOLDER, () => prefix);
+}
+
+async function resolveCachePrefix(
+	flags: BenchCommandArgs["flags"],
+	readTextFile: (path: string, maxBytes: number) => Promise<string>,
+): Promise<string> {
+	const bytes = normalizePositiveInteger("cache-prefix-bytes", flags.cachePrefixBytes, DEFAULT_CACHE_PREFIX_BYTES);
+	const prefix = flags.cachePrefixFile
+		? await readTextFile(flags.cachePrefixFile, bytes)
+		: generatedCachePrefix(bytes);
+	return truncateUtf8(prefix, bytes);
+}
+
+function cacheBenchmarkMessages(stablePrefix: string, suffix: string): Context["messages"] {
+	const timestamp = Date.now();
+	return [
+		{ role: "user", content: stablePrefix, timestamp, attribution: "user" },
+		{ role: "user", content: suffix, timestamp, attribution: "user" },
+	];
+}
+
+async function runWithConcurrency<T>(
+	count: number,
+	concurrency: number,
+	run: (index: number) => Promise<T>,
+): Promise<T[]> {
+	const results = new Array<T>(count);
+	let next = 0;
+	const worker = async (): Promise<void> => {
+		while (next < count) {
+			const index = next++;
+			results[index] = await run(index);
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(count, concurrency) }, worker));
+	return results;
+}
+
+function formatCacheCost(cost: number): string {
+	if (cost < 0.01) return `$${cost.toFixed(4)}`;
+	if (cost < 1) return `$${cost.toFixed(3)}`;
+	return `$${cost.toFixed(2)}`;
+}
+
+function formatCachePairLine(pair: BenchCachePairReport, index: number, total: number): string {
+	const formatPhase = (run: BenchCacheRunReport, alreadyWarm = false) => {
+		if (!run.result.ok) {
+			return `${run.phase} failed: ${truncateToWidth(replaceTabs(run.result.error), ERROR_WIDTH)}`;
+		}
+		const usage = run.usage;
+		return `${run.phase}${alreadyWarm ? " (already warm)" : ""} ${run.observations.join(", ")} ${chalk.dim("input")} ${usage?.inputTokens ?? 0} ${chalk.dim("cache-read")} ${usage?.cacheReadTokens ?? 0} ${chalk.dim("cache-write")} ${usage?.cacheWriteTokens ?? 0} ${chalk.dim("output")} ${usage?.outputTokens ?? run.result.outputTokens} ${chalk.dim("total")} ${usage?.totalTokens ?? 0} ${chalk.dim("cost")} ${formatCacheCost(usage?.cost ?? 0)} ${chalk.dim("TTFT")} ${formatMs(run.result.ttftMs)} ${chalk.dim("duration")} ${formatMs(run.result.durationMs)} ${chalk.dim("throughput")} ${run.result.tokensPerSecond.toFixed(1)}/s`;
+	};
+	return `  ${chalk.dim(`pair ${index + 1}/${total}`)} ${formatPhase(pair.cold, pair.coldAlreadyWarm)}; ${formatPhase(pair.warm)}`;
+}
+
 interface BenchRequestOptions {
 	apiKey: ApiKeyResolver;
 	sessionId: string;
 	prompt: string;
+	/** Native OMP messages; cache mode splits the stable prefix from the suffix. */
+	contextMessages?: Context["messages"];
 	maxTokens: number;
 	/** Explicit effort from a `:level` selector suffix; absent = provider default. */
 	reasoning?: Effort;
@@ -195,6 +423,9 @@ interface BenchRequestOptions {
 	disableReasoning?: boolean;
 	/** Requested service tier passed to `streamSimple`; absent omits the option. The provider layer applies scope/support gating before it reaches the wire. */
 	serviceTier?: ServiceTier;
+	promptCacheKey?: string;
+	statefulResponses?: false;
+	cacheCapture?: CacheRequestCapture;
 }
 
 async function runBenchRequest(
@@ -211,7 +442,9 @@ async function runBenchRequest(
 			// Codex's Responses endpoint 400s with "Instructions are required" when no
 			// system prompt is present — same guard as eval's completion bridge.
 			systemPrompt: ["You are a helpful assistant."],
-			messages: [{ role: "user", content: options.prompt, timestamp: Date.now(), attribution: "user" }],
+			messages: options.contextMessages ?? [
+				{ role: "user", content: options.prompt, timestamp: Date.now(), attribution: "user" },
+			],
 		};
 		const stream = streamFn(model, context, {
 			apiKey: options.apiKey,
@@ -221,6 +454,22 @@ async function runBenchRequest(
 					? Math.min(options.maxTokens, model.maxTokens)
 					: options.maxTokens,
 			reasoning: options.reasoning,
+			promptCacheKey: options.promptCacheKey,
+			statefulResponses: options.statefulResponses,
+			onPayload: options.cacheCapture
+				? payload => {
+						options.cacheCapture!.payloadStructure = payloadStructure(payload);
+						return undefined;
+					}
+				: undefined,
+			onResponse: options.cacheCapture
+				? response => {
+						options.cacheCapture!.requestIdObserved = Boolean(response.requestId);
+						options.cacheCapture!.responseCacheHit = RESPONSE_CACHE_STATUS_HEADERS.some(
+							header => response.headers[header]?.trim().toLowerCase() === "hit",
+						);
+					}
+				: undefined,
 			disableReasoning: options.disableReasoning,
 			serviceTier: options.serviceTier,
 			providerSessionState,
@@ -265,6 +514,7 @@ async function runBenchRequest(
 				error: `provider returned no output (0 tokens, empty stream; stop reason: ${message.stopReason ?? "unknown"})`,
 			};
 		}
+		if (options.cacheCapture) options.cacheCapture.usage = captureUsage(message);
 		return {
 			ok: true,
 			ttftMs,
@@ -480,14 +730,48 @@ function resolveBenchModels(
 	}
 	return resolved;
 }
+function assertCacheModeSupported(targets: BenchTarget[]): void {
+	if (targets.some(({ model }) => model.api === "openai-codex-responses")) {
+		throw new Error(
+			"--cache is not supported for openai-codex-responses because Codex WebSocket chaining cannot produce independent prompt-cache pairs",
+		);
+	}
+}
+
 export async function runBenchCommand(command: BenchCommandArgs, deps: BenchDependencies = {}): Promise<BenchSummary> {
-	const runs = normalizePositiveInteger("runs", command.flags.runs, DEFAULT_RUNS);
-	const maxTokens = normalizePositiveInteger("max-tokens", command.flags.maxTokens, DEFAULT_MAX_TOKENS);
+	const cacheMode = command.flags.cache === true;
+	const cacheFlagsUsed =
+		command.flags.cachePrefixFile !== undefined ||
+		command.flags.cachePrefixBytes !== undefined ||
+		command.flags.cachePairs !== undefined ||
+		command.flags.cacheConcurrency !== undefined;
+	if (!cacheMode && cacheFlagsUsed) throw new Error("Cache flags require --cache");
+	if (cacheMode && command.flags.runs !== undefined)
+		throw new Error("Use --cache-pairs instead of --runs with --cache");
+	if (cacheMode && command.flags.prompt !== undefined) throw new Error("--cache builds its own stable-prefix prompts");
+	if (cacheMode && (command.flags.par ?? 1) > 1) {
+		throw new Error("--par cannot parallelize cold/warm pairs; use --cache-concurrency instead");
+	}
+
+	const cachePairs = cacheMode
+		? normalizePositiveInteger("cache-pairs", command.flags.cachePairs, DEFAULT_CACHE_PAIRS)
+		: undefined;
+	const cacheConcurrency = cacheMode
+		? normalizePositiveInteger("cache-concurrency", command.flags.cacheConcurrency, DEFAULT_CACHE_CONCURRENCY)
+		: undefined;
+	const runs = cacheMode ? cachePairs! * 2 : normalizePositiveInteger("runs", command.flags.runs, DEFAULT_RUNS);
+	const maxTokens = normalizePositiveInteger(
+		"max-tokens",
+		command.flags.maxTokens,
+		cacheMode ? DEFAULT_CACHE_MAX_TOKENS : DEFAULT_MAX_TOKENS,
+	);
 	const par =
 		command.flags.par !== undefined ? normalizePositiveInteger("par", command.flags.par, DEFAULT_PAR) : DEFAULT_PAR;
-	const prompt = command.flags.prompt?.trim() || BENCH_PROMPT;
+	const benchmarkPrompt = command.flags.prompt?.trim() || BENCH_PROMPT;
 	const json = command.flags.json === true;
 	const randomSessionId = deps.randomSessionId ?? (() => Bun.randomUUIDv7());
+	const readTextFile = deps.readTextFile ?? readBoundedUtf8File;
+	const cachePrefix = cacheMode ? await resolveCachePrefix(command.flags, readTextFile) : undefined;
 	const writeStdout = deps.writeStdout ?? ((text: string) => process.stdout.write(text));
 	const writeStderr = deps.writeStderr ?? ((text: string) => process.stderr.write(text));
 	const setExitCode =
@@ -505,6 +789,7 @@ export async function runBenchCommand(command: BenchCommandArgs, deps: BenchDepe
 	const runtime = await (deps.createRuntime ?? createDefaultRuntime)();
 	try {
 		const targets = resolveBenchModels(command.models, runtime.modelRegistry, runtime.settings, writeStderr);
+		if (cacheMode) assertCacheModeSupported(targets);
 		// Explicit `--service-tier` (a single value broadcast across families) wins;
 		// otherwise fall back to the configured per-family `tier.*` settings. Each
 		// model resolves its own family's tier below before reaching the wire.
@@ -538,14 +823,103 @@ export async function runBenchCommand(command: BenchCommandArgs, deps: BenchDepe
 				};
 				results.push(failure);
 				if (!json) writeStdout(`${formatRunLine(failure, 0, runs)}\n`);
-				reports.push(buildModelReport(selector, model, thinking, results));
+				const report = buildModelReport(selector, model, thinking, results);
+				if (cacheMode) report.cachePairs = [];
+				reports.push(report);
+				continue;
+			}
+
+			const serviceTier = resolveModelServiceTier(serviceTierByFamily, model);
+			if (cacheMode) {
+				const pairs = await runWithConcurrency(
+					cachePairs!,
+					cacheConcurrency!,
+					async (pairIndex): Promise<BenchCachePairReport> => {
+						const cacheNamespace = randomSessionId();
+						const promptCacheKey = `bench-cache:${cacheNamespace}`;
+						const stablePrefix = renderCacheBenchmarkPrefix(cachePrefix!, cacheNamespace);
+						const coldSuffix = prompt.render(cacheSuffixTemplate, { variant: "A" }).trim();
+						const warmSuffix = prompt.render(cacheSuffixTemplate, { variant: "B" }).trim();
+						const coldCapture: CacheRequestCapture = {
+							requestIdObserved: false,
+							responseCacheHit: false,
+						};
+						// Keep the gateway's credential selection stable for both phases.
+						// Provider-session state is still recreated by runBenchRequest and
+						// stateful Responses chaining is disabled below.
+						const credentialAffinitySessionId = pairIndex === 0 ? testSessionId : randomSessionId();
+						const credentialResolver = runtime.modelRegistry.resolver(model, credentialAffinitySessionId);
+						const coldResult = await runBenchRequest(
+							model,
+							{
+								apiKey: credentialResolver,
+								sessionId: credentialAffinitySessionId,
+								prompt: coldSuffix,
+								contextMessages: cacheBenchmarkMessages(stablePrefix, coldSuffix),
+								maxTokens,
+								reasoning: toReasoningEffort(thinking),
+								disableReasoning: shouldDisableReasoning(thinking) ? true : undefined,
+								serviceTier,
+								promptCacheKey,
+								statefulResponses: false,
+								cacheCapture: coldCapture,
+							},
+							streamFn,
+							now,
+						);
+						const warmCapture: CacheRequestCapture = {
+							requestIdObserved: false,
+							responseCacheHit: false,
+						};
+						const warmResult = await runBenchRequest(
+							model,
+							{
+								apiKey: credentialResolver,
+								sessionId: credentialAffinitySessionId,
+								prompt: warmSuffix,
+								contextMessages: cacheBenchmarkMessages(stablePrefix, warmSuffix),
+								maxTokens,
+								reasoning: toReasoningEffort(thinking),
+								disableReasoning: shouldDisableReasoning(thinking) ? true : undefined,
+								serviceTier,
+								promptCacheKey,
+								statefulResponses: false,
+								cacheCapture: warmCapture,
+							},
+							streamFn,
+							now,
+						);
+						return {
+							cold: cacheRunReport("cold", coldResult, coldCapture),
+							warm: cacheRunReport("warm", warmResult, warmCapture),
+							coldAlreadyWarm: (coldCapture.usage?.cacheReadTokens ?? 0) > 0 || coldCapture.responseCacheHit,
+							stablePrefix: true,
+							suffixChanged: true,
+							promptCacheKeyStable: true,
+							statefulResponsesDisabled: true,
+							freshProviderSessionState: true,
+							payloadStructureStable:
+								coldCapture.payloadStructure === undefined || warmCapture.payloadStructure === undefined
+									? "unavailable"
+									: coldCapture.payloadStructure === warmCapture.payloadStructure,
+						};
+					},
+				);
+				for (const pair of pairs) results.push(pair.cold.result, pair.warm.result);
+				const report = buildModelReport(selector, model, thinking, results);
+				report.cachePairs = pairs;
+				reports.push(report);
+				if (!json) {
+					for (const [index, pair] of pairs.entries()) {
+						writeStdout(`${formatCachePairLine(pair, index, pairs.length)}\n`);
+					}
+				}
 				continue;
 			}
 
 			// We will launch up to `par` workers/requests concurrently.
 			// To keep output clean, non-JSON output emits entries in correct index order.
 			let nextToPrint = 0;
-
 			const runWorker = async (index: number) => {
 				const sessionId = index === 0 ? testSessionId : randomSessionId();
 				const result = await runBenchRequest(
@@ -553,56 +927,48 @@ export async function runBenchCommand(command: BenchCommandArgs, deps: BenchDepe
 					{
 						apiKey: runtime.modelRegistry.resolver(model, sessionId),
 						sessionId,
-						prompt,
+						prompt: benchmarkPrompt,
 						maxTokens,
 						reasoning: toReasoningEffort(thinking),
 						disableReasoning: shouldDisableReasoning(thinking) ? true : undefined,
-						serviceTier: resolveModelServiceTier(serviceTierByFamily, model),
+						serviceTier,
 					},
 					streamFn,
 					now,
 				);
 				results[index] = result;
 			};
-
-			// Concurrency-limited running pool
 			const queue = Array.from({ length: runs }, (_, i) => i);
 			const activeWorkers: Promise<void>[] = [];
-
 			const processNext = async (): Promise<void> => {
 				if (queue.length === 0) return;
 				const index = queue.shift()!;
-
-				// Pre-print a status update if requested and interactive
-				if (!json && interactive) {
-					writeStdout(chalk.dim(`  … run ${index + 1}/${runs} streaming\n`));
-				}
-
+				if (!json && interactive) writeStdout(chalk.dim(`  … run ${index + 1}/${runs} streaming\n`));
 				await runWorker(index);
-
-				// Attempt to print completed results that are in-order
 				if (!json) {
 					while (nextToPrint < runs && results[nextToPrint] !== undefined) {
-						const res = results[nextToPrint];
-						writeStdout(`${formatRunLine(res, nextToPrint, runs)}\n`);
+						writeStdout(`${formatRunLine(results[nextToPrint], nextToPrint, runs)}\n`);
 						nextToPrint++;
 					}
 				}
-
 				await processNext();
 			};
-
-			for (let w = 0; w < Math.min(par, runs); w++) {
-				activeWorkers.push(processNext());
-			}
+			for (let worker = 0; worker < Math.min(par, runs); worker++) activeWorkers.push(processNext());
 			await Promise.all(activeWorkers);
 			reports.push(buildModelReport(selector, model, thinking, results));
 		}
 		const failures = reports.reduce((sum, report) => sum + report.results.filter(result => !result.ok).length, 0);
-		const summary: BenchSummary = { runs, maxTokens, models: reports, failures, serviceTierByFamily };
+		const summary: BenchSummary = {
+			runs,
+			maxTokens,
+			models: reports,
+			failures,
+			serviceTierByFamily,
+			...(cacheMode ? { cache: { pairs: cachePairs!, concurrency: cacheConcurrency! } } : {}),
+		};
 		if (json) {
 			writeStdout(`${JSON.stringify(summary, null, 2)}\n`);
-		} else if (reports.length > 1 || runs > 1) {
+		} else if (!cacheMode && (reports.length > 1 || runs > 1)) {
 			writeStdout(`\n${formatBenchTable(summary)}`);
 		}
 		if (failures > 0) setExitCode(1);

package/src/cli/flag-tables.ts

@@ -117,6 +117,9 @@ export const STRING_SETTERS: Record<string, StringSetter> = {
 	"--config": (result, value) => {
 		result.config = [...(result.config ?? []), value];
 	},
+	"--add-dir": (result, value) => {
+		result.addDir = [...(result.addDir ?? []), value];
+	},
 	"--mode": (result, value) => {
 		if (value === "text" || value === "json" || value === "rpc" || value === "acp" || value === "rpc-ui") {
 			result.mode = value;

package/src/cli/gc-cli.ts

@@ -5,10 +5,10 @@ import { gunzipSync, gzipSync } from "node:zlib";
 import { getAgentDir, getBlobsDir, getHistoryDbPath, getModelDbPath, getSessionsDir } from "@oh-my-pi/pi-utils";
 import { Settings } from "../config/settings";
 import { getDefault } from "../config/settings-schema";
+import { BLOB_HASH_RE } from "../session/blob-store";
 import { listSessionsReadOnly, type SessionInfo, type SessionStatus } from "../session/session-listing";
 import { FileSessionStorage } from "../session/session-storage";
 
-const HASH_RE = /^[a-f0-9]{64}$/;
 const BLOB_FILE_RE = /^([a-f0-9]{64})(?:\.[A-Za-z0-9][A-Za-z0-9._-]{0,31})?$/;
 const BLOB_REF_RE = /\bblob:sha256:([a-f0-9]{64})\b/gi;
 const JSONL_GLOB = new Bun.Glob("**/*.jsonl");
@@ -268,7 +268,7 @@ async function collectReferencedBlobHashes(sessionRoots: string[]): Promise<Set<
 			const text = await readTextIfPresent(file);
 			for (const match of text.matchAll(BLOB_REF_RE)) {
 				const hash = match[1]?.toLowerCase();
-				if (hash && HASH_RE.test(hash)) hashes.add(hash);
+				if (hash && BLOB_HASH_RE.test(hash)) hashes.add(hash);
 			}
 		}
 	}

package/src/cli/usage-cli.ts

@@ -208,14 +208,23 @@ function describeAmount(limit: UsageLimit): string {
 	const amount = limit.amount;
 	const parts: string[] = [];
 	const absoluteUnit = amount.unit !== "percent" && amount.unit !== "unknown";
+	const fraction = resolveUsedFraction(limit);
 	if (absoluteUnit && amount.used !== undefined && amount.limit !== undefined) {
 		parts.push(
 			`${formatUnitValue(amount.used, amount.unit)} / ${formatUnitValue(amount.limit, amount.unit)}${UNIT_SUFFIX[amount.unit]}`,
 		);
 	} else if (absoluteUnit && amount.remaining !== undefined) {
 		parts.push(`${formatUnitValue(amount.remaining, amount.unit)}${UNIT_SUFFIX[amount.unit]} left`);
+	} else if (
+		absoluteUnit &&
+		amount.used !== undefined &&
+		Number.isFinite(amount.used) &&
+		amount.limit === undefined &&
+		amount.remaining === undefined &&
+		fraction === undefined
+	) {
+		parts.push(`${formatUnitValue(amount.used, amount.unit)}${UNIT_SUFFIX[amount.unit]} used`);
 	}
-	const fraction = resolveUsedFraction(limit);
 	if (fraction !== undefined) {
 		parts.push(`${(fraction * 100).toFixed(1)}% used`);
 	} else if (amount.remainingFraction !== undefined) {

package/src/collab/host.ts

@@ -614,10 +614,11 @@ export class CollabHost {
 			case "kill": {
 				const kill = async () => {
 					const ref = AgentRegistry.global().get(agentId);
-					if (ref && ref.status === "running" && ref.session) {
+					if (!ref) return;
+					if (ref.status === "running" && ref.session) {
 						await ref.session.abort({ reason: USER_INTERRUPT_LABEL });
 					}
-					await AgentLifecycleManager.global().release(agentId);
+					await AgentLifecycleManager.global().release(agentId, ref);
 				};
 				kill().catch(fail);
 				break;