@danypops/pi-eval-harness 0.2.0 → 0.3.0

package/README.md

@@ -48,6 +48,20 @@ would only hide the assertion, not simplify it.
 A `Checker` that only ever gets exercised against a real, expensive `pi-process-harness` run has
 no fast, deterministic proof it is correct in isolation -- write the fixture test first.
 
+## Real live-LLM smoke test (opt-in, real cost -- never run automatically)
+
+`scripts/real-llm-smoke-test.ts` spawns a genuine `pi` process against a real model (no faux
+provider, no scripted tool calls) and runs the captured trace through this package's own
+matching/rollup functions -- confirming they handle a real model's own event shape, not just the
+faux provider's conveniences every other test here relies on. Run explicitly:
+
+```sh
+bun scripts/real-llm-smoke-test.ts
+```
+
+Requires real provider credentials already configured in your own ambient Pi profile (this
+script deliberately runs with `isolatedHome: false`) and incurs a real, billed API call.
+
 ## License
 
 MIT

package/dist/ablation.d.ts

@@ -17,6 +17,8 @@ export interface AblationDelta {
     readonly meanScoreDelta: number;
     readonly meanDurationMsDelta: number;
     readonly meanTokensInDelta: number;
+    readonly meanCacheReadTokensDelta: number;
+    readonly meanCacheWriteTokensDelta: number;
 }
 /** One config's own trial metrics, plus its delta vs baseline. `delta` is undefined for the baseline itself. */
 export interface AblationResult {

package/dist/ablation.js

@@ -32,6 +32,8 @@ export async function ablate(configs, n, options = {}) {
                 meanScoreDelta: metrics.meanScore - baseline.meanScore,
                 meanDurationMsDelta: metrics.meanDurationMs - baseline.meanDurationMs,
                 meanTokensInDelta: metrics.meanTokensIn - baseline.meanTokensIn,
+                meanCacheReadTokensDelta: metrics.meanCacheReadTokens - baseline.meanCacheReadTokens,
+                meanCacheWriteTokensDelta: metrics.meanCacheWriteTokens - baseline.meanCacheWriteTokens,
             },
         };
     });
@@ -45,10 +47,10 @@ export function formatAblation(label, results) {
     const lines = [`=== Ablation: ${label} ===`];
     for (const result of results) {
         const m = result.metrics;
-        lines.push(`  ${result.config.name.padEnd(14)}  pass_rate=${m.passRate.toFixed(2)}  mean_score=${m.meanScore.toFixed(2)}  mean_duration_ms=${m.meanDurationMs.toFixed(0)}  tokens_in=${m.meanTokensIn.toFixed(0)}`);
+        lines.push(`  ${result.config.name.padEnd(14)}  pass_rate=${m.passRate.toFixed(2)}  mean_score=${m.meanScore.toFixed(2)}  mean_duration_ms=${m.meanDurationMs.toFixed(0)}  tokens_in=${m.meanTokensIn.toFixed(0)}  cache_read=${m.meanCacheReadTokens.toFixed(0)}  cache_write=${m.meanCacheWriteTokens.toFixed(0)}`);
         if (result.delta) {
             const d = result.delta;
-            lines.push(`  ${"(vs baseline)".padEnd(14)}  Δpass_rate=${signed(d.passRateDelta, 2)}  Δscore=${signed(d.meanScoreDelta, 2)}  Δduration_ms=${signed(d.meanDurationMsDelta, 0)}  Δtokens_in=${signed(d.meanTokensInDelta, 0)}`);
+            lines.push(`  ${"(vs baseline)".padEnd(14)}  Δpass_rate=${signed(d.passRateDelta, 2)}  Δscore=${signed(d.meanScoreDelta, 2)}  Δduration_ms=${signed(d.meanDurationMsDelta, 0)}  Δtokens_in=${signed(d.meanTokensInDelta, 0)}  Δcache_read=${signed(d.meanCacheReadTokensDelta, 0)}  Δcache_write=${signed(d.meanCacheWriteTokensDelta, 0)}`);
         }
     }
     return lines.join("\n");

package/dist/trials.d.ts

@@ -11,6 +11,9 @@ export interface TrialResult {
     readonly durationMs: number;
     readonly tokensIn: number;
     readonly tokensOut: number;
+    /** Real cache-read/cache-write tokens -- under prompt caching, the dominant real component of total context size; tokensIn alone (a provider's own incremental, non-cached count) understates it. */
+    readonly cacheReadTokens: number;
+    readonly cacheWriteTokens: number;
     readonly costUsd: number;
     readonly error?: string;
 }
@@ -26,6 +29,8 @@ export interface TrialMetrics {
     readonly meanDurationMs: number;
     readonly meanTokensIn: number;
     readonly meanTokensOut: number;
+    readonly meanCacheReadTokens: number;
+    readonly meanCacheWriteTokens: number;
     readonly meanCostUsd: number;
 }
 /**

package/dist/trials.js

@@ -55,6 +55,8 @@ export function aggregateTrials(results, options = {}) {
         meanDurationMs: mean(results.map((result) => result.durationMs)),
         meanTokensIn: mean(results.map((result) => result.tokensIn)),
         meanTokensOut: mean(results.map((result) => result.tokensOut)),
+        meanCacheReadTokens: mean(results.map((result) => result.cacheReadTokens)),
+        meanCacheWriteTokens: mean(results.map((result) => result.cacheWriteTokens)),
         meanCostUsd: mean(results.map((result) => result.costUsd)),
     };
 }
@@ -80,6 +82,8 @@ export async function runTrials(runOne, n, options = {}) {
                     durationMs: 0,
                     tokensIn: 0,
                     tokensOut: 0,
+                    cacheReadTokens: 0,
+                    cacheWriteTokens: 0,
                     costUsd: 0,
                     error: error instanceof Error ? error.message : String(error),
                 };

package/dist/turns.d.ts

@@ -14,6 +14,8 @@ export interface Turn {
     readonly tokensIn: number;
     readonly tokensOut: number;
     readonly cacheReadTokens: number;
+    /** New context tokens written to the provider's cache this turn -- the other real component of total context size under prompt caching, alongside cacheReadTokens. */
+    readonly cacheWriteTokens: number;
     /** Real cost in USD from Usage.cost.total, when the provider reports pricing. */
     readonly costUsd: number;
     /** Number of tool calls dispatched from this turn. */
@@ -29,6 +31,7 @@ export interface RunUsageSummary {
     readonly tokensIn: number;
     readonly tokensOut: number;
     readonly cacheReadTokens: number;
+    readonly cacheWriteTokens: number;
     readonly costUsd: number;
     readonly toolCalls: number;
     /** Every tool name called across the whole run, in dispatch order. */

package/dist/turns.js

@@ -15,6 +15,7 @@ export function deriveTurns(events) {
             tokensIn: usage?.input ?? 0,
             tokensOut: usage?.output ?? 0,
             cacheReadTokens: usage?.cacheRead ?? 0,
+            cacheWriteTokens: usage?.cacheWrite ?? 0,
             costUsd: usage?.cost.total ?? 0,
             toolCalls: event.toolResults.length,
             toolNames: event.toolResults.map((result) => result.toolName),
@@ -29,6 +30,7 @@ export function summarizeRunUsage(turns) {
         tokensIn: turns.reduce((sum, turn) => sum + turn.tokensIn, 0),
         tokensOut: turns.reduce((sum, turn) => sum + turn.tokensOut, 0),
         cacheReadTokens: turns.reduce((sum, turn) => sum + turn.cacheReadTokens, 0),
+        cacheWriteTokens: turns.reduce((sum, turn) => sum + turn.cacheWriteTokens, 0),
         costUsd: turns.reduce((sum, turn) => sum + turn.costUsd, 0),
         toolCalls: turns.reduce((sum, turn) => sum + turn.toolCalls, 0),
         toolNames: turns.flatMap((turn) => turn.toolNames),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@danypops/pi-eval-harness",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Scores a real agent run's own tool-call behavior -- AND/OR tool-call matching, graduated checker composition, and turn/tool-call/token-usage rollups -- over Pi's own real AgentSessionEvent stream (e.g. from @danypops/pi-process-harness).",
   "license": "MIT",
   "type": "module",

package/src/ablation.ts

@@ -19,6 +19,8 @@ export interface AblationDelta {
 	readonly meanScoreDelta: number;
 	readonly meanDurationMsDelta: number;
 	readonly meanTokensInDelta: number;
+	readonly meanCacheReadTokensDelta: number;
+	readonly meanCacheWriteTokensDelta: number;
 }
 
 /** One config's own trial metrics, plus its delta vs baseline. `delta` is undefined for the baseline itself. */
@@ -55,6 +57,8 @@ export async function ablate(configs: readonly AblationConfig[], n: number, opti
 				meanScoreDelta: metrics.meanScore - baseline.meanScore,
 				meanDurationMsDelta: metrics.meanDurationMs - baseline.meanDurationMs,
 				meanTokensInDelta: metrics.meanTokensIn - baseline.meanTokensIn,
+				meanCacheReadTokensDelta: metrics.meanCacheReadTokens - baseline.meanCacheReadTokens,
+				meanCacheWriteTokensDelta: metrics.meanCacheWriteTokens - baseline.meanCacheWriteTokens,
 			},
 		};
 	});
@@ -71,12 +75,12 @@ export function formatAblation(label: string, results: readonly AblationResult[]
 	for (const result of results) {
 		const m = result.metrics;
 		lines.push(
-			`  ${result.config.name.padEnd(14)}  pass_rate=${m.passRate.toFixed(2)}  mean_score=${m.meanScore.toFixed(2)}  mean_duration_ms=${m.meanDurationMs.toFixed(0)}  tokens_in=${m.meanTokensIn.toFixed(0)}`,
+			`  ${result.config.name.padEnd(14)}  pass_rate=${m.passRate.toFixed(2)}  mean_score=${m.meanScore.toFixed(2)}  mean_duration_ms=${m.meanDurationMs.toFixed(0)}  tokens_in=${m.meanTokensIn.toFixed(0)}  cache_read=${m.meanCacheReadTokens.toFixed(0)}  cache_write=${m.meanCacheWriteTokens.toFixed(0)}`,
 		);
 		if (result.delta) {
 			const d = result.delta;
 			lines.push(
-				`  ${"(vs baseline)".padEnd(14)}  Δpass_rate=${signed(d.passRateDelta, 2)}  Δscore=${signed(d.meanScoreDelta, 2)}  Δduration_ms=${signed(d.meanDurationMsDelta, 0)}  Δtokens_in=${signed(d.meanTokensInDelta, 0)}`,
+				`  ${"(vs baseline)".padEnd(14)}  Δpass_rate=${signed(d.passRateDelta, 2)}  Δscore=${signed(d.meanScoreDelta, 2)}  Δduration_ms=${signed(d.meanDurationMsDelta, 0)}  Δtokens_in=${signed(d.meanTokensInDelta, 0)}  Δcache_read=${signed(d.meanCacheReadTokensDelta, 0)}  Δcache_write=${signed(d.meanCacheWriteTokensDelta, 0)}`,
 			);
 		}
 	}

package/src/trials.ts

@@ -12,6 +12,9 @@ export interface TrialResult {
 	readonly durationMs: number;
 	readonly tokensIn: number;
 	readonly tokensOut: number;
+	/** Real cache-read/cache-write tokens -- under prompt caching, the dominant real component of total context size; tokensIn alone (a provider's own incremental, non-cached count) understates it. */
+	readonly cacheReadTokens: number;
+	readonly cacheWriteTokens: number;
 	readonly costUsd: number;
 	readonly error?: string;
 }
@@ -28,6 +31,8 @@ export interface TrialMetrics {
 	readonly meanDurationMs: number;
 	readonly meanTokensIn: number;
 	readonly meanTokensOut: number;
+	readonly meanCacheReadTokens: number;
+	readonly meanCacheWriteTokens: number;
 	readonly meanCostUsd: number;
 }
 
@@ -86,6 +91,8 @@ export function aggregateTrials(results: readonly TrialResult[], options: { read
 		meanDurationMs: mean(results.map((result) => result.durationMs)),
 		meanTokensIn: mean(results.map((result) => result.tokensIn)),
 		meanTokensOut: mean(results.map((result) => result.tokensOut)),
+		meanCacheReadTokens: mean(results.map((result) => result.cacheReadTokens)),
+		meanCacheWriteTokens: mean(results.map((result) => result.cacheWriteTokens)),
 		meanCostUsd: mean(results.map((result) => result.costUsd)),
 	};
 }
@@ -119,6 +126,8 @@ export async function runTrials(runOne: () => Promise<TrialResult>, n: number, o
 						durationMs: 0,
 						tokensIn: 0,
 						tokensOut: 0,
+						cacheReadTokens: 0,
+						cacheWriteTokens: 0,
 						costUsd: 0,
 						error: error instanceof Error ? error.message : String(error),
 					};

package/src/turns.ts

@@ -15,6 +15,8 @@ export interface Turn {
 	readonly tokensIn: number;
 	readonly tokensOut: number;
 	readonly cacheReadTokens: number;
+	/** New context tokens written to the provider's cache this turn -- the other real component of total context size under prompt caching, alongside cacheReadTokens. */
+	readonly cacheWriteTokens: number;
 	/** Real cost in USD from Usage.cost.total, when the provider reports pricing. */
 	readonly costUsd: number;
 	/** Number of tool calls dispatched from this turn. */
@@ -39,6 +41,7 @@ export function deriveTurns(events: readonly AgentSessionEvent[]): Turn[] {
 			tokensIn: usage?.input ?? 0,
 			tokensOut: usage?.output ?? 0,
 			cacheReadTokens: usage?.cacheRead ?? 0,
+			cacheWriteTokens: usage?.cacheWrite ?? 0,
 			costUsd: usage?.cost.total ?? 0,
 			toolCalls: event.toolResults.length,
 			toolNames: event.toolResults.map((result) => result.toolName),
@@ -53,6 +56,7 @@ export interface RunUsageSummary {
 	readonly tokensIn: number;
 	readonly tokensOut: number;
 	readonly cacheReadTokens: number;
+	readonly cacheWriteTokens: number;
 	readonly costUsd: number;
 	readonly toolCalls: number;
 	/** Every tool name called across the whole run, in dispatch order. */
@@ -66,6 +70,7 @@ export function summarizeRunUsage(turns: readonly Turn[]): RunUsageSummary {
 		tokensIn: turns.reduce((sum, turn) => sum + turn.tokensIn, 0),
 		tokensOut: turns.reduce((sum, turn) => sum + turn.tokensOut, 0),
 		cacheReadTokens: turns.reduce((sum, turn) => sum + turn.cacheReadTokens, 0),
+		cacheWriteTokens: turns.reduce((sum, turn) => sum + turn.cacheWriteTokens, 0),
 		costUsd: turns.reduce((sum, turn) => sum + turn.costUsd, 0),
 		toolCalls: turns.reduce((sum, turn) => sum + turn.toolCalls, 0),
 		toolNames: turns.flatMap((turn) => turn.toolNames),