anpord 0.1.22 → 0.1.24

package/dist/api-context.d.cts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-B8a93t2t.cjs";
+import { t as ApiCall } from "./api-mocks-ofqP55P_.cjs";
 //#region src/mock-api/context.d.ts
 declare const apiContext: {
   url: (name: string) => Promise<string>;

package/dist/api-context.d.mts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-B8a93t2t.mjs";
+import { t as ApiCall } from "./api-mocks-ofqP55P_.mjs";
 //#region src/mock-api/context.d.ts
 declare const apiContext: {
   url: (name: string) => Promise<string>;

package/dist/{api-mocks-B8a93t2t.d.cts → api-mocks-ofqP55P_.d.cts}

@@ -58,6 +58,14 @@ declare const ApiCall_base: Schema.Class<ApiCall, {
   }>>>;
 }>, never, {
   readonly api: string;
+} & {
+  readonly status: number;
+} & {
+  readonly startedAt: number;
+} & {
+  readonly path: string;
+} & {
+  readonly index: number;
 } & {
   readonly output: {
     readonly text: string;
@@ -72,14 +80,6 @@ declare const ApiCall_base: Schema.Class<ApiCall, {
     readonly format: "text" | "json";
     readonly state: "captured" | "disabled" | "unavailable";
   };
-} & {
-  readonly status: number;
-} & {
-  readonly startedAt: number;
-} & {
-  readonly path: string;
-} & {
-  readonly index: number;
 } & {
   readonly error: string | null;
 } & {

package/dist/{api-mocks-B8a93t2t.d.mts → api-mocks-ofqP55P_.d.mts}

@@ -58,6 +58,14 @@ declare const ApiCall_base: Schema.Class<ApiCall, {
   }>>>;
 }>, never, {
   readonly api: string;
+} & {
+  readonly status: number;
+} & {
+  readonly startedAt: number;
+} & {
+  readonly path: string;
+} & {
+  readonly index: number;
 } & {
   readonly output: {
     readonly text: string;
@@ -72,14 +80,6 @@ declare const ApiCall_base: Schema.Class<ApiCall, {
     readonly format: "text" | "json";
     readonly state: "captured" | "disabled" | "unavailable";
   };
-} & {
-  readonly status: number;
-} & {
-  readonly startedAt: number;
-} & {
-  readonly path: string;
-} & {
-  readonly index: number;
 } & {
   readonly error: string | null;
 } & {

package/dist/api-runtime.d.cts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-B8a93t2t.cjs";
+import { t as ApiCall } from "./api-mocks-ofqP55P_.cjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.cjs";
 //#region src/mock-api/runtime.d.ts
 declare const withApi: <A>({ api, run }: {

package/dist/api-runtime.d.mts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-B8a93t2t.mjs";
+import { t as ApiCall } from "./api-mocks-ofqP55P_.mjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.mjs";
 //#region src/mock-api/runtime.d.ts
 declare const withApi: <A>({ api, run }: {

package/dist/api.d.cts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-B8a93t2t.cjs";
+import { t as ApiCall } from "./api-mocks-ofqP55P_.cjs";
 import { a as endpoint, i as api, n as ApiHandlerContext, r as EndpointDefinition, t as ApiDefinition } from "./define-NZvmQuIv.cjs";
 import { withApi } from "./api-runtime.cjs";
 export { type ApiCall, type ApiDefinition, type ApiHandlerContext, type EndpointDefinition, api, endpoint, withApi };

package/dist/api.d.mts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-B8a93t2t.mjs";
+import { t as ApiCall } from "./api-mocks-ofqP55P_.mjs";
 import { a as endpoint, i as api, n as ApiHandlerContext, r as EndpointDefinition, t as ApiDefinition } from "./define-NZvmQuIv.mjs";
 import { withApi } from "./api-runtime.mjs";
 export { type ApiCall, type ApiDefinition, type ApiHandlerContext, type EndpointDefinition, api, endpoint, withApi };

package/dist/bin.cjs

@@ -19,7 +19,7 @@ let yaml = require("yaml");
 let node_crypto = require("node:crypto");
 let node_child_process = require("node:child_process");
 //#region package.json
-var version$1 = "0.1.22";
+var version$1 = "0.1.24";
 //#endregion
 //#region ../template/src/extract.ts
 /** Reads escapes too, so text the renderer leaves literal is never reported as
@@ -220,6 +220,86 @@ var NoEvalFiles = class extends effect.Data.TaggedError("NoEvalFiles") {
 	}
 };
 //#endregion
+//#region ../schema/src/domain/usage-health.ts
+const CACHE_FLOOR = 5e4;
+const PER_TURN_CEILING = 12e3;
+const perTurnInput = ({ turns, usage }) => turns <= 0 ? usage.inputTokens : Math.round(usage.inputTokens / turns);
+const usageConcerns = (reading) => {
+	const { usage } = reading;
+	const served = usage.inputTokens + usage.cacheReadTokens;
+	const concerns = [];
+	if (perTurnInput(reading) >= PER_TURN_CEILING) concerns.push("context-grew-fast");
+	if (served >= CACHE_FLOOR && usage.cacheReadTokens === 0) concerns.push("nothing-cached");
+	return concerns;
+};
+const CONCERN_REASONS = {
+	"context-grew-fast": "Each turn carried a large context, which is what happens when an agent reads a dependency or a long file into the conversation and then re-sends it every turn after.",
+	"nothing-cached": "Nothing was served from cache, so every turn paid full price for the turns before it."
+};
+//#endregion
+//#region src/cli/eval-usage.ts
+const THOUSAND = 1e3;
+const MILLION = 1e6;
+const CENT = .01;
+const trialsOf = (run) => run.cells.flatMap((cell) => cell.trials);
+const runUsage = (run) => {
+	const concerns = /* @__PURE__ */ new Set();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let totalTokens = 0;
+	for (const trial of trialsOf(run)) {
+		if (!trial.usage) continue;
+		inputTokens += trial.usage.inputTokens;
+		outputTokens += trial.usage.outputTokens;
+		totalTokens += trial.usage.totalTokens;
+		for (const concern of usageConcerns({
+			turns: trial.commands,
+			usage: trial.usage
+		})) concerns.add(concern);
+	}
+	return {
+		concerns: [...concerns],
+		inputTokens,
+		outputTokens,
+		totalTokens,
+		usd: run.costs?.estimatedEquivalentUsd ?? null
+	};
+};
+const formatTokens = (value) => {
+	if (value >= MILLION) return `${(value / MILLION).toFixed(1)}M`;
+	return value >= THOUSAND ? `${Math.round(value / THOUSAND)}k` : String(value);
+};
+const formatUsd = (value) => value < CENT ? "<$0.01" : `$${value.toFixed(2)}`;
+const usageLines = (usage) => {
+	if (usage.totalTokens === 0) return [];
+	const spend = usage.usd === null ? "" : `, ${formatUsd(usage.usd)} est.`;
+	return [`${formatTokens(usage.totalTokens)} tokens (${formatTokens(usage.inputTokens)} in, ${formatTokens(usage.outputTokens)} out)${spend}`, ...usage.concerns.map((concern) => CONCERN_REASONS[concern])];
+};
+const localUsage = (cases) => {
+	const concerns = /* @__PURE__ */ new Set();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let totalTokens = 0;
+	for (const one of cases) {
+		if (!one.usage) continue;
+		inputTokens += one.usage.inputTokens;
+		outputTokens += one.usage.outputTokens;
+		totalTokens += one.usage.inputTokens + one.usage.outputTokens;
+		for (const concern of usageConcerns({
+			turns: one.turns,
+			usage: one.usage
+		})) concerns.add(concern);
+	}
+	return {
+		concerns: [...concerns],
+		inputTokens,
+		outputTokens,
+		totalTokens,
+		usd: null
+	};
+};
+const localUsageLines = (cases) => usageLines(localUsage(cases)).map((line) => `  ${line}`);
+//#endregion
 //#region src/cli/eval-grid.ts
 const DIM = "\x1B[2m";
 const BOLD = "\x1B[1m";
@@ -289,7 +369,10 @@ const formatGrid = (run, trials, elapsedMs) => {
 		}
 		lines.push("");
 	}
-	lines.push(paint(DIM, `  ${formatElapsed(elapsedMs)} elapsed`));
+	const usage = runUsage(run);
+	const [spend, ...concerns] = usageLines(usage);
+	lines.push(paint(DIM, spend === void 0 ? `  ${formatElapsed(elapsedMs)} elapsed` : `  ${formatElapsed(elapsedMs)} elapsed  ·  ${spend}`));
+	for (const concern of concerns) lines.push(paint(YELLOW, `  ${concern}`));
 	return lines;
 };
 const up = (rows) => `[${rows}A`;
@@ -4485,7 +4568,7 @@ const SandboxProviderLive = effect.Layer.effect(SandboxProvider, effect.Effect.g
 //#endregion
 //#region ../eval/src/local-layer.ts
 const LocalAdaptersLive = effect.Layer.effect(SandboxAdapters, effect.Effect.map(effect.Effect.cached(makeLocalAdapter), (local) => SandboxAdapters.of({ resolve: (provider) => provider === "local" ? local : effect.Effect.dieMessage(`a local run cannot open a ${provider} sandbox`) })));
-const evalLocalWith = (credentials) => LocalTrialsLive.pipe(effect.Layer.provide(AgentTrialJudgedLive.pipe(effect.Layer.provide(AgentTrialLive.pipe(effect.Layer.provide(ScorerChecksLive.pipe(effect.Layer.provide(ScorerGroundTruthLive))), effect.Layer.provide(SuspenderSleeping), effect.Layer.provide(SimulatedUserLive.pipe(effect.Layer.provide(_effect_platform.FetchHttpClient.layer))))), effect.Layer.provide(JudgeModelLive.pipe(effect.Layer.provide(_effect_platform.FetchHttpClient.layer))), effect.Layer.provide(effect.Layer.mergeAll(HarnessesLive, HarnessVersionsLive)))), effect.Layer.provide(credentials), effect.Layer.provide(SandboxProviderLive.pipe(effect.Layer.provide(LocalAdaptersLive))));
+const evalLocalWith = (credentials) => LocalTrialsLive.pipe(effect.Layer.provide(AgentTrialJudgedLive.pipe(effect.Layer.provide(AgentTrialLive.pipe(effect.Layer.provide(ScorerChecksLive.pipe(effect.Layer.provide(ScorerGroundTruthLive))), effect.Layer.provide(SuspenderSleeping), effect.Layer.provide(SimulatedUserLive.pipe(effect.Layer.provide(_effect_platform.FetchHttpClient.layer))))), effect.Layer.provide(JudgeModelLive.pipe(effect.Layer.provide(_effect_platform.FetchHttpClient.layer))), effect.Layer.provide(effect.Layer.mergeAll(HarnessesLive, HarnessVersionsLive)))), effect.Layer.provide(credentials), effect.Layer.provide(SandboxProviderLive.pipe(effect.Layer.provide(LocalAdaptersLive))), effect.Layer.provideMerge(HarnessVersionsLive));
 const EvalLocalLive = evalLocalWith(CredentialResolverFromEnv);
 //#endregion
 //#region src/cli/local-env.ts
@@ -4514,19 +4597,21 @@ const localEnv = (directory) => Object.assign({}, ...FILES.map((file) => read(di
 //#region src/cli/eval-local.ts
 const runLocally = (request, options = {}) => effect.Effect.gen(function* () {
 	const trials = yield* LocalTrials;
+	const versions = yield* HarnessVersions;
 	const forwarded = localEnv(process.cwd());
 	const task = request.tasks[0];
 	if (task === void 0) return [];
+	const harnessVersion = yield* versions.version(task.harness);
 	return yield* effect.Effect.forEach(request.cases, (subject) => trials.run({
 		caseName: subject.name,
 		forwarded,
 		harness: task.harness,
-		harnessVersion: "local",
+		harnessVersion,
 		model: task.model,
 		prepare: subject.prepare ?? null,
 		profile: profileOfRequest(task.profile),
 		prompt: request.prompt,
-		source: { kind: "empty" },
+		source: subject.source ?? { kind: "empty" },
 		user: subject.user ?? null,
 		validator: subject.validator ?? null,
 		verifyCommand: subject.verify
@@ -4541,11 +4626,17 @@ const runLocally = (request, options = {}) => effect.Effect.gen(function* () {
 	}) ?? effect.Effect.void), effect.Effect.map((outcome) => ({
 		durationMs: outcome.durationMs,
 		name: subject.name,
-		status: outcome.outcome.status
+		status: outcome.outcome.status,
+		turns: outcome.outcome.commandCount,
+		usage: effect.Option.getOrNull(outcome.result.usage)
 	}))), { concurrency: 1 });
 }).pipe(effect.Effect.provide(options.credentials === void 0 ? EvalLocalLive : evalLocalWith(credentialResolverFrom(options.credentials))), effect.Effect.scoped, effect.Effect.withConfigProvider(effect.ConfigProvider.fromEnv().pipe(effect.ConfigProvider.orElse(() => effect.ConfigProvider.fromMap(/* @__PURE__ */ new Map([["ANPORD_LOCAL_SANDBOX", "true"]]))))));
 const seconds = (ms) => `${(ms / 1e3).toFixed(1)}s`;
-const reportLocal = (file, cases) => effect.Effect.forEach([`${file}: ${cases.length} case(s) on this machine`, ...cases.map((one) => `  ${one.status} ${one.name} ${seconds(one.durationMs)}`)], note);
+const reportLocal = (file, cases) => effect.Effect.forEach([
+	`${file}: ${cases.length} case(s) on this machine`,
+	...cases.map((one) => `  ${one.status} ${one.name} ${seconds(one.durationMs)}`),
+	...localUsageLines(cases)
+], note);
 const localProblems = (cases) => cases.flatMap((one) => one.status === "passed" ? [] : [`${one.name} ${one.status}`]);
 //#endregion
 //#region src/cli/eval-outcome.ts
@@ -4822,20 +4913,19 @@ const recordedLocally = (file) => effect.Effect.gen(function* () {
 		trigger
 	} });
 	yield* reportStarted(file, started.id);
+	yield* effect.Effect.addFinalizer(() => api.evals.finishRun({ payload: { id: started.id } }).pipe(effect.Effect.ignore));
 	const leased = yield* api.evals.credentials({ payload: {
 		harness: payload.tasks[0]?.harness ?? "codex",
 		id: started.id
 	} }).pipe(effect.Effect.map((lease) => lease.values), effect.Effect.catchAll(() => effect.Effect.succeed(void 0)));
-	const cases = yield* runLocally(payload, {
+	return yield* runLocally(payload, {
 		credentials: leased,
 		onTrial: (trial) => api.evals.reportTrial({ payload: {
 			id: started.id,
 			trial
 		} }).pipe(effect.Effect.ignore)
 	});
-	yield* api.evals.finishRun({ payload: { id: started.id } }).pipe(effect.Effect.ignore);
-	return cases;
-}).pipe(effect.Effect.provide(require_config.ClientLayer));
+}).pipe(effect.Effect.scoped, effect.Effect.provide(require_config.ClientLayer));
 const runOneEvalLocally = (file) => require_config.apiKeyConfig.pipe(effect.Effect.matchEffect({
 	onFailure: () => require_compiler.compileEvalEffect(file).pipe(effect.Effect.flatMap((payload) => runLocally(payload))),
 	onSuccess: () => recordedLocally(file)

package/dist/bin.mjs

@@ -19,7 +19,7 @@ import { parseDocument } from "yaml";
 import { createHash } from "node:crypto";
 import { spawn } from "node:child_process";
 //#region package.json
-var version$1 = "0.1.22";
+var version$1 = "0.1.24";
 //#endregion
 //#region ../template/src/extract.ts
 /** Reads escapes too, so text the renderer leaves literal is never reported as
@@ -220,6 +220,86 @@ var NoEvalFiles = class extends Data.TaggedError("NoEvalFiles") {
 	}
 };
 //#endregion
+//#region ../schema/src/domain/usage-health.ts
+const CACHE_FLOOR = 5e4;
+const PER_TURN_CEILING = 12e3;
+const perTurnInput = ({ turns, usage }) => turns <= 0 ? usage.inputTokens : Math.round(usage.inputTokens / turns);
+const usageConcerns = (reading) => {
+	const { usage } = reading;
+	const served = usage.inputTokens + usage.cacheReadTokens;
+	const concerns = [];
+	if (perTurnInput(reading) >= PER_TURN_CEILING) concerns.push("context-grew-fast");
+	if (served >= CACHE_FLOOR && usage.cacheReadTokens === 0) concerns.push("nothing-cached");
+	return concerns;
+};
+const CONCERN_REASONS = {
+	"context-grew-fast": "Each turn carried a large context, which is what happens when an agent reads a dependency or a long file into the conversation and then re-sends it every turn after.",
+	"nothing-cached": "Nothing was served from cache, so every turn paid full price for the turns before it."
+};
+//#endregion
+//#region src/cli/eval-usage.ts
+const THOUSAND = 1e3;
+const MILLION = 1e6;
+const CENT = .01;
+const trialsOf = (run) => run.cells.flatMap((cell) => cell.trials);
+const runUsage = (run) => {
+	const concerns = /* @__PURE__ */ new Set();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let totalTokens = 0;
+	for (const trial of trialsOf(run)) {
+		if (!trial.usage) continue;
+		inputTokens += trial.usage.inputTokens;
+		outputTokens += trial.usage.outputTokens;
+		totalTokens += trial.usage.totalTokens;
+		for (const concern of usageConcerns({
+			turns: trial.commands,
+			usage: trial.usage
+		})) concerns.add(concern);
+	}
+	return {
+		concerns: [...concerns],
+		inputTokens,
+		outputTokens,
+		totalTokens,
+		usd: run.costs?.estimatedEquivalentUsd ?? null
+	};
+};
+const formatTokens = (value) => {
+	if (value >= MILLION) return `${(value / MILLION).toFixed(1)}M`;
+	return value >= THOUSAND ? `${Math.round(value / THOUSAND)}k` : String(value);
+};
+const formatUsd = (value) => value < CENT ? "<$0.01" : `$${value.toFixed(2)}`;
+const usageLines = (usage) => {
+	if (usage.totalTokens === 0) return [];
+	const spend = usage.usd === null ? "" : `, ${formatUsd(usage.usd)} est.`;
+	return [`${formatTokens(usage.totalTokens)} tokens (${formatTokens(usage.inputTokens)} in, ${formatTokens(usage.outputTokens)} out)${spend}`, ...usage.concerns.map((concern) => CONCERN_REASONS[concern])];
+};
+const localUsage = (cases) => {
+	const concerns = /* @__PURE__ */ new Set();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let totalTokens = 0;
+	for (const one of cases) {
+		if (!one.usage) continue;
+		inputTokens += one.usage.inputTokens;
+		outputTokens += one.usage.outputTokens;
+		totalTokens += one.usage.inputTokens + one.usage.outputTokens;
+		for (const concern of usageConcerns({
+			turns: one.turns,
+			usage: one.usage
+		})) concerns.add(concern);
+	}
+	return {
+		concerns: [...concerns],
+		inputTokens,
+		outputTokens,
+		totalTokens,
+		usd: null
+	};
+};
+const localUsageLines = (cases) => usageLines(localUsage(cases)).map((line) => `  ${line}`);
+//#endregion
 //#region src/cli/eval-grid.ts
 const DIM = "\x1B[2m";
 const BOLD = "\x1B[1m";
@@ -289,7 +369,10 @@ const formatGrid = (run, trials, elapsedMs) => {
 		}
 		lines.push("");
 	}
-	lines.push(paint(DIM, `  ${formatElapsed(elapsedMs)} elapsed`));
+	const usage = runUsage(run);
+	const [spend, ...concerns] = usageLines(usage);
+	lines.push(paint(DIM, spend === void 0 ? `  ${formatElapsed(elapsedMs)} elapsed` : `  ${formatElapsed(elapsedMs)} elapsed  ·  ${spend}`));
+	for (const concern of concerns) lines.push(paint(YELLOW, `  ${concern}`));
 	return lines;
 };
 const up = (rows) => `[${rows}A`;
@@ -4485,7 +4568,7 @@ const SandboxProviderLive = Layer.effect(SandboxProvider, Effect.gen(function* (
 //#endregion
 //#region ../eval/src/local-layer.ts
 const LocalAdaptersLive = Layer.effect(SandboxAdapters, Effect.map(Effect.cached(makeLocalAdapter), (local) => SandboxAdapters.of({ resolve: (provider) => provider === "local" ? local : Effect.dieMessage(`a local run cannot open a ${provider} sandbox`) })));
-const evalLocalWith = (credentials) => LocalTrialsLive.pipe(Layer.provide(AgentTrialJudgedLive.pipe(Layer.provide(AgentTrialLive.pipe(Layer.provide(ScorerChecksLive.pipe(Layer.provide(ScorerGroundTruthLive))), Layer.provide(SuspenderSleeping), Layer.provide(SimulatedUserLive.pipe(Layer.provide(FetchHttpClient.layer))))), Layer.provide(JudgeModelLive.pipe(Layer.provide(FetchHttpClient.layer))), Layer.provide(Layer.mergeAll(HarnessesLive, HarnessVersionsLive)))), Layer.provide(credentials), Layer.provide(SandboxProviderLive.pipe(Layer.provide(LocalAdaptersLive))));
+const evalLocalWith = (credentials) => LocalTrialsLive.pipe(Layer.provide(AgentTrialJudgedLive.pipe(Layer.provide(AgentTrialLive.pipe(Layer.provide(ScorerChecksLive.pipe(Layer.provide(ScorerGroundTruthLive))), Layer.provide(SuspenderSleeping), Layer.provide(SimulatedUserLive.pipe(Layer.provide(FetchHttpClient.layer))))), Layer.provide(JudgeModelLive.pipe(Layer.provide(FetchHttpClient.layer))), Layer.provide(Layer.mergeAll(HarnessesLive, HarnessVersionsLive)))), Layer.provide(credentials), Layer.provide(SandboxProviderLive.pipe(Layer.provide(LocalAdaptersLive))), Layer.provideMerge(HarnessVersionsLive));
 const EvalLocalLive = evalLocalWith(CredentialResolverFromEnv);
 //#endregion
 //#region src/cli/local-env.ts
@@ -4514,19 +4597,21 @@ const localEnv = (directory) => Object.assign({}, ...FILES.map((file) => read(di
 //#region src/cli/eval-local.ts
 const runLocally = (request, options = {}) => Effect.gen(function* () {
 	const trials = yield* LocalTrials;
+	const versions = yield* HarnessVersions;
 	const forwarded = localEnv(process.cwd());
 	const task = request.tasks[0];
 	if (task === void 0) return [];
+	const harnessVersion = yield* versions.version(task.harness);
 	return yield* Effect.forEach(request.cases, (subject) => trials.run({
 		caseName: subject.name,
 		forwarded,
 		harness: task.harness,
-		harnessVersion: "local",
+		harnessVersion,
 		model: task.model,
 		prepare: subject.prepare ?? null,
 		profile: profileOfRequest(task.profile),
 		prompt: request.prompt,
-		source: { kind: "empty" },
+		source: subject.source ?? { kind: "empty" },
 		user: subject.user ?? null,
 		validator: subject.validator ?? null,
 		verifyCommand: subject.verify
@@ -4541,11 +4626,17 @@ const runLocally = (request, options = {}) => Effect.gen(function* () {
 	}) ?? Effect.void), Effect.map((outcome) => ({
 		durationMs: outcome.durationMs,
 		name: subject.name,
-		status: outcome.outcome.status
+		status: outcome.outcome.status,
+		turns: outcome.outcome.commandCount,
+		usage: Option.getOrNull(outcome.result.usage)
 	}))), { concurrency: 1 });
 }).pipe(Effect.provide(options.credentials === void 0 ? EvalLocalLive : evalLocalWith(credentialResolverFrom(options.credentials))), Effect.scoped, Effect.withConfigProvider(ConfigProvider.fromEnv().pipe(ConfigProvider.orElse(() => ConfigProvider.fromMap(/* @__PURE__ */ new Map([["ANPORD_LOCAL_SANDBOX", "true"]]))))));
 const seconds = (ms) => `${(ms / 1e3).toFixed(1)}s`;
-const reportLocal = (file, cases) => Effect.forEach([`${file}: ${cases.length} case(s) on this machine`, ...cases.map((one) => `  ${one.status} ${one.name} ${seconds(one.durationMs)}`)], note);
+const reportLocal = (file, cases) => Effect.forEach([
+	`${file}: ${cases.length} case(s) on this machine`,
+	...cases.map((one) => `  ${one.status} ${one.name} ${seconds(one.durationMs)}`),
+	...localUsageLines(cases)
+], note);
 const localProblems = (cases) => cases.flatMap((one) => one.status === "passed" ? [] : [`${one.name} ${one.status}`]);
 //#endregion
 //#region src/cli/eval-outcome.ts
@@ -4822,20 +4913,19 @@ const recordedLocally = (file) => Effect.gen(function* () {
 		trigger
 	} });
 	yield* reportStarted(file, started.id);
+	yield* Effect.addFinalizer(() => api.evals.finishRun({ payload: { id: started.id } }).pipe(Effect.ignore));
 	const leased = yield* api.evals.credentials({ payload: {
 		harness: payload.tasks[0]?.harness ?? "codex",
 		id: started.id
 	} }).pipe(Effect.map((lease) => lease.values), Effect.catchAll(() => Effect.succeed(void 0)));
-	const cases = yield* runLocally(payload, {
+	return yield* runLocally(payload, {
 		credentials: leased,
 		onTrial: (trial) => api.evals.reportTrial({ payload: {
 			id: started.id,
 			trial
 		} }).pipe(Effect.ignore)
 	});
-	yield* api.evals.finishRun({ payload: { id: started.id } }).pipe(Effect.ignore);
-	return cases;
-}).pipe(Effect.provide(ClientLayer));
+}).pipe(Effect.scoped, Effect.provide(ClientLayer));
 const runOneEvalLocally = (file) => apiKeyConfig.pipe(Effect.matchEffect({
 	onFailure: () => compileEvalEffect(file).pipe(Effect.flatMap((payload) => runLocally(payload))),
 	onSuccess: () => recordedLocally(file)