anpord 0.1.23 → 0.1.24

package/dist/bin.cjs

@@ -19,7 +19,7 @@ let yaml = require("yaml");
 let node_crypto = require("node:crypto");
 let node_child_process = require("node:child_process");
 //#region package.json
-var version$1 = "0.1.23";
+var version$1 = "0.1.24";
 //#endregion
 //#region ../template/src/extract.ts
 /** Reads escapes too, so text the renderer leaves literal is never reported as
@@ -220,6 +220,86 @@ var NoEvalFiles = class extends effect.Data.TaggedError("NoEvalFiles") {
 	}
 };
 //#endregion
+//#region ../schema/src/domain/usage-health.ts
+const CACHE_FLOOR = 5e4;
+const PER_TURN_CEILING = 12e3;
+const perTurnInput = ({ turns, usage }) => turns <= 0 ? usage.inputTokens : Math.round(usage.inputTokens / turns);
+const usageConcerns = (reading) => {
+	const { usage } = reading;
+	const served = usage.inputTokens + usage.cacheReadTokens;
+	const concerns = [];
+	if (perTurnInput(reading) >= PER_TURN_CEILING) concerns.push("context-grew-fast");
+	if (served >= CACHE_FLOOR && usage.cacheReadTokens === 0) concerns.push("nothing-cached");
+	return concerns;
+};
+const CONCERN_REASONS = {
+	"context-grew-fast": "Each turn carried a large context, which is what happens when an agent reads a dependency or a long file into the conversation and then re-sends it every turn after.",
+	"nothing-cached": "Nothing was served from cache, so every turn paid full price for the turns before it."
+};
+//#endregion
+//#region src/cli/eval-usage.ts
+const THOUSAND = 1e3;
+const MILLION = 1e6;
+const CENT = .01;
+const trialsOf = (run) => run.cells.flatMap((cell) => cell.trials);
+const runUsage = (run) => {
+	const concerns = /* @__PURE__ */ new Set();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let totalTokens = 0;
+	for (const trial of trialsOf(run)) {
+		if (!trial.usage) continue;
+		inputTokens += trial.usage.inputTokens;
+		outputTokens += trial.usage.outputTokens;
+		totalTokens += trial.usage.totalTokens;
+		for (const concern of usageConcerns({
+			turns: trial.commands,
+			usage: trial.usage
+		})) concerns.add(concern);
+	}
+	return {
+		concerns: [...concerns],
+		inputTokens,
+		outputTokens,
+		totalTokens,
+		usd: run.costs?.estimatedEquivalentUsd ?? null
+	};
+};
+const formatTokens = (value) => {
+	if (value >= MILLION) return `${(value / MILLION).toFixed(1)}M`;
+	return value >= THOUSAND ? `${Math.round(value / THOUSAND)}k` : String(value);
+};
+const formatUsd = (value) => value < CENT ? "<$0.01" : `$${value.toFixed(2)}`;
+const usageLines = (usage) => {
+	if (usage.totalTokens === 0) return [];
+	const spend = usage.usd === null ? "" : `, ${formatUsd(usage.usd)} est.`;
+	return [`${formatTokens(usage.totalTokens)} tokens (${formatTokens(usage.inputTokens)} in, ${formatTokens(usage.outputTokens)} out)${spend}`, ...usage.concerns.map((concern) => CONCERN_REASONS[concern])];
+};
+const localUsage = (cases) => {
+	const concerns = /* @__PURE__ */ new Set();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let totalTokens = 0;
+	for (const one of cases) {
+		if (!one.usage) continue;
+		inputTokens += one.usage.inputTokens;
+		outputTokens += one.usage.outputTokens;
+		totalTokens += one.usage.inputTokens + one.usage.outputTokens;
+		for (const concern of usageConcerns({
+			turns: one.turns,
+			usage: one.usage
+		})) concerns.add(concern);
+	}
+	return {
+		concerns: [...concerns],
+		inputTokens,
+		outputTokens,
+		totalTokens,
+		usd: null
+	};
+};
+const localUsageLines = (cases) => usageLines(localUsage(cases)).map((line) => `  ${line}`);
+//#endregion
 //#region src/cli/eval-grid.ts
 const DIM = "\x1B[2m";
 const BOLD = "\x1B[1m";
@@ -289,7 +369,10 @@ const formatGrid = (run, trials, elapsedMs) => {
 		}
 		lines.push("");
 	}
-	lines.push(paint(DIM, `  ${formatElapsed(elapsedMs)} elapsed`));
+	const usage = runUsage(run);
+	const [spend, ...concerns] = usageLines(usage);
+	lines.push(paint(DIM, spend === void 0 ? `  ${formatElapsed(elapsedMs)} elapsed` : `  ${formatElapsed(elapsedMs)} elapsed  ·  ${spend}`));
+	for (const concern of concerns) lines.push(paint(YELLOW, `  ${concern}`));
 	return lines;
 };
 const up = (rows) => `[${rows}A`;
@@ -4543,11 +4626,17 @@ const runLocally = (request, options = {}) => effect.Effect.gen(function* () {
 	}) ?? effect.Effect.void), effect.Effect.map((outcome) => ({
 		durationMs: outcome.durationMs,
 		name: subject.name,
-		status: outcome.outcome.status
+		status: outcome.outcome.status,
+		turns: outcome.outcome.commandCount,
+		usage: effect.Option.getOrNull(outcome.result.usage)
 	}))), { concurrency: 1 });
 }).pipe(effect.Effect.provide(options.credentials === void 0 ? EvalLocalLive : evalLocalWith(credentialResolverFrom(options.credentials))), effect.Effect.scoped, effect.Effect.withConfigProvider(effect.ConfigProvider.fromEnv().pipe(effect.ConfigProvider.orElse(() => effect.ConfigProvider.fromMap(/* @__PURE__ */ new Map([["ANPORD_LOCAL_SANDBOX", "true"]]))))));
 const seconds = (ms) => `${(ms / 1e3).toFixed(1)}s`;
-const reportLocal = (file, cases) => effect.Effect.forEach([`${file}: ${cases.length} case(s) on this machine`, ...cases.map((one) => `  ${one.status} ${one.name} ${seconds(one.durationMs)}`)], note);
+const reportLocal = (file, cases) => effect.Effect.forEach([
+	`${file}: ${cases.length} case(s) on this machine`,
+	...cases.map((one) => `  ${one.status} ${one.name} ${seconds(one.durationMs)}`),
+	...localUsageLines(cases)
+], note);
 const localProblems = (cases) => cases.flatMap((one) => one.status === "passed" ? [] : [`${one.name} ${one.status}`]);
 //#endregion
 //#region src/cli/eval-outcome.ts

package/dist/bin.mjs

@@ -19,7 +19,7 @@ import { parseDocument } from "yaml";
 import { createHash } from "node:crypto";
 import { spawn } from "node:child_process";
 //#region package.json
-var version$1 = "0.1.23";
+var version$1 = "0.1.24";
 //#endregion
 //#region ../template/src/extract.ts
 /** Reads escapes too, so text the renderer leaves literal is never reported as
@@ -220,6 +220,86 @@ var NoEvalFiles = class extends Data.TaggedError("NoEvalFiles") {
 	}
 };
 //#endregion
+//#region ../schema/src/domain/usage-health.ts
+const CACHE_FLOOR = 5e4;
+const PER_TURN_CEILING = 12e3;
+const perTurnInput = ({ turns, usage }) => turns <= 0 ? usage.inputTokens : Math.round(usage.inputTokens / turns);
+const usageConcerns = (reading) => {
+	const { usage } = reading;
+	const served = usage.inputTokens + usage.cacheReadTokens;
+	const concerns = [];
+	if (perTurnInput(reading) >= PER_TURN_CEILING) concerns.push("context-grew-fast");
+	if (served >= CACHE_FLOOR && usage.cacheReadTokens === 0) concerns.push("nothing-cached");
+	return concerns;
+};
+const CONCERN_REASONS = {
+	"context-grew-fast": "Each turn carried a large context, which is what happens when an agent reads a dependency or a long file into the conversation and then re-sends it every turn after.",
+	"nothing-cached": "Nothing was served from cache, so every turn paid full price for the turns before it."
+};
+//#endregion
+//#region src/cli/eval-usage.ts
+const THOUSAND = 1e3;
+const MILLION = 1e6;
+const CENT = .01;
+const trialsOf = (run) => run.cells.flatMap((cell) => cell.trials);
+const runUsage = (run) => {
+	const concerns = /* @__PURE__ */ new Set();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let totalTokens = 0;
+	for (const trial of trialsOf(run)) {
+		if (!trial.usage) continue;
+		inputTokens += trial.usage.inputTokens;
+		outputTokens += trial.usage.outputTokens;
+		totalTokens += trial.usage.totalTokens;
+		for (const concern of usageConcerns({
+			turns: trial.commands,
+			usage: trial.usage
+		})) concerns.add(concern);
+	}
+	return {
+		concerns: [...concerns],
+		inputTokens,
+		outputTokens,
+		totalTokens,
+		usd: run.costs?.estimatedEquivalentUsd ?? null
+	};
+};
+const formatTokens = (value) => {
+	if (value >= MILLION) return `${(value / MILLION).toFixed(1)}M`;
+	return value >= THOUSAND ? `${Math.round(value / THOUSAND)}k` : String(value);
+};
+const formatUsd = (value) => value < CENT ? "<$0.01" : `$${value.toFixed(2)}`;
+const usageLines = (usage) => {
+	if (usage.totalTokens === 0) return [];
+	const spend = usage.usd === null ? "" : `, ${formatUsd(usage.usd)} est.`;
+	return [`${formatTokens(usage.totalTokens)} tokens (${formatTokens(usage.inputTokens)} in, ${formatTokens(usage.outputTokens)} out)${spend}`, ...usage.concerns.map((concern) => CONCERN_REASONS[concern])];
+};
+const localUsage = (cases) => {
+	const concerns = /* @__PURE__ */ new Set();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let totalTokens = 0;
+	for (const one of cases) {
+		if (!one.usage) continue;
+		inputTokens += one.usage.inputTokens;
+		outputTokens += one.usage.outputTokens;
+		totalTokens += one.usage.inputTokens + one.usage.outputTokens;
+		for (const concern of usageConcerns({
+			turns: one.turns,
+			usage: one.usage
+		})) concerns.add(concern);
+	}
+	return {
+		concerns: [...concerns],
+		inputTokens,
+		outputTokens,
+		totalTokens,
+		usd: null
+	};
+};
+const localUsageLines = (cases) => usageLines(localUsage(cases)).map((line) => `  ${line}`);
+//#endregion
 //#region src/cli/eval-grid.ts
 const DIM = "\x1B[2m";
 const BOLD = "\x1B[1m";
@@ -289,7 +369,10 @@ const formatGrid = (run, trials, elapsedMs) => {
 		}
 		lines.push("");
 	}
-	lines.push(paint(DIM, `  ${formatElapsed(elapsedMs)} elapsed`));
+	const usage = runUsage(run);
+	const [spend, ...concerns] = usageLines(usage);
+	lines.push(paint(DIM, spend === void 0 ? `  ${formatElapsed(elapsedMs)} elapsed` : `  ${formatElapsed(elapsedMs)} elapsed  ·  ${spend}`));
+	for (const concern of concerns) lines.push(paint(YELLOW, `  ${concern}`));
 	return lines;
 };
 const up = (rows) => `[${rows}A`;
@@ -4543,11 +4626,17 @@ const runLocally = (request, options = {}) => Effect.gen(function* () {
 	}) ?? Effect.void), Effect.map((outcome) => ({
 		durationMs: outcome.durationMs,
 		name: subject.name,
-		status: outcome.outcome.status
+		status: outcome.outcome.status,
+		turns: outcome.outcome.commandCount,
+		usage: Option.getOrNull(outcome.result.usage)
 	}))), { concurrency: 1 });
 }).pipe(Effect.provide(options.credentials === void 0 ? EvalLocalLive : evalLocalWith(credentialResolverFrom(options.credentials))), Effect.scoped, Effect.withConfigProvider(ConfigProvider.fromEnv().pipe(ConfigProvider.orElse(() => ConfigProvider.fromMap(/* @__PURE__ */ new Map([["ANPORD_LOCAL_SANDBOX", "true"]]))))));
 const seconds = (ms) => `${(ms / 1e3).toFixed(1)}s`;
-const reportLocal = (file, cases) => Effect.forEach([`${file}: ${cases.length} case(s) on this machine`, ...cases.map((one) => `  ${one.status} ${one.name} ${seconds(one.durationMs)}`)], note);
+const reportLocal = (file, cases) => Effect.forEach([
+	`${file}: ${cases.length} case(s) on this machine`,
+	...cases.map((one) => `  ${one.status} ${one.name} ${seconds(one.durationMs)}`),
+	...localUsageLines(cases)
+], note);
 const localProblems = (cases) => cases.flatMap((one) => one.status === "passed" ? [] : [`${one.name} ${one.status}`]);
 //#endregion
 //#region src/cli/eval-outcome.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anpord",
-  "version": "0.1.23",
+  "version": "0.1.24",
   "description": "TypeScript SDK and CLI for coding agent evals and prompt management.",
   "license": "MIT",
   "type": "module",