@ag-eco/agentplate-cli 0.13.0

package/src/metrics/pricing.test.ts

@@ -0,0 +1,258 @@
+import { describe, expect, test } from "bun:test";
+import { estimateCost, getPricingForModel } from "./pricing";
+
+describe("getPricingForModel()", () => {
+	describe("Claude tiers", () => {
+		test("matches opus by substring in full model ID", () => {
+			const result = getPricingForModel("claude-opus-4-20250514");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(15);
+			expect(result?.outputPerMTok).toBe(75);
+		});
+
+		test("matches sonnet by substring in full model ID", () => {
+			const result = getPricingForModel("claude-sonnet-4-20250514");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(3);
+			expect(result?.outputPerMTok).toBe(15);
+		});
+
+		test("matches haiku by substring in full model ID", () => {
+			const result = getPricingForModel("claude-haiku-3-5-20241022");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(0.8);
+			expect(result?.outputPerMTok).toBe(4);
+		});
+	});
+
+	describe("OpenAI tiers", () => {
+		test("matches gpt-4o-mini", () => {
+			const result = getPricingForModel("gpt-4o-mini");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(0.15);
+		});
+
+		test("matches gpt-4o", () => {
+			const result = getPricingForModel("gpt-4o");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(2.5);
+		});
+
+		test("matches gpt-5", () => {
+			const result = getPricingForModel("gpt-5");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(10);
+		});
+
+		test("matches o3", () => {
+			const result = getPricingForModel("o3");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(10);
+			expect(result?.outputPerMTok).toBe(40);
+		});
+
+		test("matches o1", () => {
+			const result = getPricingForModel("o1");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(15);
+			expect(result?.outputPerMTok).toBe(60);
+		});
+	});
+
+	describe("Priority ordering", () => {
+		test("gpt-4o-mini matches before gpt-4o (substring overlap)", () => {
+			const mini = getPricingForModel("gpt-4o-mini");
+			const full = getPricingForModel("gpt-4o");
+			expect(mini).not.toBeNull();
+			expect(full).not.toBeNull();
+			if (mini === null || full === null) return;
+			// gpt-4o-mini is cheaper
+			expect(mini.inputPerMTok).toBeLessThan(full.inputPerMTok);
+			// A model string "gpt-4o-mini" resolves to mini pricing, not gpt-4o
+			expect(mini.inputPerMTok).toBe(0.15);
+		});
+
+		test("o3 matches before o1 (o1 string contains o1, o3 does not contain o1)", () => {
+			const o3 = getPricingForModel("o3");
+			const o1 = getPricingForModel("o1");
+			expect(o3).not.toBeNull();
+			expect(o1).not.toBeNull();
+			if (o3 === null || o1 === null) return;
+			expect(o3.outputPerMTok).toBe(40);
+			expect(o1.outputPerMTok).toBe(60);
+		});
+	});
+
+	describe("Gemini tiers", () => {
+		test("matches gemini-flash by 'flash' substring", () => {
+			const result = getPricingForModel("gemini-flash-2.0");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(0.1);
+			expect(result?.outputPerMTok).toBe(0.4);
+		});
+
+		test("matches gemini-pro by 'gemini' + 'pro' substrings", () => {
+			const result = getPricingForModel("gemini-2.0-pro-exp");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(1.25);
+			expect(result?.outputPerMTok).toBe(5);
+		});
+	});
+
+	describe("Case insensitivity", () => {
+		test("Claude-OPUS-4 resolves correctly", () => {
+			const result = getPricingForModel("Claude-OPUS-4");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(15);
+		});
+
+		test("SONNET resolves correctly", () => {
+			const result = getPricingForModel("SONNET");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(3);
+		});
+
+		test("Haiku resolves correctly", () => {
+			const result = getPricingForModel("Haiku");
+			expect(result).not.toBeNull();
+			expect(result?.inputPerMTok).toBe(0.8);
+		});
+	});
+
+	describe("Unknown models", () => {
+		test("returns null for llama-3-70b", () => {
+			expect(getPricingForModel("llama-3-70b")).toBeNull();
+		});
+
+		test("returns null for empty string", () => {
+			expect(getPricingForModel("")).toBeNull();
+		});
+
+		test("returns null for random gibberish", () => {
+			expect(getPricingForModel("xyzzy-foo-bar-9000")).toBeNull();
+		});
+	});
+});
+
+describe("estimateCost()", () => {
+	test("Typical Claude Opus usage: 1M input, 100K output, 500K cacheRead, 200K cacheCreation → $24.00", () => {
+		const cost = estimateCost({
+			inputTokens: 1_000_000,
+			outputTokens: 100_000,
+			cacheReadTokens: 500_000,
+			cacheCreationTokens: 200_000,
+			modelUsed: "claude-opus-4-20250514",
+		});
+		// inputCost = 1 * 15 = 15.00
+		// outputCost = 0.1 * 75 = 7.50
+		// cacheReadCost = 0.5 * 1.5 = 0.75
+		// cacheCreationCost = 0.2 * 3.75 = 0.75
+		// total = 24.00
+		expect(cost).toBe(24.0);
+	});
+
+	test("Typical Claude Sonnet usage: 500K input, 50K output, 100K cacheRead, 50K cacheCreation", () => {
+		const cost = estimateCost({
+			inputTokens: 500_000,
+			outputTokens: 50_000,
+			cacheReadTokens: 100_000,
+			cacheCreationTokens: 50_000,
+			modelUsed: "claude-sonnet-4-20250514",
+		});
+		// inputCost = 0.5 * 3 = 1.50
+		// outputCost = 0.05 * 15 = 0.75
+		// cacheReadCost = 0.1 * 0.3 = 0.03
+		// cacheCreationCost = 0.05 * 0.75 = 0.0375
+		// total = 2.3175
+		expect(cost).toBeCloseTo(2.3175, 4);
+	});
+
+	test("Zero tokens returns 0 (not null)", () => {
+		const cost = estimateCost({
+			inputTokens: 0,
+			outputTokens: 0,
+			cacheReadTokens: 0,
+			cacheCreationTokens: 0,
+			modelUsed: "claude-opus-4",
+		});
+		expect(cost).toBe(0);
+	});
+
+	test("Null modelUsed returns null", () => {
+		const cost = estimateCost({
+			inputTokens: 1000,
+			outputTokens: 500,
+			cacheReadTokens: 0,
+			cacheCreationTokens: 0,
+			modelUsed: null,
+		});
+		expect(cost).toBeNull();
+	});
+
+	test("Unknown model returns null", () => {
+		const cost = estimateCost({
+			inputTokens: 1000,
+			outputTokens: 500,
+			cacheReadTokens: 0,
+			cacheCreationTokens: 0,
+			modelUsed: "llama-3-70b",
+		});
+		expect(cost).toBeNull();
+	});
+
+	test("Input-only usage: only inputTokens > 0, rest zero", () => {
+		const cost = estimateCost({
+			inputTokens: 1_000_000,
+			outputTokens: 0,
+			cacheReadTokens: 0,
+			cacheCreationTokens: 0,
+			modelUsed: "claude-sonnet-4",
+		});
+		// inputCost = 1 * 3 = 3.00
+		expect(cost).toBe(3.0);
+	});
+
+	test("Output-only usage: only outputTokens > 0, rest zero", () => {
+		const cost = estimateCost({
+			inputTokens: 0,
+			outputTokens: 1_000_000,
+			cacheReadTokens: 0,
+			cacheCreationTokens: 0,
+			modelUsed: "claude-sonnet-4",
+		});
+		// outputCost = 1 * 15 = 15.00
+		expect(cost).toBe(15.0);
+	});
+
+	test("Cache-heavy usage: large cacheRead + cacheCreation, verify math", () => {
+		const cost = estimateCost({
+			inputTokens: 0,
+			outputTokens: 0,
+			cacheReadTokens: 10_000_000,
+			cacheCreationTokens: 5_000_000,
+			modelUsed: "claude-opus-4",
+		});
+		// cacheReadCost = 10 * 1.5 = 15.00
+		// cacheCreationCost = 5 * 3.75 = 18.75
+		// total = 33.75
+		expect(cost).toBeCloseTo(33.75, 5);
+	});
+});
+
+describe("Cache pricing ratios", () => {
+	test("Claude cache read is 10% of input price (verified on opus)", () => {
+		const pricing = getPricingForModel("claude-opus-4");
+		expect(pricing).not.toBeNull();
+		if (pricing === null) return;
+		const ratio = pricing.cacheReadPerMTok / pricing.inputPerMTok;
+		expect(ratio).toBeCloseTo(0.1, 10);
+	});
+
+	test("Claude cache creation is 25% of input price (verified on sonnet)", () => {
+		const pricing = getPricingForModel("claude-sonnet-4");
+		expect(pricing).not.toBeNull();
+		if (pricing === null) return;
+		const ratio = pricing.cacheCreationPerMTok / pricing.inputPerMTok;
+		expect(ratio).toBeCloseTo(0.25, 10);
+	});
+});

package/src/metrics/pricing.ts

@@ -0,0 +1,135 @@
+/**
+ * Runtime-agnostic pricing and cost estimation for AI models.
+ *
+ * Extracted from transcript.ts so any runtime can use cost estimation
+ * without pulling in Claude Code-specific JSONL parsing logic.
+ *
+ * To add support for a new provider model, add an entry to MODEL_PRICING
+ * using a lowercase substring that uniquely identifies the model tier
+ * (e.g. "opus", "sonnet", "haiku").
+ */
+
+/** Canonical token usage representation shared across all runtimes. */
+export interface TokenUsage {
+	inputTokens: number;
+	outputTokens: number;
+	cacheReadTokens: number;
+	cacheCreationTokens: number;
+	modelUsed: string | null;
+}
+
+/** Pricing per million tokens (USD). */
+export interface ModelPricing {
+	inputPerMTok: number;
+	outputPerMTok: number;
+	cacheReadPerMTok: number;
+	cacheCreationPerMTok: number;
+}
+
+/** Pricing for known AI models across providers. */
+const MODEL_PRICING: Record<string, ModelPricing> = {
+	// --- Claude ---
+	opus: {
+		inputPerMTok: 15,
+		outputPerMTok: 75,
+		cacheReadPerMTok: 1.5, // 10% of input
+		cacheCreationPerMTok: 3.75, // 25% of input
+	},
+	sonnet: {
+		inputPerMTok: 3,
+		outputPerMTok: 15,
+		cacheReadPerMTok: 0.3, // 10% of input
+		cacheCreationPerMTok: 0.75, // 25% of input
+	},
+	haiku: {
+		inputPerMTok: 0.8,
+		outputPerMTok: 4,
+		cacheReadPerMTok: 0.08, // 10% of input
+		cacheCreationPerMTok: 0.2, // 25% of input
+	},
+	// --- OpenAI GPT ---
+	"gpt-4o-mini": {
+		inputPerMTok: 0.15,
+		outputPerMTok: 0.6,
+		cacheReadPerMTok: 0.075, // 50% of input
+		cacheCreationPerMTok: 0.15,
+	},
+	"gpt-4o": {
+		inputPerMTok: 2.5,
+		outputPerMTok: 10,
+		cacheReadPerMTok: 1.25,
+		cacheCreationPerMTok: 2.5,
+	},
+	"gpt-5": {
+		inputPerMTok: 10,
+		outputPerMTok: 40,
+		cacheReadPerMTok: 5,
+		cacheCreationPerMTok: 10,
+	},
+	o1: {
+		inputPerMTok: 15,
+		outputPerMTok: 60,
+		cacheReadPerMTok: 7.5,
+		cacheCreationPerMTok: 15,
+	},
+	o3: {
+		inputPerMTok: 10,
+		outputPerMTok: 40,
+		cacheReadPerMTok: 5,
+		cacheCreationPerMTok: 10,
+	},
+	// --- Google Gemini ---
+	"gemini-flash": {
+		inputPerMTok: 0.1,
+		outputPerMTok: 0.4,
+		cacheReadPerMTok: 0.025,
+		cacheCreationPerMTok: 0.1,
+	},
+	"gemini-pro": {
+		inputPerMTok: 1.25,
+		outputPerMTok: 5,
+		cacheReadPerMTok: 0.3125,
+		cacheCreationPerMTok: 1.25,
+	},
+};
+
+/**
+ * Determine the pricing tier for a given model string.
+ * Matches on substring in priority order to avoid ambiguous overlaps.
+ * Returns null if unrecognized.
+ */
+export function getPricingForModel(model: string): ModelPricing | null {
+	const lower = model.toLowerCase();
+	// --- Claude ---
+	if (lower.includes("opus")) return MODEL_PRICING.opus ?? null;
+	if (lower.includes("sonnet")) return MODEL_PRICING.sonnet ?? null;
+	if (lower.includes("haiku")) return MODEL_PRICING.haiku ?? null;
+	// --- OpenAI GPT --- (gpt-4o-mini before gpt-4o; o3 before o1)
+	if (lower.includes("gpt-4o-mini")) return MODEL_PRICING["gpt-4o-mini"] ?? null;
+	if (lower.includes("gpt-4o")) return MODEL_PRICING["gpt-4o"] ?? null;
+	if (lower.includes("gpt-5")) return MODEL_PRICING["gpt-5"] ?? null;
+	if (lower.includes("o3")) return MODEL_PRICING.o3 ?? null;
+	if (lower.includes("o1")) return MODEL_PRICING.o1 ?? null;
+	// --- Google Gemini --- (flash before generic gemini+pro check)
+	if (lower.includes("flash")) return MODEL_PRICING["gemini-flash"] ?? null;
+	if (lower.includes("gemini") && lower.includes("pro")) return MODEL_PRICING["gemini-pro"] ?? null;
+	return null;
+}
+
+/**
+ * Calculate the estimated cost in USD for a given usage and model.
+ * Returns null if the model is unrecognized.
+ */
+export function estimateCost(usage: TokenUsage): number | null {
+	if (usage.modelUsed === null) return null;
+
+	const pricing = getPricingForModel(usage.modelUsed);
+	if (pricing === null) return null;
+
+	const inputCost = (usage.inputTokens / 1_000_000) * pricing.inputPerMTok;
+	const outputCost = (usage.outputTokens / 1_000_000) * pricing.outputPerMTok;
+	const cacheReadCost = (usage.cacheReadTokens / 1_000_000) * pricing.cacheReadPerMTok;
+	const cacheCreationCost = (usage.cacheCreationTokens / 1_000_000) * pricing.cacheCreationPerMTok;
+
+	return inputCost + outputCost + cacheReadCost + cacheCreationCost;
+}