@os-eco/overstory-cli 0.6.1

package/src/metrics/transcript.test.ts

@@ -0,0 +1,356 @@
+/**
+ * Tests for Claude Code transcript JSONL parser.
+ *
+ * Uses temp files with real-format JSONL data. No mocks.
+ * Philosophy: "never mock what you can use for real" (mx-252b16).
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { cleanupTempDir } from "../test-helpers.ts";
+import { estimateCost, parseTranscriptUsage } from "./transcript.ts";
+
+let tempDir: string;
+
+beforeEach(async () => {
+	tempDir = await mkdtemp(join(tmpdir(), "overstory-transcript-test-"));
+});
+
+afterEach(async () => {
+	await cleanupTempDir(tempDir);
+});
+
+/** Write a JSONL file with the given lines. */
+async function writeJsonl(filename: string, lines: unknown[]): Promise<string> {
+	const path = join(tempDir, filename);
+	const content = `${lines.map((l) => JSON.stringify(l)).join("\n")}\n`;
+	await Bun.write(path, content);
+	return path;
+}
+
+// === parseTranscriptUsage ===
+
+describe("parseTranscriptUsage", () => {
+	test("parses a single assistant entry with all usage fields", async () => {
+		const path = await writeJsonl("single.jsonl", [
+			{
+				type: "assistant",
+				message: {
+					model: "claude-opus-4-6",
+					usage: {
+						input_tokens: 100,
+						output_tokens: 50,
+						cache_read_input_tokens: 1000,
+						cache_creation_input_tokens: 500,
+					},
+				},
+			},
+		]);
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.inputTokens).toBe(100);
+		expect(usage.outputTokens).toBe(50);
+		expect(usage.cacheReadTokens).toBe(1000);
+		expect(usage.cacheCreationTokens).toBe(500);
+		expect(usage.modelUsed).toBe("claude-opus-4-6");
+	});
+
+	test("aggregates usage across multiple assistant turns", async () => {
+		const path = await writeJsonl("multi.jsonl", [
+			{
+				type: "assistant",
+				message: {
+					model: "claude-sonnet-4-20250514",
+					usage: {
+						input_tokens: 100,
+						output_tokens: 50,
+						cache_read_input_tokens: 1000,
+						cache_creation_input_tokens: 500,
+					},
+				},
+			},
+			{
+				type: "human",
+				message: { content: "follow-up question" },
+			},
+			{
+				type: "assistant",
+				message: {
+					model: "claude-sonnet-4-20250514",
+					usage: {
+						input_tokens: 200,
+						output_tokens: 75,
+						cache_read_input_tokens: 2000,
+						cache_creation_input_tokens: 0,
+					},
+				},
+			},
+		]);
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.inputTokens).toBe(300);
+		expect(usage.outputTokens).toBe(125);
+		expect(usage.cacheReadTokens).toBe(3000);
+		expect(usage.cacheCreationTokens).toBe(500);
+		expect(usage.modelUsed).toBe("claude-sonnet-4-20250514");
+	});
+
+	test("skips non-assistant entries (human, system, tool_use, etc.)", async () => {
+		const path = await writeJsonl("mixed.jsonl", [
+			{ type: "system", content: "system prompt" },
+			{
+				type: "assistant",
+				message: {
+					model: "claude-opus-4-6",
+					usage: {
+						input_tokens: 100,
+						output_tokens: 50,
+						cache_read_input_tokens: 0,
+						cache_creation_input_tokens: 0,
+					},
+				},
+			},
+			{ type: "human", message: { content: "hello" } },
+			{ type: "tool_result", content: "result" },
+		]);
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.inputTokens).toBe(100);
+		expect(usage.outputTokens).toBe(50);
+	});
+
+	test("returns zeros for empty file", async () => {
+		const path = join(tempDir, "empty.jsonl");
+		await Bun.write(path, "");
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.inputTokens).toBe(0);
+		expect(usage.outputTokens).toBe(0);
+		expect(usage.cacheReadTokens).toBe(0);
+		expect(usage.cacheCreationTokens).toBe(0);
+		expect(usage.modelUsed).toBeNull();
+	});
+
+	test("returns zeros for file with no assistant entries", async () => {
+		const path = await writeJsonl("no-assistant.jsonl", [
+			{ type: "human", message: { content: "hello" } },
+			{ type: "system", content: "system prompt" },
+		]);
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.inputTokens).toBe(0);
+		expect(usage.outputTokens).toBe(0);
+		expect(usage.modelUsed).toBeNull();
+	});
+
+	test("gracefully handles malformed JSON lines", async () => {
+		const path = join(tempDir, "malformed.jsonl");
+		const content = [
+			'{"type":"assistant","message":{"model":"claude-opus-4-6","usage":{"input_tokens":100,"output_tokens":50,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}}}',
+			"this is not valid json",
+			"",
+			'{"type":"assistant","message":{"model":"claude-opus-4-6","usage":{"input_tokens":200,"output_tokens":75,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}}}',
+		].join("\n");
+		await Bun.write(path, content);
+
+		const usage = await parseTranscriptUsage(path);
+
+		// Should parse the two valid assistant entries, skip the malformed line
+		expect(usage.inputTokens).toBe(300);
+		expect(usage.outputTokens).toBe(125);
+	});
+
+	test("handles assistant entries with missing usage fields (defaults to 0)", async () => {
+		const path = await writeJsonl("partial.jsonl", [
+			{
+				type: "assistant",
+				message: {
+					model: "claude-haiku-3-5-20241022",
+					usage: {
+						input_tokens: 100,
+						output_tokens: 50,
+						// No cache fields
+					},
+				},
+			},
+		]);
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.inputTokens).toBe(100);
+		expect(usage.outputTokens).toBe(50);
+		expect(usage.cacheReadTokens).toBe(0);
+		expect(usage.cacheCreationTokens).toBe(0);
+	});
+
+	test("handles assistant entries with no usage object", async () => {
+		const path = await writeJsonl("no-usage.jsonl", [
+			{
+				type: "assistant",
+				message: {
+					model: "claude-opus-4-6",
+					content: "response without usage",
+				},
+			},
+		]);
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.inputTokens).toBe(0);
+		expect(usage.outputTokens).toBe(0);
+		expect(usage.modelUsed).toBeNull();
+	});
+
+	test("captures model from first assistant turn only", async () => {
+		const path = await writeJsonl("model-change.jsonl", [
+			{
+				type: "assistant",
+				message: {
+					model: "claude-sonnet-4-20250514",
+					usage: {
+						input_tokens: 10,
+						output_tokens: 5,
+						cache_read_input_tokens: 0,
+						cache_creation_input_tokens: 0,
+					},
+				},
+			},
+			{
+				type: "assistant",
+				message: {
+					model: "claude-opus-4-6",
+					usage: {
+						input_tokens: 20,
+						output_tokens: 10,
+						cache_read_input_tokens: 0,
+						cache_creation_input_tokens: 0,
+					},
+				},
+			},
+		]);
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.modelUsed).toBe("claude-sonnet-4-20250514");
+		expect(usage.inputTokens).toBe(30);
+	});
+
+	test("handles real-world transcript format with trailing newlines", async () => {
+		const path = join(tempDir, "trailing.jsonl");
+		const content =
+			'{"type":"assistant","message":{"model":"claude-opus-4-6","usage":{"input_tokens":3,"output_tokens":9,"cache_read_input_tokens":19401,"cache_creation_input_tokens":9918}}}\n\n\n';
+		await Bun.write(path, content);
+
+		const usage = await parseTranscriptUsage(path);
+
+		expect(usage.inputTokens).toBe(3);
+		expect(usage.outputTokens).toBe(9);
+		expect(usage.cacheReadTokens).toBe(19401);
+		expect(usage.cacheCreationTokens).toBe(9918);
+	});
+});
+
+// === estimateCost ===
+
+describe("estimateCost", () => {
+	test("calculates cost for opus model", () => {
+		const cost = estimateCost({
+			inputTokens: 1_000_000,
+			outputTokens: 1_000_000,
+			cacheReadTokens: 1_000_000,
+			cacheCreationTokens: 1_000_000,
+			modelUsed: "claude-opus-4-6",
+		});
+
+		// opus: input=$15, output=$75, cacheRead=$1.50, cacheCreation=$3.75
+		expect(cost).toBeCloseTo(95.25, 2);
+	});
+
+	test("calculates cost for sonnet model", () => {
+		const cost = estimateCost({
+			inputTokens: 1_000_000,
+			outputTokens: 1_000_000,
+			cacheReadTokens: 1_000_000,
+			cacheCreationTokens: 1_000_000,
+			modelUsed: "claude-sonnet-4-20250514",
+		});
+
+		// sonnet: input=$3, output=$15, cacheRead=$0.30, cacheCreation=$0.75
+		expect(cost).toBeCloseTo(19.05, 2);
+	});
+
+	test("calculates cost for haiku model", () => {
+		const cost = estimateCost({
+			inputTokens: 1_000_000,
+			outputTokens: 1_000_000,
+			cacheReadTokens: 1_000_000,
+			cacheCreationTokens: 1_000_000,
+			modelUsed: "claude-haiku-3-5-20241022",
+		});
+
+		// haiku: input=$0.80, output=$4, cacheRead=$0.08, cacheCreation=$0.20
+		expect(cost).toBeCloseTo(5.08, 2);
+	});
+
+	test("returns null for unknown model", () => {
+		const cost = estimateCost({
+			inputTokens: 1_000_000,
+			outputTokens: 1_000_000,
+			cacheReadTokens: 0,
+			cacheCreationTokens: 0,
+			modelUsed: "gpt-4o",
+		});
+
+		expect(cost).toBeNull();
+	});
+
+	test("returns null when modelUsed is null", () => {
+		const cost = estimateCost({
+			inputTokens: 1_000_000,
+			outputTokens: 1_000_000,
+			cacheReadTokens: 0,
+			cacheCreationTokens: 0,
+			modelUsed: null,
+		});
+
+		expect(cost).toBeNull();
+	});
+
+	test("zero tokens yields zero cost", () => {
+		const cost = estimateCost({
+			inputTokens: 0,
+			outputTokens: 0,
+			cacheReadTokens: 0,
+			cacheCreationTokens: 0,
+			modelUsed: "claude-opus-4-6",
+		});
+
+		expect(cost).toBe(0);
+	});
+
+	test("realistic session cost calculation", () => {
+		// A typical agent session: ~20K input, ~5K output, heavy cache reads
+		const cost = estimateCost({
+			inputTokens: 20_000,
+			outputTokens: 5_000,
+			cacheReadTokens: 100_000,
+			cacheCreationTokens: 15_000,
+			modelUsed: "claude-sonnet-4-20250514",
+		});
+
+		// sonnet: (20K/1M)*3 + (5K/1M)*15 + (100K/1M)*0.30 + (15K/1M)*0.75
+		// = 0.06 + 0.075 + 0.03 + 0.01125 = $0.17625
+		expect(cost).not.toBeNull();
+		if (cost !== null) {
+			expect(cost).toBeGreaterThan(0.1);
+			expect(cost).toBeLessThan(1.0);
+		}
+	});
+});

package/src/metrics/transcript.ts

@@ -0,0 +1,175 @@
+/**
+ * Parser for Claude Code transcript JSONL files.
+ *
+ * Extracts token usage data from assistant-type entries in transcript files
+ * at ~/.claude/projects/{project-slug}/{session-id}.jsonl.
+ *
+ * Each assistant entry contains per-turn usage:
+ * {
+ *   "type": "assistant",
+ *   "message": {
+ *     "model": "claude-opus-4-6",
+ *     "usage": {
+ *       "input_tokens": 3,
+ *       "output_tokens": 9,
+ *       "cache_read_input_tokens": 19401,
+ *       "cache_creation_input_tokens": 9918
+ *     }
+ *   }
+ * }
+ */
+
+export interface TranscriptUsage {
+	inputTokens: number;
+	outputTokens: number;
+	cacheReadTokens: number;
+	cacheCreationTokens: number;
+	modelUsed: string | null;
+}
+
+/** Pricing per million tokens (USD). */
+interface ModelPricing {
+	inputPerMTok: number;
+	outputPerMTok: number;
+	cacheReadPerMTok: number;
+	cacheCreationPerMTok: number;
+}
+
+/** Hardcoded pricing for known Claude models. */
+const MODEL_PRICING: Record<string, ModelPricing> = {
+	opus: {
+		inputPerMTok: 15,
+		outputPerMTok: 75,
+		cacheReadPerMTok: 1.5, // 10% of input
+		cacheCreationPerMTok: 3.75, // 25% of input
+	},
+	sonnet: {
+		inputPerMTok: 3,
+		outputPerMTok: 15,
+		cacheReadPerMTok: 0.3, // 10% of input
+		cacheCreationPerMTok: 0.75, // 25% of input
+	},
+	haiku: {
+		inputPerMTok: 0.8,
+		outputPerMTok: 4,
+		cacheReadPerMTok: 0.08, // 10% of input
+		cacheCreationPerMTok: 0.2, // 25% of input
+	},
+};
+
+/**
+ * Determine the pricing tier for a given model string.
+ * Matches on substring: "opus" -> opus pricing, "sonnet" -> sonnet, "haiku" -> haiku.
+ * Returns null if unrecognized.
+ */
+function getPricingForModel(model: string): ModelPricing | null {
+	const lower = model.toLowerCase();
+	if (lower.includes("opus")) return MODEL_PRICING.opus ?? null;
+	if (lower.includes("sonnet")) return MODEL_PRICING.sonnet ?? null;
+	if (lower.includes("haiku")) return MODEL_PRICING.haiku ?? null;
+	return null;
+}
+
+/**
+ * Calculate the estimated cost in USD for a given usage and model.
+ * Returns null if the model is unrecognized.
+ */
+export function estimateCost(usage: TranscriptUsage): number | null {
+	if (usage.modelUsed === null) return null;
+
+	const pricing = getPricingForModel(usage.modelUsed);
+	if (pricing === null) return null;
+
+	const inputCost = (usage.inputTokens / 1_000_000) * pricing.inputPerMTok;
+	const outputCost = (usage.outputTokens / 1_000_000) * pricing.outputPerMTok;
+	const cacheReadCost = (usage.cacheReadTokens / 1_000_000) * pricing.cacheReadPerMTok;
+	const cacheCreationCost = (usage.cacheCreationTokens / 1_000_000) * pricing.cacheCreationPerMTok;
+
+	return inputCost + outputCost + cacheReadCost + cacheCreationCost;
+}
+
+/**
+ * Narrow an unknown value to determine if it looks like a transcript assistant entry.
+ * Returns the usage fields if valid, or null otherwise.
+ */
+function extractUsageFromEntry(entry: unknown): {
+	inputTokens: number;
+	outputTokens: number;
+	cacheReadTokens: number;
+	cacheCreationTokens: number;
+	model: string | undefined;
+} | null {
+	if (typeof entry !== "object" || entry === null) return null;
+
+	const obj = entry as Record<string, unknown>;
+	if (obj.type !== "assistant") return null;
+
+	const message = obj.message;
+	if (typeof message !== "object" || message === null) return null;
+
+	const msg = message as Record<string, unknown>;
+	const usage = msg.usage;
+	if (typeof usage !== "object" || usage === null) return null;
+
+	const u = usage as Record<string, unknown>;
+
+	return {
+		inputTokens: typeof u.input_tokens === "number" ? u.input_tokens : 0,
+		outputTokens: typeof u.output_tokens === "number" ? u.output_tokens : 0,
+		cacheReadTokens: typeof u.cache_read_input_tokens === "number" ? u.cache_read_input_tokens : 0,
+		cacheCreationTokens:
+			typeof u.cache_creation_input_tokens === "number" ? u.cache_creation_input_tokens : 0,
+		model: typeof msg.model === "string" ? msg.model : undefined,
+	};
+}
+
+/**
+ * Parse a Claude Code transcript JSONL file and aggregate token usage.
+ *
+ * Reads the file line by line, extracting usage data from each assistant
+ * entry. Returns aggregated totals and the model from the first assistant turn.
+ *
+ * @param transcriptPath - Absolute path to the transcript JSONL file
+ * @returns Aggregated usage data across all assistant turns
+ */
+export async function parseTranscriptUsage(transcriptPath: string): Promise<TranscriptUsage> {
+	const file = Bun.file(transcriptPath);
+	const text = await file.text();
+	const lines = text.split("\n");
+
+	const result: TranscriptUsage = {
+		inputTokens: 0,
+		outputTokens: 0,
+		cacheReadTokens: 0,
+		cacheCreationTokens: 0,
+		modelUsed: null,
+	};
+
+	for (const line of lines) {
+		const trimmed = line.trim();
+		if (trimmed.length === 0) continue;
+
+		let parsed: unknown;
+		try {
+			parsed = JSON.parse(trimmed);
+		} catch {
+			// Skip malformed lines
+			continue;
+		}
+
+		const usage = extractUsageFromEntry(parsed);
+		if (usage === null) continue;
+
+		result.inputTokens += usage.inputTokens;
+		result.outputTokens += usage.outputTokens;
+		result.cacheReadTokens += usage.cacheReadTokens;
+		result.cacheCreationTokens += usage.cacheCreationTokens;
+
+		// Capture model from first assistant turn
+		if (result.modelUsed === null && usage.model !== undefined) {
+			result.modelUsed = usage.model;
+		}
+	}
+
+	return result;
+}