@kata-sh/cli 0.1.0 → 0.1.2

package/src/resources/extensions/kata/tests/metrics-io.test.ts

@@ -0,0 +1,254 @@
+/**
+ * Tests for Kata metrics disk I/O — init, snapshot, load/save cycle.
+ * Uses a temp directory to avoid touching real .kata/ state.
+ */
+
+import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import {
+  initMetrics,
+  resetMetrics,
+  getLedger,
+  snapshotUnitMetrics,
+  type MetricsLedger,
+} from "../metrics.js";
+
+let passed = 0;
+let failed = 0;
+
+function assert(condition: boolean, message: string): void {
+  if (condition) {
+    passed++;
+  } else {
+    failed++;
+    console.error(`  FAIL: ${message}`);
+  }
+}
+
+function assertEq<T>(actual: T, expected: T, message: string): void {
+  if (JSON.stringify(actual) === JSON.stringify(expected)) {
+    passed++;
+  } else {
+    failed++;
+    console.error(
+      `  FAIL: ${message} — expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
+    );
+  }
+}
+
+// ─── Setup ────────────────────────────────────────────────────────────────────
+
+const tmpBase = mkdtempSync(join(tmpdir(), "kata-metrics-test-"));
+mkdirSync(join(tmpBase, ".kata"), { recursive: true });
+
+// Mock ExtensionContext with session entries
+function mockCtx(messages: any[] = []): any {
+  const entries = messages.map((msg, i) => ({
+    type: "message",
+    id: `entry-${i}`,
+    parentId: i > 0 ? `entry-${i - 1}` : null,
+    timestamp: new Date().toISOString(),
+    message: msg,
+  }));
+  return {
+    sessionManager: {
+      getEntries: () => entries,
+    },
+    model: { id: "claude-sonnet-4-20250514" },
+  };
+}
+
+// ─── Tests ────────────────────────────────────────────────────────────────────
+
+console.log("\n=== initMetrics / getLedger ===");
+
+{
+  resetMetrics();
+  assert(getLedger() === null, "ledger null before init");
+
+  initMetrics(tmpBase);
+  const ledger = getLedger();
+  assert(ledger !== null, "ledger not null after init");
+  assertEq(ledger!.version, 1, "version is 1");
+  assertEq(ledger!.units.length, 0, "no units initially");
+}
+
+console.log("\n=== snapshotUnitMetrics ===");
+
+{
+  resetMetrics();
+  initMetrics(tmpBase);
+
+  // Simulate a session with assistant messages containing usage data
+  const ctx = mockCtx([
+    { role: "user", content: "Do the thing" },
+    {
+      role: "assistant",
+      content: [
+        { type: "text", text: "I'll do the thing" },
+        { type: "tool_call", id: "tc1", name: "bash", input: {} },
+      ],
+      usage: {
+        input: 5000,
+        output: 2000,
+        cacheRead: 3000,
+        cacheWrite: 500,
+        totalTokens: 10500,
+        cost: {
+          input: 0.015,
+          output: 0.03,
+          cacheRead: 0.003,
+          cacheWrite: 0.002,
+          total: 0.05,
+        },
+      },
+    },
+    {
+      role: "toolResult",
+      toolCallId: "tc1",
+      content: [{ type: "text", text: "ok" }],
+    },
+    {
+      role: "assistant",
+      content: [{ type: "text", text: "Done!" }],
+      usage: {
+        input: 8000,
+        output: 1000,
+        cacheRead: 6000,
+        cacheWrite: 200,
+        totalTokens: 15200,
+        cost: {
+          input: 0.024,
+          output: 0.015,
+          cacheRead: 0.006,
+          cacheWrite: 0.001,
+          total: 0.046,
+        },
+      },
+    },
+  ]);
+
+  const unit = snapshotUnitMetrics(
+    ctx,
+    "execute-task",
+    "M001/S01/T01",
+    Date.now() - 5000,
+    "claude-sonnet-4-20250514",
+  );
+
+  assert(unit !== null, "unit returned");
+  assertEq(unit!.type, "execute-task", "type");
+  assertEq(unit!.id, "M001/S01/T01", "id");
+  assertEq(unit!.tokens.input, 13000, "input tokens (5000+8000)");
+  assertEq(unit!.tokens.output, 3000, "output tokens (2000+1000)");
+  assertEq(unit!.tokens.cacheRead, 9000, "cacheRead (3000+6000)");
+  assertEq(unit!.tokens.total, 25700, "total tokens (10500+15200)");
+  assert(
+    Math.abs(unit!.cost - 0.096) < 0.001,
+    `cost ~0.096 (got ${unit!.cost})`,
+  );
+  assertEq(unit!.toolCalls, 1, "1 tool call");
+  assertEq(unit!.assistantMessages, 2, "2 assistant messages");
+  assertEq(unit!.userMessages, 1, "1 user message");
+
+  // Verify ledger persisted
+  const ledger = getLedger()!;
+  assertEq(ledger.units.length, 1, "1 unit in ledger");
+}
+
+console.log("\n=== Persistence across init/reset cycles ===");
+
+{
+  // Reset and re-init — should load from disk
+  resetMetrics();
+  initMetrics(tmpBase);
+
+  const ledger = getLedger()!;
+  assertEq(ledger.units.length, 1, "unit survived reset+init");
+  assertEq(ledger.units[0].id, "M001/S01/T01", "correct unit ID");
+
+  // Add another unit
+  const ctx = mockCtx([
+    {
+      role: "assistant",
+      content: [{ type: "text", text: "Research complete" }],
+      usage: {
+        input: 3000,
+        output: 1500,
+        cacheRead: 1000,
+        cacheWrite: 300,
+        totalTokens: 5800,
+        cost: {
+          input: 0.009,
+          output: 0.023,
+          cacheRead: 0.001,
+          cacheWrite: 0.001,
+          total: 0.034,
+        },
+      },
+    },
+  ]);
+
+  snapshotUnitMetrics(
+    ctx,
+    "research-slice",
+    "M001/S02",
+    Date.now() - 3000,
+    "claude-sonnet-4-20250514",
+  );
+
+  // Verify both units persisted
+  resetMetrics();
+  initMetrics(tmpBase);
+  const final = getLedger()!;
+  assertEq(final.units.length, 2, "2 units after second snapshot");
+}
+
+console.log("\n=== File content verification ===");
+
+{
+  const raw = readFileSync(join(tmpBase, ".kata", "metrics.json"), "utf-8");
+  const parsed: MetricsLedger = JSON.parse(raw);
+  assertEq(parsed.version, 1, "file version is 1");
+  assertEq(parsed.units.length, 2, "file has 2 units");
+  assert(parsed.projectStartedAt > 0, "projectStartedAt is set");
+}
+
+console.log("\n=== Empty session handling ===");
+
+{
+  resetMetrics();
+  initMetrics(tmpBase);
+
+  // Empty session — no messages
+  const ctx = mockCtx([]);
+  const unit = snapshotUnitMetrics(
+    ctx,
+    "plan-slice",
+    "M001/S01",
+    Date.now(),
+    "test-model",
+  );
+  assert(unit === null, "returns null for empty session");
+
+  // Ledger shouldn't have grown
+  assertEq(
+    getLedger()!.units.length,
+    2,
+    "still 2 units (empty session not added)",
+  );
+}
+
+// ─── Cleanup ──────────────────────────────────────────────────────────────────
+
+resetMetrics();
+rmSync(tmpBase, { recursive: true, force: true });
+
+console.log(`\n${"=".repeat(40)}`);
+console.log(`Results: ${passed} passed, ${failed} failed`);
+if (failed > 0) {
+  process.exit(1);
+} else {
+  console.log("All tests passed ✓");
+}

package/src/resources/extensions/kata/tests/metrics.test.ts

@@ -0,0 +1,217 @@
+/**
+ * Tests for Kata metrics aggregation logic.
+ * Tests the pure functions — no file I/O, no extension context.
+ */
+
+import {
+  type UnitMetrics,
+  type TokenCounts,
+  classifyUnitPhase,
+  aggregateByPhase,
+  aggregateBySlice,
+  aggregateByModel,
+  getProjectTotals,
+  formatCost,
+  formatTokenCount,
+} from "../metrics.js";
+
+// ─── Test helpers ─────────────────────────────────────────────────────────────
+
+function makeUnit(overrides: Partial<UnitMetrics> = {}): UnitMetrics {
+  return {
+    type: "execute-task",
+    id: "M001/S01/T01",
+    model: "claude-sonnet-4-20250514",
+    startedAt: 1000,
+    finishedAt: 2000,
+    tokens: { input: 1000, output: 500, cacheRead: 200, cacheWrite: 100, total: 1800 },
+    cost: 0.05,
+    toolCalls: 3,
+    assistantMessages: 2,
+    userMessages: 1,
+    ...overrides,
+  };
+}
+
+let passed = 0;
+let failed = 0;
+
+function assert(condition: boolean, message: string): void {
+  if (condition) {
+    passed++;
+  } else {
+    failed++;
+    console.error(`  FAIL: ${message}`);
+  }
+}
+
+function assertEq<T>(actual: T, expected: T, message: string): void {
+  if (actual === expected) {
+    passed++;
+  } else {
+    failed++;
+    console.error(`  FAIL: ${message} — expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
+  }
+}
+
+function assertClose(actual: number, expected: number, tolerance: number, message: string): void {
+  if (Math.abs(actual - expected) <= tolerance) {
+    passed++;
+  } else {
+    failed++;
+    console.error(`  FAIL: ${message} — expected ~${expected}, got ${actual}`);
+  }
+}
+
+// ─── Phase classification ─────────────────────────────────────────────────────
+
+console.log("\n=== classifyUnitPhase ===");
+
+assertEq(classifyUnitPhase("research-milestone"), "research", "research-milestone → research");
+assertEq(classifyUnitPhase("research-slice"), "research", "research-slice → research");
+assertEq(classifyUnitPhase("plan-milestone"), "planning", "plan-milestone → planning");
+assertEq(classifyUnitPhase("plan-slice"), "planning", "plan-slice → planning");
+assertEq(classifyUnitPhase("execute-task"), "execution", "execute-task → execution");
+assertEq(classifyUnitPhase("complete-slice"), "completion", "complete-slice → completion");
+assertEq(classifyUnitPhase("reassess-roadmap"), "reassessment", "reassess-roadmap → reassessment");
+assertEq(classifyUnitPhase("unknown-thing"), "execution", "unknown → execution (fallback)");
+
+// ─── getProjectTotals ─────────────────────────────────────────────────────────
+
+console.log("\n=== getProjectTotals ===");
+
+{
+  const units = [
+    makeUnit({ tokens: { input: 1000, output: 500, cacheRead: 200, cacheWrite: 100, total: 1800 }, cost: 0.05, toolCalls: 3, startedAt: 1000, finishedAt: 2000 }),
+    makeUnit({ tokens: { input: 2000, output: 1000, cacheRead: 400, cacheWrite: 200, total: 3600 }, cost: 0.10, toolCalls: 5, startedAt: 2000, finishedAt: 4000 }),
+  ];
+  const totals = getProjectTotals(units);
+
+  assertEq(totals.units, 2, "total units");
+  assertEq(totals.tokens.input, 3000, "total input tokens");
+  assertEq(totals.tokens.output, 1500, "total output tokens");
+  assertEq(totals.tokens.cacheRead, 600, "total cacheRead");
+  assertEq(totals.tokens.cacheWrite, 300, "total cacheWrite");
+  assertEq(totals.tokens.total, 5400, "total tokens");
+  assertClose(totals.cost, 0.15, 0.001, "total cost");
+  assertEq(totals.toolCalls, 8, "total tool calls");
+  assertEq(totals.duration, 3000, "total duration");
+}
+
+{
+  const totals = getProjectTotals([]);
+  assertEq(totals.units, 0, "empty: zero units");
+  assertEq(totals.cost, 0, "empty: zero cost");
+  assertEq(totals.tokens.total, 0, "empty: zero tokens");
+}
+
+// ─── aggregateByPhase ─────────────────────────────────────────────────────────
+
+console.log("\n=== aggregateByPhase ===");
+
+{
+  const units = [
+    makeUnit({ type: "research-milestone", cost: 0.02 }),
+    makeUnit({ type: "research-slice", cost: 0.03 }),
+    makeUnit({ type: "plan-milestone", cost: 0.01 }),
+    makeUnit({ type: "plan-slice", cost: 0.02 }),
+    makeUnit({ type: "execute-task", cost: 0.10 }),
+    makeUnit({ type: "execute-task", cost: 0.08 }),
+    makeUnit({ type: "complete-slice", cost: 0.01 }),
+    makeUnit({ type: "reassess-roadmap", cost: 0.005 }),
+  ];
+  const phases = aggregateByPhase(units);
+
+  assertEq(phases.length, 5, "5 phases");
+  assertEq(phases[0].phase, "research", "first phase is research");
+  assertEq(phases[0].units, 2, "2 research units");
+  assertClose(phases[0].cost, 0.05, 0.001, "research cost");
+
+  assertEq(phases[1].phase, "planning", "second phase is planning");
+  assertEq(phases[1].units, 2, "2 planning units");
+
+  assertEq(phases[2].phase, "execution", "third phase is execution");
+  assertEq(phases[2].units, 2, "2 execution units");
+  assertClose(phases[2].cost, 0.18, 0.001, "execution cost");
+
+  assertEq(phases[3].phase, "completion", "fourth phase is completion");
+  assertEq(phases[4].phase, "reassessment", "fifth phase is reassessment");
+}
+
+// ─── aggregateBySlice ─────────────────────────────────────────────────────────
+
+console.log("\n=== aggregateBySlice ===");
+
+{
+  const units = [
+    makeUnit({ id: "M001/S01/T01", cost: 0.05 }),
+    makeUnit({ id: "M001/S01/T02", cost: 0.04 }),
+    makeUnit({ id: "M001/S02/T01", cost: 0.10 }),
+    makeUnit({ id: "M001", type: "research-milestone", cost: 0.02 }),
+  ];
+  const slices = aggregateBySlice(units);
+
+  assertEq(slices.length, 3, "3 slice groups");
+
+  const s01 = slices.find(s => s.sliceId === "M001/S01");
+  assert(!!s01, "M001/S01 exists");
+  assertEq(s01!.units, 2, "M001/S01 has 2 units");
+  assertClose(s01!.cost, 0.09, 0.001, "M001/S01 cost");
+
+  const s02 = slices.find(s => s.sliceId === "M001/S02");
+  assert(!!s02, "M001/S02 exists");
+  assertEq(s02!.units, 1, "M001/S02 has 1 unit");
+
+  const mLevel = slices.find(s => s.sliceId === "M001");
+  assert(!!mLevel, "M001 (milestone-level) exists");
+}
+
+// ─── aggregateByModel ─────────────────────────────────────────────────────────
+
+console.log("\n=== aggregateByModel ===");
+
+{
+  const units = [
+    makeUnit({ model: "claude-sonnet-4-20250514", cost: 0.05 }),
+    makeUnit({ model: "claude-sonnet-4-20250514", cost: 0.04 }),
+    makeUnit({ model: "claude-opus-4-20250514", cost: 0.30 }),
+  ];
+  const models = aggregateByModel(units);
+
+  assertEq(models.length, 2, "2 models");
+  // Sorted by cost desc — opus should be first
+  assertEq(models[0].model, "claude-opus-4-20250514", "opus first (higher cost)");
+  assertClose(models[0].cost, 0.30, 0.001, "opus cost");
+  assertEq(models[1].model, "claude-sonnet-4-20250514", "sonnet second");
+  assertEq(models[1].units, 2, "sonnet has 2 units");
+}
+
+// ─── formatCost ───────────────────────────────────────────────────────────────
+
+console.log("\n=== formatCost ===");
+
+assertEq(formatCost(0), "$0.0000", "zero cost");
+assertEq(formatCost(0.001), "$0.0010", "sub-cent cost");
+assertEq(formatCost(0.05), "$0.050", "5 cents");
+assertEq(formatCost(1.50), "$1.50", "dollar+");
+assertEq(formatCost(14.20), "$14.20", "double digits");
+
+// ─── formatTokenCount ─────────────────────────────────────────────────────────
+
+console.log("\n=== formatTokenCount ===");
+
+assertEq(formatTokenCount(0), "0", "zero tokens");
+assertEq(formatTokenCount(500), "500", "sub-k");
+assertEq(formatTokenCount(1500), "1.5k", "1.5k");
+assertEq(formatTokenCount(150000), "150.0k", "150k");
+assertEq(formatTokenCount(1500000), "1.50M", "1.5M");
+
+// ─── Summary ──────────────────────────────────────────────────────────────────
+
+console.log(`\n${"=".repeat(40)}`);
+console.log(`Results: ${passed} passed, ${failed} failed`);
+if (failed > 0) {
+  process.exit(1);
+} else {
+  console.log("All tests passed ✓");
+}