@app-vitals/agent-runtime 0.0.1 → 0.1.0

package/progress-milestones.unit.test.ts

@@ -0,0 +1,188 @@
+/**
+ * Tests for agent/src/runtime/progress-milestones.ts
+ *
+ * Strategy: call phaseForBlock directly with plain content-block objects.
+ * The load-bearing invariant under test — `block.input` must never be read —
+ * is verified both via string-absence assertions and via a `Proxy`/getter
+ * trap that throws if `input` is ever accessed, proving the code path
+ * genuinely never touches it (not just that it doesn't leak downstream).
+ */
+
+import { describe, expect, test } from "bun:test";
+import { phaseForBlock } from "./progress-milestones.ts";
+
+describe("phaseForBlock — thinking / text blocks", () => {
+  test("maps a thinking block to the thinking phase", () => {
+    expect(phaseForBlock({ type: "thinking", thinking: "hmm" })).toBe(
+      "thinking",
+    );
+  });
+
+  test("maps a text block to the writing phase", () => {
+    expect(phaseForBlock({ type: "text", text: "Here you go." })).toBe(
+      "writing",
+    );
+  });
+});
+
+describe("phaseForBlock — reading tools", () => {
+  for (const name of ["Read", "NotebookRead"]) {
+    test(`maps ${name} to the reading phase`, () => {
+      expect(phaseForBlock({ type: "tool_use", name, input: {} })).toBe(
+        "reading",
+      );
+    });
+  }
+});
+
+describe("phaseForBlock — searching tools", () => {
+  for (const name of ["Grep", "Glob"]) {
+    test(`maps ${name} to the searching phase`, () => {
+      expect(phaseForBlock({ type: "tool_use", name, input: {} })).toBe(
+        "searching",
+      );
+    });
+  }
+});
+
+describe("phaseForBlock — web tools", () => {
+  for (const name of ["WebSearch", "WebFetch"]) {
+    test(`maps ${name} to the web phase`, () => {
+      expect(phaseForBlock({ type: "tool_use", name, input: {} })).toBe("web");
+    });
+  }
+});
+
+describe("phaseForBlock — editing tools", () => {
+  for (const name of ["Edit", "Write", "NotebookEdit"]) {
+    test(`maps ${name} to the editing phase`, () => {
+      expect(phaseForBlock({ type: "tool_use", name, input: {} })).toBe(
+        "editing",
+      );
+    });
+  }
+});
+
+describe("phaseForBlock — running tools", () => {
+  for (const name of ["Bash", "KillShell", "BashOutput"]) {
+    test(`maps ${name} to the running phase`, () => {
+      expect(phaseForBlock({ type: "tool_use", name, input: {} })).toBe(
+        "running",
+      );
+    });
+  }
+});
+
+describe("phaseForBlock — delegating tools", () => {
+  for (const name of ["Agent", "Task"]) {
+    test(`maps ${name} to the delegating phase`, () => {
+      expect(phaseForBlock({ type: "tool_use", name, input: {} })).toBe(
+        "delegating",
+      );
+    });
+  }
+});
+
+describe("phaseForBlock — planning tools", () => {
+  for (const name of ["TodoWrite", "ExitPlanMode", "EnterPlanMode"]) {
+    test(`maps ${name} to the planning phase`, () => {
+      expect(phaseForBlock({ type: "tool_use", name, input: {} })).toBe(
+        "planning",
+      );
+    });
+  }
+});
+
+describe("phaseForBlock — generic fallback", () => {
+  test("maps an mcp__*-prefixed tool name to the generic tool phase", () => {
+    expect(
+      phaseForBlock({
+        type: "tool_use",
+        name: "mcp__linear__create_issue",
+        input: {},
+      }),
+    ).toBe("tool");
+  });
+
+  test("maps an unrecognized tool name to the generic tool phase", () => {
+    expect(
+      phaseForBlock({ type: "tool_use", name: "SomeFutureTool", input: {} }),
+    ).toBe("tool");
+  });
+
+  test("never throws for an unknown tool name — always returns a phase", () => {
+    expect(() =>
+      phaseForBlock({ type: "tool_use", name: "", input: {} }),
+    ).not.toThrow();
+    expect(phaseForBlock({ type: "tool_use", name: "", input: {} })).toBe(
+      "tool",
+    );
+  });
+});
+
+describe("phaseForBlock — unmapped block types", () => {
+  test("returns undefined for a tool_result block (out of scope)", () => {
+    expect(
+      phaseForBlock({
+        type: "tool_result",
+        tool_use_id: "tu_1",
+        content: "ok",
+      } as never),
+    ).toBeUndefined();
+  });
+});
+
+// ─── The load-bearing invariant: block.input is NEVER read ──────────────────
+
+describe("phaseForBlock — never reads block.input", () => {
+  test("phase output never contains secret/path substrings embedded in input", () => {
+    const forbidden = ["/home/alice/.ssh/id_rsa", "sk-fake-12345"];
+    const block = {
+      type: "tool_use" as const,
+      name: "Bash",
+      input: {
+        command: "cat /home/alice/.ssh/id_rsa",
+        api_key: "sk-fake-12345",
+      },
+    };
+    const phase = phaseForBlock(block);
+    expect(phase).toBe("running");
+    for (const secret of forbidden) {
+      expect(String(phase)).not.toContain(secret);
+    }
+  });
+
+  test("throws if `input` is ever accessed — proves the code path never reads it", () => {
+    let accessed = false;
+    const block = {
+      type: "tool_use" as const,
+      name: "Read",
+      get input() {
+        accessed = true;
+        throw new Error("block.input must never be read by phaseForBlock");
+      },
+    };
+
+    const phase = phaseForBlock(block);
+
+    expect(accessed).toBe(false);
+    expect(phase).toBe("reading");
+  });
+
+  test("throws-on-access input still works for an unrecognized tool name (generic fallback path)", () => {
+    let accessed = false;
+    const block = {
+      type: "tool_use" as const,
+      name: "SomeFutureTool",
+      get input() {
+        accessed = true;
+        throw new Error("block.input must never be read by phaseForBlock");
+      },
+    };
+
+    const phase = phaseForBlock(block);
+
+    expect(accessed).toBe(false);
+    expect(phase).toBe("tool");
+  });
+});

package/run-telemetry.ts

@@ -0,0 +1,241 @@
+/**
+ * agent/src/runtime/run-telemetry.ts
+ *
+ * Per-run measurement derived from the Claude CLI's `stream-json` events,
+ * accumulated incrementally by `claude.ts`'s `_consumeStream` alongside the
+ * existing per-model usage map. Three things are captured:
+ *
+ *   1. **First-turn context baseline** — the usage of the first `assistant`
+ *      message. `input + cache_creation + cache_read` of that turn is the
+ *      full context the model saw before any work happened (system prompt,
+ *      tool schemas, always-loaded CLAUDE.md/rules, the skill listing, the
+ *      cron prompt). Summing all three makes the number independent of cache
+ *      warmth: a warm prefix cache just moves tokens from `cache_creation`
+ *      to `cache_read`. It is the measured, per-model cost of the agent's
+ *      always-loaded markdown — the number the prompt-audit patrol compares
+ *      before/after a CLAUDE.md change.
+ *   2. **Turn and tool-call counts** — distinct usage-bearing message ids and
+ *      distinct `tool_use` block ids.
+ *   3. **Per-skill attribution** — which plugin skills were invoked (the
+ *      `Skill` tool), how often, and the tokens of every turn that ran while
+ *      that skill was the most recently invoked one. Attribution is
+ *      deterministic and last-wins: a skill's body stays in the conversation
+ *      for the rest of the session, so every later turn is charged to the
+ *      most recent invoke until another skill is invoked. Turns before any
+ *      invoke accrue to the `root` bucket. Sub-agent turns (events carrying
+ *      `parent_tool_use_id`) are charged to an `agent:<subagent_type>`
+ *      bucket and never change the current skill.
+ *
+ * Pure accumulator — no I/O. The same event shapes appear in Claude Code's
+ * transcript JSONL, so the accumulator can also replay a saved transcript.
+ */
+
+import type { ContentBlock } from "./progress-milestones.ts";
+
+interface FirstTurnUsage {
+  model: string;
+  messageId: string;
+  inputTokens: number;
+  cacheCreationTokens: number;
+  cacheReadTokens: number;
+  /** input + cacheCreation + cacheRead — the full context at turn 1. */
+  contextTokens: number;
+}
+
+type AttributionKind = "skill" | "agent" | "root";
+
+export interface SkillUsageRow {
+  kind: AttributionKind;
+  /** Skill name as invoked (e.g. "shipwright:task-store"), subagent type, or "root". */
+  name: string;
+  /** Distinct Skill/Agent tool_use blocks that invoked this name. 0 for root. */
+  invocations: number;
+  /** Usage-bearing turns charged to this bucket. */
+  turns: number;
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens: number;
+  cacheCreationTokens: number;
+  /**
+   * `input + cache_creation` of the first usage-bearing turn after this
+   * skill's first invoke — the new context admitted at that turn, which is
+   * dominated by the skill body just loaded. Null until that turn arrives,
+   * and always null for root/agent buckets.
+   */
+  invokeContextDelta: number | null;
+}
+
+export interface RunTelemetry {
+  firstTurn?: FirstTurnUsage;
+  turns: number;
+  toolCalls: number;
+  skillUsage: SkillUsageRow[];
+}
+
+/** Minimal usage shape shared by stream-json and transcript lines. */
+interface TurnUsageLike {
+  input_tokens?: number;
+  output_tokens?: number;
+  cache_read_input_tokens?: number;
+  cache_creation_input_tokens?: number;
+}
+
+interface Bucket extends SkillUsageRow {
+  awaitingInvokeDelta: boolean;
+}
+
+function bucketKey(kind: AttributionKind, name: string): string {
+  return `${kind}:${name}`;
+}
+
+function newBucket(kind: AttributionKind, name: string): Bucket {
+  return {
+    kind,
+    name,
+    invocations: 0,
+    turns: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: 0,
+    cacheCreationTokens: 0,
+    invokeContextDelta: null,
+    awaitingInvokeDelta: false,
+  };
+}
+
+function toolUseInput(block: ContentBlock): Record<string, unknown> {
+  const input = (block as { input?: unknown }).input;
+  return input && typeof input === "object"
+    ? (input as Record<string, unknown>)
+    : {};
+}
+
+export class RunTelemetryAccumulator {
+  private firstTurn: FirstTurnUsage | undefined;
+  private turns = 0;
+  private readonly seenToolUseIds = new Set<string>();
+  private readonly buckets = new Map<string, Bucket>();
+  /** Bucket key the next non-subagent usage-bearing turn is charged to. */
+  private current = bucketKey("root", "root");
+  /** Agent tool_use id → subagent type, for `parent_tool_use_id` lookups. */
+  private readonly subagentByToolUseId = new Map<string, string>();
+
+  constructor() {
+    this.buckets.set(this.current, newBucket("root", "root"));
+  }
+
+  /**
+   * Observe an assistant message's content blocks. Safe to call on every
+   * stream line, including repeats of the same message id — tool_use blocks
+   * are deduped by their own id.
+   */
+  observeContent(blocks: ContentBlock[] | undefined): void {
+    if (!Array.isArray(blocks)) return;
+    for (const block of blocks) {
+      if (block.type !== "tool_use") continue;
+      const id = (block as { id?: unknown }).id;
+      if (typeof id !== "string" || this.seenToolUseIds.has(id)) continue;
+      this.seenToolUseIds.add(id);
+
+      const name = (block as { name?: unknown }).name;
+      if (name === "Skill") {
+        const skill = toolUseInput(block).skill;
+        if (typeof skill === "string" && skill.length > 0) {
+          const key = bucketKey("skill", skill);
+          const bucket = this.ensureBucket("skill", skill);
+          bucket.invocations += 1;
+          if (bucket.invokeContextDelta === null)
+            bucket.awaitingInvokeDelta = true;
+          this.current = key;
+        }
+      } else if (name === "Agent" || name === "Task") {
+        const subagent = toolUseInput(block).subagent_type;
+        const type =
+          typeof subagent === "string" && subagent.length > 0
+            ? subagent
+            : "unknown";
+        this.subagentByToolUseId.set(id, type);
+        this.ensureBucket("agent", type).invocations += 1;
+      }
+    }
+  }
+
+  /**
+   * Observe a usage-bearing assistant turn. The caller dedupes by message id
+   * (same contract as `_consumeStream`'s per-model accumulation).
+   */
+  observeUsage(
+    messageId: string,
+    model: string,
+    usage: TurnUsageLike,
+    parentToolUseId?: string,
+  ): void {
+    const input = usage.input_tokens ?? 0;
+    const output = usage.output_tokens ?? 0;
+    const cacheRead = usage.cache_read_input_tokens ?? 0;
+    const cacheCreation = usage.cache_creation_input_tokens ?? 0;
+
+    this.turns += 1;
+    if (this.firstTurn === undefined && parentToolUseId === undefined) {
+      this.firstTurn = {
+        model,
+        messageId,
+        inputTokens: input,
+        cacheCreationTokens: cacheCreation,
+        cacheReadTokens: cacheRead,
+        contextTokens: input + cacheCreation + cacheRead,
+      };
+    }
+
+    const bucket =
+      parentToolUseId !== undefined
+        ? this.ensureBucket(
+            "agent",
+            this.subagentByToolUseId.get(parentToolUseId) ?? "unknown",
+          )
+        : (this.buckets.get(this.current) ?? this.ensureBucket("root", "root"));
+
+    bucket.turns += 1;
+    bucket.inputTokens += input;
+    bucket.outputTokens += output;
+    bucket.cacheReadTokens += cacheRead;
+    bucket.cacheCreationTokens += cacheCreation;
+    if (bucket.awaitingInvokeDelta) {
+      bucket.invokeContextDelta = input + cacheCreation;
+      bucket.awaitingInvokeDelta = false;
+    }
+  }
+
+  get toolCalls(): number {
+    return this.seenToolUseIds.size;
+  }
+
+  snapshot(): RunTelemetry {
+    const skillUsage: SkillUsageRow[] = [];
+    for (const {
+      awaitingInvokeDelta: _pending,
+      ...row
+    } of this.buckets.values()) {
+      // Drop an untouched root bucket so a run that never produced a turn
+      // reports an empty attribution list rather than a zero row.
+      if (row.kind === "root" && row.turns === 0) continue;
+      skillUsage.push({ ...row });
+    }
+    return {
+      ...(this.firstTurn !== undefined && { firstTurn: { ...this.firstTurn } }),
+      turns: this.turns,
+      toolCalls: this.toolCalls,
+      skillUsage,
+    };
+  }
+
+  private ensureBucket(kind: AttributionKind, name: string): Bucket {
+    const key = bucketKey(kind, name);
+    let bucket = this.buckets.get(key);
+    if (!bucket) {
+      bucket = newBucket(kind, name);
+      this.buckets.set(key, bucket);
+    }
+    return bucket;
+  }
+}

package/run-telemetry.unit.test.ts

@@ -0,0 +1,235 @@
+import { describe, expect, test } from "bun:test";
+import type { ContentBlock } from "./progress-milestones.ts";
+import { RunTelemetryAccumulator } from "./run-telemetry.ts";
+
+const MODEL = "claude-sonnet-4-6";
+
+function usage(
+  input: number,
+  output: number,
+  cacheRead = 0,
+  cacheCreation = 0,
+) {
+  return {
+    input_tokens: input,
+    output_tokens: output,
+    cache_read_input_tokens: cacheRead,
+    cache_creation_input_tokens: cacheCreation,
+  };
+}
+
+function skillBlock(id: string, skill: string): ContentBlock {
+  return {
+    type: "tool_use",
+    id,
+    name: "Skill",
+    input: { skill },
+  } as ContentBlock;
+}
+
+function agentBlock(
+  id: string,
+  subagentType: string,
+  name: "Agent" | "Task" = "Agent",
+): ContentBlock {
+  return {
+    type: "tool_use",
+    id,
+    name,
+    input: { subagent_type: subagentType, prompt: "go" },
+  } as ContentBlock;
+}
+
+function readBlock(id: string): ContentBlock {
+  return { type: "tool_use", id, name: "Read", input: {} } as ContentBlock;
+}
+
+describe("RunTelemetryAccumulator — first-turn baseline", () => {
+  test("captures the first usage-bearing turn and sums all three input kinds", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeUsage("msg_1", MODEL, usage(2, 297, 37_771, 41_415));
+    acc.observeUsage("msg_2", MODEL, usage(500, 10, 79_000, 600));
+
+    const snap = acc.snapshot();
+    expect(snap.firstTurn).toEqual({
+      model: MODEL,
+      messageId: "msg_1",
+      inputTokens: 2,
+      cacheCreationTokens: 41_415,
+      cacheReadTokens: 37_771,
+      contextTokens: 2 + 41_415 + 37_771,
+    });
+    expect(snap.turns).toBe(2);
+  });
+
+  test("a sub-agent turn never becomes the first-turn baseline", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeContent([agentBlock("tu_agent", "Explore")]);
+    acc.observeUsage("sub_1", MODEL, usage(50, 5), "tu_agent");
+    acc.observeUsage("msg_1", MODEL, usage(1, 1, 100, 200));
+
+    expect(acc.snapshot().firstTurn?.messageId).toBe("msg_1");
+  });
+
+  test("a Task-named dispatch buckets and attributes sub-agent turns like an Agent-named one", () => {
+    const run = (name: "Agent" | "Task") => {
+      const acc = new RunTelemetryAccumulator();
+      acc.observeContent([agentBlock("tu_agent", "Explore", name)]);
+      acc.observeUsage("sub_1", MODEL, usage(50, 5), "tu_agent");
+      acc.observeUsage("msg_1", MODEL, usage(1, 1, 100, 200));
+      return acc.snapshot();
+    };
+
+    const taskSnap = run("Task");
+    expect(taskSnap).toEqual(run("Agent"));
+    expect(taskSnap.skillUsage.map((r) => r.name)).toContain("Explore");
+    expect(taskSnap.firstTurn?.messageId).toBe("msg_1");
+  });
+
+  test("snapshot of an empty run has no firstTurn, zero counts, and no rows", () => {
+    expect(new RunTelemetryAccumulator().snapshot()).toEqual({
+      turns: 0,
+      toolCalls: 0,
+      skillUsage: [],
+    });
+  });
+});
+
+describe("RunTelemetryAccumulator — tool calls", () => {
+  test("counts distinct tool_use ids across repeated lines of the same message", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeContent([readBlock("tu_1")]);
+    acc.observeContent([readBlock("tu_1"), readBlock("tu_2")]);
+    acc.observeContent(undefined);
+    acc.observeContent([{ type: "text", text: "hi" } as ContentBlock]);
+
+    expect(acc.toolCalls).toBe(2);
+    expect(acc.snapshot().toolCalls).toBe(2);
+  });
+});
+
+describe("RunTelemetryAccumulator — skill attribution", () => {
+  test("turns before any invoke accrue to root; later turns to the last invoked skill", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeUsage("msg_1", MODEL, usage(10, 1));
+    acc.observeContent([skillBlock("tu_skill", "shipwright:task-store")]);
+    acc.observeUsage("msg_2", MODEL, usage(20, 2, 0, 9_000));
+    acc.observeUsage("msg_3", MODEL, usage(30, 3));
+
+    const rows = acc.snapshot().skillUsage;
+    expect(rows).toEqual([
+      {
+        kind: "root",
+        name: "root",
+        invocations: 0,
+        turns: 1,
+        inputTokens: 10,
+        outputTokens: 1,
+        cacheReadTokens: 0,
+        cacheCreationTokens: 0,
+        invokeContextDelta: null,
+      },
+      {
+        kind: "skill",
+        name: "shipwright:task-store",
+        invocations: 1,
+        turns: 2,
+        inputTokens: 50,
+        outputTokens: 5,
+        cacheReadTokens: 0,
+        cacheCreationTokens: 9_000,
+        invokeContextDelta: 20 + 9_000,
+      },
+    ]);
+  });
+
+  test("a second invoke of the same skill counts as an invocation but keeps the first delta", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeContent([skillBlock("tu_a", "shipwright:review")]);
+    acc.observeUsage("msg_1", MODEL, usage(5, 1, 0, 100));
+    acc.observeContent([skillBlock("tu_b", "shipwright:review")]);
+    acc.observeUsage("msg_2", MODEL, usage(5, 1, 0, 999));
+
+    const [row] = acc.snapshot().skillUsage;
+    expect(row.name).toBe("shipwright:review");
+    expect(row.invocations).toBe(2);
+    expect(row.turns).toBe(2);
+    expect(row.invokeContextDelta).toBe(105);
+  });
+
+  test("a skill invoked with no usage-bearing turn afterwards has a null delta", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeContent([skillBlock("tu_a", "shipwright:deploy")]);
+    const [row] = acc.snapshot().skillUsage;
+    expect(row.invocations).toBe(1);
+    expect(row.turns).toBe(0);
+    expect(row.invokeContextDelta).toBeNull();
+  });
+
+  test("a Skill block without a skill name is counted as a tool call but not attributed", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeContent([
+      {
+        type: "tool_use",
+        id: "tu_x",
+        name: "Skill",
+        input: {},
+      } as ContentBlock,
+    ]);
+    acc.observeUsage("msg_1", MODEL, usage(1, 1));
+    const snap = acc.snapshot();
+    expect(snap.toolCalls).toBe(1);
+    expect(snap.skillUsage.map((r) => r.kind)).toEqual(["root"]);
+  });
+
+  test("switching skills charges later turns to the newest invoke only", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeContent([skillBlock("tu_a", "a")]);
+    acc.observeUsage("msg_1", MODEL, usage(1, 1));
+    acc.observeContent([skillBlock("tu_b", "b")]);
+    acc.observeUsage("msg_2", MODEL, usage(2, 2));
+    acc.observeUsage("msg_3", MODEL, usage(3, 3));
+
+    const byName = Object.fromEntries(
+      acc.snapshot().skillUsage.map((r) => [r.name, r]),
+    );
+    expect(byName.a.turns).toBe(1);
+    expect(byName.b.turns).toBe(2);
+    expect(byName.b.inputTokens).toBe(5);
+  });
+});
+
+describe("RunTelemetryAccumulator — sub-agent attribution", () => {
+  test("turns carrying parent_tool_use_id go to agent:<type> and leave the current skill alone", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeContent([skillBlock("tu_skill", "shipwright:patch")]);
+    acc.observeContent([agentBlock("tu_agent", "shipwright:code-reviewer")]);
+    acc.observeUsage("sub_1", MODEL, usage(100, 10), "tu_agent");
+    acc.observeUsage("sub_2", MODEL, usage(100, 10), "tu_agent");
+    acc.observeUsage("msg_2", MODEL, usage(7, 1, 0, 50));
+
+    const byKey = Object.fromEntries(
+      acc.snapshot().skillUsage.map((r) => [`${r.kind}:${r.name}`, r]),
+    );
+    expect(byKey["agent:shipwright:code-reviewer"]).toMatchObject({
+      invocations: 1,
+      turns: 2,
+      inputTokens: 200,
+      invokeContextDelta: null,
+    });
+    expect(byKey["skill:shipwright:patch"]).toMatchObject({
+      turns: 1,
+      inputTokens: 7,
+      invokeContextDelta: 57,
+    });
+    expect(acc.snapshot().turns).toBe(3);
+  });
+
+  test("a sub-agent turn whose parent tool_use was never seen lands in agent:unknown", () => {
+    const acc = new RunTelemetryAccumulator();
+    acc.observeUsage("sub_1", MODEL, usage(1, 1), "tu_missing");
+    expect(acc.snapshot().skillUsage).toEqual([
+      expect.objectContaining({ kind: "agent", name: "unknown", turns: 1 }),
+    ]);
+  });
+});