@saadnvd1/agent-os 0.3.470 → 0.3.472

package/lib/usage/check-runs.ts

@@ -0,0 +1,170 @@
+/**
+ * What the orchestrator's one-shot checks cost: each `claude -p` run of the
+ * independent reviewer and the scope check, read from its JSON result.
+ * They run with --no-session-persistence and leave no transcript, so this
+ * is the only record of their tokens.
+ */
+
+import { db } from "../db";
+import { formatTokens } from "../chat/context";
+
+export interface RunUsage {
+  model: string | null;
+  // Where the model ran (lib/models); Anthropic when absent.
+  provider?: string;
+  inputTokens: number;
+  cacheWriteTokens: number;
+  cacheReadTokens: number;
+  outputTokens: number;
+  costUsd: number;
+  durationMs: number | null;
+}
+
+interface ModelUsage {
+  inputTokens?: number;
+  outputTokens?: number;
+  cacheReadInputTokens?: number;
+  cacheCreationInputTokens?: number;
+  costUSD?: number;
+}
+
+const n = (v: unknown) => (typeof v === "number" && v > 0 ? v : 0);
+
+// The usage in a `claude -p --output-format json` result, or null when it
+// carries none. Per-model usage covers every model the run used; the model
+// named is the one that cost the most.
+export function runUsageOf(result: unknown): RunUsage | null {
+  if (!result || typeof result !== "object") return null;
+  const r = result as {
+    usage?: {
+      input_tokens?: number;
+      output_tokens?: number;
+      cache_read_input_tokens?: number;
+      cache_creation_input_tokens?: number;
+    };
+    modelUsage?: Record<string, ModelUsage>;
+    total_cost_usd?: number;
+    duration_ms?: number;
+  };
+  const models = Object.entries(r.modelUsage ?? {});
+  if (!r.usage && !models.length) return null;
+  const durationMs = n(r.duration_ms) || null;
+  if (models.length) {
+    const sum = (k: keyof ModelUsage) =>
+      models.reduce((t, [, m]) => t + n(m[k]), 0);
+    const [model] = models.reduce((a, b) =>
+      n(b[1].costUSD) > n(a[1].costUSD) ? b : a
+    );
+    return {
+      model,
+      inputTokens: sum("inputTokens"),
+      cacheWriteTokens: sum("cacheCreationInputTokens"),
+      cacheReadTokens: sum("cacheReadInputTokens"),
+      outputTokens: sum("outputTokens"),
+      costUsd: n(r.total_cost_usd) || sum("costUSD"),
+      durationMs,
+    };
+  }
+  const u = r.usage!;
+  return {
+    model: null,
+    inputTokens: n(u.input_tokens),
+    cacheWriteTokens: n(u.cache_creation_input_tokens),
+    cacheReadTokens: n(u.cache_read_input_tokens),
+    outputTokens: n(u.output_tokens),
+    costUsd: n(r.total_cost_usd),
+    durationMs,
+  };
+}
+
+export interface CheckRunTarget {
+  workspaceId: string;
+  // The task's session id, or an external PR's id.
+  sessionId: string;
+  // How the Usage page names it: "task add-a", "PR o/r#5".
+  label: string;
+  prNumber: number | null;
+  sha: string;
+  kind: "review" | "scope";
+}
+
+export function recordCheckRun(
+  target: CheckRunTarget,
+  u: RunUsage,
+  at = Date.now()
+): void {
+  db.prepare(
+    `INSERT INTO check_runs (workspace_id, session_id, label, pr_number, sha, kind, model, provider,
+       input_tokens, cache_write_tokens, cache_read_tokens, output_tokens, cost_usd, duration_ms, at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+  ).run(
+    target.workspaceId,
+    target.sessionId,
+    target.label,
+    target.prNumber,
+    target.sha,
+    target.kind,
+    u.model,
+    u.provider ?? "anthropic",
+    u.inputTokens,
+    u.cacheWriteTokens,
+    u.cacheReadTokens,
+    u.outputTokens,
+    u.costUsd,
+    u.durationMs,
+    at
+  );
+}
+
+// A recorder for one check, that never lets a write break the check.
+export function checkRunRecorder(target: CheckRunTarget) {
+  return (u: RunUsage) => {
+    try {
+      recordCheckRun(target, u);
+    } catch (e) {
+      console.error(`[check-runs] recording ${target.label}:`, e);
+    }
+  };
+}
+
+export interface CheckRunTotals {
+  runs: number;
+  tokens: number;
+  costUsd: number;
+  durationMs: number;
+  models: string[];
+}
+
+// Everything the checks of one commit cost, every part and retry.
+export function checkRunTotals(
+  sessionId: string,
+  sha: string
+): CheckRunTotals | null {
+  const row = db
+    .prepare(
+      `SELECT count(*) AS runs,
+         coalesce(sum(input_tokens + cache_write_tokens + cache_read_tokens + output_tokens), 0) AS tokens,
+         coalesce(sum(cost_usd), 0) AS costUsd,
+         coalesce(sum(duration_ms), 0) AS durationMs,
+         group_concat(DISTINCT model) AS models
+       FROM check_runs WHERE session_id = ? AND sha = ?`
+    )
+    .get(sessionId, sha) as Omit<CheckRunTotals, "models"> & {
+    models: string | null;
+  };
+  if (!row.runs) return null;
+  return { ...row, models: row.models ? row.models.split(",") : [] };
+}
+
+// "2 runs · 412k tokens · $0.84 · 3m 10s · claude-opus-5-5"
+export function describeCheckRuns(t: CheckRunTotals): string {
+  const s = Math.round(t.durationMs / 1000);
+  const time = s >= 60 ? `${Math.floor(s / 60)}m ${s % 60}s` : `${s}s`;
+  return [
+    `${t.runs} run${t.runs === 1 ? "" : "s"}`,
+    `${formatTokens(t.tokens)} tokens`,
+    `$${t.costUsd.toFixed(2)}`,
+    time,
+    ...t.models,
+  ].join(" · ");
+}

package/lib/usage/compare.ts

@@ -0,0 +1,280 @@
+/**
+ * Claude-only tasks against routed ones, side by side: what each cost
+ * (tokens, dollars at list prices, share of the plan's window, wall time)
+ * and how it went (CI green on the first push, the first review passing,
+ * gate failures, commits after the first review, merged or dropped). Built
+ * on the usage ledgers (chat_turns, and check_runs for the gate's own runs)
+ * and the orchestrator's checks: no record of its own.
+ *
+ * A task's cohort is the provider its own turns (planning and implementing)
+ * ran on: anthropic is the baseline; one that used two is "mixed". Its
+ * reviews and scope checks count toward its cost, whatever they ran on.
+ */
+
+import { db } from "../db";
+import { effectiveModels } from "../models/config";
+import { ANTHROPIC } from "../models/roles";
+import type { ModelPrice } from "../models/providers";
+import { planShares, rowModel, UNRECORDED } from "./cost";
+import {
+  addRow,
+  emptySpend,
+  type ModelTurnRow,
+  type TokenSpend,
+} from "./by-model";
+
+export const SMALL_SAMPLE = 10;
+const CHECK_ROLES = new Set(["review", "scope", "helper"]);
+
+export interface CompareTurn extends ModelTurnRow {
+  session_id: string;
+  duration_ms: number | null;
+  window_pct: number | null;
+}
+
+export interface TaskOutcome {
+  id: string;
+  status: string | null;
+  // The first finished CI run, and review, on its PR: passed or not; null
+  // when it never had one (no PR, or not under an orchestrator).
+  ciFirstGreen: boolean | null;
+  reviewFirstPass: boolean | null;
+  gateFailures: number;
+  // Heads checked after its first review: the rework the review caused.
+  rework: number;
+}
+
+export interface Rate {
+  yes: number;
+  of: number;
+}
+
+export interface Cohort {
+  key: string;
+  baseline: boolean;
+  tasks: number;
+  smallSample: boolean;
+  spend: TokenSpend;
+  // Points of the plan's 5-hour window (an estimate), Anthropic turns only.
+  planPct: number;
+  wallMs: number;
+  ciFirstGreen: Rate;
+  reviewFirstPass: Rate;
+  gateFailures: number;
+  rework: number;
+  merged: Rate;
+  dropped: Rate;
+}
+
+export interface RoleModelRow extends TokenSpend {
+  role: string;
+  provider: string;
+  model: string;
+  planPct: number;
+  wallMs: number;
+}
+
+export interface Comparison {
+  from: number;
+  to: number;
+  cohorts: Cohort[];
+  roles: RoleModelRow[];
+}
+
+const rate = (): Rate => ({ yes: 0, of: 0 });
+const count = (r: Rate, v: boolean | null) => {
+  if (v === null) return;
+  r.of += 1;
+  if (v) r.yes += 1;
+};
+
+function cohortOf(turns: CompareTurn[]): string {
+  const providers = new Set(
+    turns
+      .filter((t) => !CHECK_ROLES.has(t.role ?? ""))
+      .map((t) => rowModel(t).provider)
+  );
+  if (providers.size === 0) return ANTHROPIC;
+  return providers.size === 1 ? [...providers][0] : "mixed";
+}
+
+export function compareRuns(
+  turns: CompareTurn[],
+  outcomes: Map<string, TaskOutcome>,
+  prices: Record<string, ModelPrice>,
+  range: { from: number; to: number }
+): Comparison {
+  const shares = planShares(turns);
+  const share = new Map(turns.map((t, i) => [t, shares[i]]));
+  const inRange = turns.filter((t) => t.at >= range.from && t.at <= range.to);
+
+  const byTask = new Map<string, CompareTurn[]>();
+  for (const t of inRange)
+    if (outcomes.has(t.session_id))
+      byTask.set(t.session_id, [...(byTask.get(t.session_id) ?? []), t]);
+
+  const cohorts = new Map<string, Cohort>();
+  for (const [id, ts] of byTask) {
+    const key = cohortOf(ts);
+    let c = cohorts.get(key);
+    if (!c)
+      cohorts.set(
+        key,
+        (c = {
+          key,
+          baseline: key === ANTHROPIC,
+          tasks: 0,
+          smallSample: true,
+          spend: emptySpend(),
+          planPct: 0,
+          wallMs: 0,
+          ciFirstGreen: rate(),
+          reviewFirstPass: rate(),
+          gateFailures: 0,
+          rework: 0,
+          merged: rate(),
+          dropped: rate(),
+        })
+      );
+    const o = outcomes.get(id)!;
+    c.tasks += 1;
+    for (const t of ts) {
+      addRow(c.spend, t, prices);
+      c.planPct += share.get(t) ?? 0;
+      c.wallMs += t.duration_ms ?? 0;
+    }
+    count(c.ciFirstGreen, o.ciFirstGreen);
+    count(c.reviewFirstPass, o.reviewFirstPass);
+    c.gateFailures += o.gateFailures;
+    c.rework += o.rework;
+    // Still running counts toward neither.
+    const finished = o.status !== "running" && o.status !== null;
+    count(c.merged, finished ? o.status === "merged" : null);
+    count(c.dropped, finished ? o.status === "dropped" : null);
+  }
+  for (const c of cohorts.values()) c.smallSample = c.tasks < SMALL_SAMPLE;
+
+  const roles = new Map<string, RoleModelRow>();
+  for (const t of inRange) {
+    const { provider, model } = rowModel(t);
+    const role = t.role ?? UNRECORDED;
+    const k = `${role} ${provider}/${model}`;
+    let r = roles.get(k);
+    if (!r)
+      roles.set(
+        k,
+        (r = { ...emptySpend(), role, provider, model, planPct: 0, wallMs: 0 })
+      );
+    addRow(r, t, prices);
+    r.planPct += share.get(t) ?? 0;
+    r.wallMs += t.duration_ms ?? 0;
+  }
+
+  return {
+    ...range,
+    // The baseline first, then the most tasks.
+    cohorts: [...cohorts.values()].sort(
+      (a, b) => Number(b.baseline) - Number(a.baseline) || b.tasks - a.tasks
+    ),
+    roles: [...roles.values()].sort(
+      (a, b) => a.role.localeCompare(b.role) || b.costUsd - a.costUsd
+    ),
+  };
+}
+
+// The ledger's turns, with the gate reviews and scope checks recorded in
+// check_runs (lib/usage/check-runs) as turns of their own role.
+export const LEDGER_SQL = `
+  SELECT session_id, at, cost_usd, input_tokens, output_tokens,
+    cache_read_tokens, cache_write_tokens, model, provider, role, duration_ms,
+    window_pct FROM chat_turns WHERE at >= @from AND at <= @to
+  UNION ALL
+  SELECT session_id, at, cost_usd, input_tokens, output_tokens,
+    cache_read_tokens, cache_write_tokens, model, provider, kind AS role,
+    duration_ms, NULL AS window_pct FROM check_runs WHERE at >= @from AND at <= @to
+  ORDER BY at`;
+
+// The outcome of each task with turns in the range.
+export function taskOutcomes(ids: string[]): Map<string, TaskOutcome> {
+  const out = new Map<string, TaskOutcome>();
+  if (!ids.length) return out;
+  const marks = ids.map(() => "?").join(",");
+  const tasks = db
+    .prepare(
+      `SELECT id, task_status FROM sessions
+       WHERE id IN (${marks}) AND task_status IS NOT NULL`
+    )
+    .all(...ids) as { id: string; task_status: string }[];
+  const checks = db
+    .prepare(
+      `SELECT id, session_id, sha, kind, status FROM orchestrator_checks
+       WHERE session_id IN (${marks}) ORDER BY id`
+    )
+    .all(...ids) as {
+    id: number;
+    session_id: string;
+    sha: string;
+    kind: string;
+    status: string;
+  }[];
+  const gates = db
+    .prepare(
+      `SELECT session_id, SUM(count) AS n FROM orchestrator_gate_failures
+       WHERE session_id IN (${marks}) GROUP BY session_id`
+    )
+    .all(...ids) as { session_id: string; n: number }[];
+  for (const t of tasks)
+    out.set(t.id, {
+      id: t.id,
+      status: t.task_status,
+      ciFirstGreen: null,
+      reviewFirstPass: null,
+      gateFailures: 0,
+      rework: 0,
+    });
+  const firstReview = new Map<string, number>();
+  const reworked = new Map<string, Set<string>>();
+  for (const c of checks) {
+    const o = out.get(c.session_id);
+    if (!o) continue;
+    const settled = c.status === "pass" || c.status === "block";
+    if (c.kind === "ci" && settled && o.ciFirstGreen === null)
+      o.ciFirstGreen = c.status === "pass";
+    if (c.kind === "review" && settled && o.reviewFirstPass === null) {
+      o.reviewFirstPass = c.status === "pass";
+      firstReview.set(c.session_id, c.id);
+    }
+    const after = firstReview.get(c.session_id);
+    if (after !== undefined && c.id > after) {
+      const shas = reworked.get(c.session_id) ?? new Set();
+      shas.add(c.sha);
+      reworked.set(c.session_id, shas);
+    }
+  }
+  for (const [id, shas] of reworked) {
+    // The reviewed head itself isn't rework.
+    const o = out.get(id)!;
+    const reviewed = checks.find((c) => c.id === firstReview.get(id))?.sha;
+    o.rework = [...shas].filter((s) => s !== reviewed).length;
+  }
+  for (const g of gates) {
+    const o = out.get(g.session_id);
+    if (o) o.gateFailures = g.n;
+  }
+  return out;
+}
+
+export function comparisonReport(from: number, to: number): Comparison {
+  // From a day earlier, so the first turn in range has a reading before it
+  // to measure its share of the window from.
+  const turns = db
+    .prepare(LEDGER_SQL)
+    .all({ from: from - 24 * 3600 * 1000, to }) as CompareTurn[];
+  const ids = [
+    ...new Set(turns.filter((t) => t.at >= from).map((t) => t.session_id)),
+  ];
+  return compareRuns(turns, taskOutcomes(ids), effectiveModels().prices, {
+    from,
+    to,
+  });
+}

package/lib/usage/cost.ts

@@ -0,0 +1,71 @@
+/**
+ * What a turn would cost at a provider's list prices (the price table in
+ * Settings > Models). For Anthropic that's the API's price, not what the
+ * plan charges: the plan's own measure is its share of the 5-hour window.
+ */
+
+import { ANTHROPIC_ALIASES, type ModelPrice } from "../models/providers";
+import { ANTHROPIC } from "../models/roles";
+
+export interface PricedRow {
+  provider: string | null;
+  model: string | null;
+  cost_usd: number;
+  input_tokens: number;
+  output_tokens: number;
+  cache_read_tokens: number;
+  cache_write_tokens: number;
+}
+
+export const UNRECORDED = "unrecorded";
+
+// The row's provider and model as the table names them: turns from before
+// routing have neither, and ran on Anthropic.
+export function rowModel(r: Pick<PricedRow, "provider" | "model">): {
+  provider: string;
+  model: string;
+} {
+  const provider = r.provider ?? ANTHROPIC;
+  const raw = r.model ?? UNRECORDED;
+  const model = provider === ANTHROPIC ? (ANTHROPIC_ALIASES[raw] ?? raw) : raw;
+  return { provider, model };
+}
+
+// Dollars, and whether it came from the table (false: the agent's own
+// figure, for a model the table doesn't price).
+export function estimateCost(
+  r: PricedRow,
+  prices: Record<string, ModelPrice>
+): { usd: number; priced: boolean } {
+  const { provider, model } = rowModel(r);
+  const p = prices[`${provider}/${model}`];
+  if (!p) return { usd: r.cost_usd, priced: false };
+  const usd =
+    (r.input_tokens * p.input +
+      r.cache_write_tokens * p.cacheWrite +
+      r.cache_read_tokens * p.cacheRead +
+      r.output_tokens * p.output) /
+    1_000_000;
+  return { usd, priced: true };
+}
+
+// Each Anthropic turn's share of the plan's 5-hour window, in points: how
+// far the window moved since the turn before it (any session's) ended.
+// Turns overlap, so one turn's share is rough; summed over many it isn't.
+// A drop means the window reset, and all of the new reading is this turn's;
+// the first reading has nothing to measure from.
+export function planShares(
+  rows: { at: number; provider: string | null; window_pct: number | null }[]
+): number[] {
+  const order = rows.map((r, i) => ({ r, i })).sort((a, b) => a.r.at - b.r.at);
+  const out = new Array<number>(rows.length).fill(0);
+  let prev: number | null = null;
+  for (const { r, i } of order) {
+    if ((r.provider ?? ANTHROPIC) !== ANTHROPIC || r.window_pct == null)
+      continue;
+    const pct = r.window_pct;
+    out[i] = prev === null ? 0 : pct < prev ? pct : pct - prev;
+    prev = pct;
+  }
+  return out;
+}

package/lib/usage/report.ts

@@ -1,4 +1,7 @@
 import { db } from "../db";
+import { effectiveModels } from "../models/config";
+import { LEDGER_SQL } from "./compare";
+import { spendByModel, type ModelTurnRow, type SpendByModel } from "./by-model";
 import { listWorkspaces } from "../workspaces";
 import {
   aggregateUsage,
@@ -18,3 +21,51 @@ export function usageReport(range: UsageRange, now = Date.now()) {
   const names = Object.fromEntries(listWorkspaces().map((w) => [w.id, w.name]));
   return aggregateUsage(rows, range, names, now);
 }
+
+// What the orchestrator's reviewer and scope check cost over the range
+// (lib/usage/check-runs), by PR: a "turn" is one run.
+export function checkRunReport(range: UsageRange, now = Date.now()) {
+  const rows = db
+    .prepare(
+      `SELECT session_id, label, pr_number, workspace_id, at, cost_usd, input_tokens,
+         output_tokens, cache_read_tokens, cache_write_tokens
+       FROM check_runs WHERE at >= ? ORDER BY at`
+    )
+    .all(rangeStart(range, now)) as (Omit<TurnRow, "session_name"> & {
+    label: string;
+    pr_number: number | null;
+  })[];
+  const named: TurnRow[] = rows.map(({ label, pr_number, ...r }) => ({
+    ...r,
+    // An external PR's label already names it.
+    session_name:
+      pr_number === null || label.startsWith("PR ")
+        ? label
+        : `${label} · PR #${pr_number}`,
+  }));
+  const names = Object.fromEntries(listWorkspaces().map((w) => [w.id, w.name]));
+  return aggregateUsage(named, range, names, now);
+}
+
+const HOUR = 3600 * 1000;
+
+// Spend by model and role in the plan's current 5-hour window (its start
+// from when it resets, or the last five hours) and the last 24 hours.
+export function modelUsage(
+  fiveHourResetsIn: number | null | undefined,
+  now = Date.now()
+): { fiveHour: SpendByModel; day: SpendByModel } {
+  const day = now - 24 * HOUR;
+  const fiveHour =
+    fiveHourResetsIn != null
+      ? now + fiveHourResetsIn * 1000 - 5 * HOUR
+      : now - 5 * HOUR;
+  const rows = db
+    .prepare(LEDGER_SQL)
+    .all({ from: Math.min(day, fiveHour), to: now }) as ModelTurnRow[];
+  const { prices } = effectiveModels();
+  return {
+    fiveHour: spendByModel(rows, fiveHour, prices),
+    day: spendByModel(rows, day, prices),
+  };
+}