catherd-cli 0.2.1 → 1.0.0

package/src/services/summary.ts

@@ -0,0 +1,227 @@
+import { resolve } from "node:path";
+import { type BudgetStatus, budgetStatus } from "../domain/budget.ts";
+import { isCatherdError } from "../domain/errors.ts";
+import type { Tokens } from "../domain/record.ts";
+import { median } from "../domain/util.ts";
+import { nonBlankLines, readJsonl } from "../infra/store.ts";
+import { spendOf } from "./budget.ts";
+import { type DispatchState, liveDispatches } from "./dispatches.ts";
+import type { Deps } from "./ports.ts";
+import {
+  findRun,
+  listRuns,
+  readAgentRuns,
+  readRecords,
+  readRoutes,
+  type Run,
+  runPaths,
+} from "./run-store.ts";
+
+export interface RunSummary {
+  id: string;
+  title: string;
+  repo: string;
+  createdAt: string;
+  stateTail: string[];
+  live: { name: string; rung: string; state: DispatchState; secs: number; dispatchId: string }[];
+  totals: { runs: number; ok: number; notOk: string[]; tokens: Tokens; costUsd: number; wallMinutes: number };
+  /** native subagent runs, as the orchestrator reported them: reported, not measured (spec §14) */
+  agents: { runs: number; totalTokens: number; costUsd: number };
+  jev: { decisions: number; fallbacks: number };
+  /** per backend, how many of this run's dispatches ran with the user's harness (native) and isolated */
+  harness: { backend: string; native: number; isolated: number }[];
+  budget: BudgetStatus | null;
+  milestones: string[];
+  warnings: string[];
+}
+
+/** One run at a glance. Spec §3.4: reads are pure; nothing here finalizes a dispatch or writes a file. */
+export function summarizeRun(deps: Deps, run: Run): RunSummary {
+  const now = deps.now();
+  const { records, corrupt } = readRecords(run);
+  const live = liveDispatches(run, now);
+  const agents = readAgentRuns(run);
+  const jev = readJsonl<{ source?: string }>(runPaths(run.dir).jev).rows;
+  const sum = (f: (t: Tokens) => number) => records.reduce((n, r) => n + f(r.tokens), 0);
+  let budget: BudgetStatus | null = null;
+  const warnings = corrupt ? [`runs.jsonl: skipped ${corrupt} unreadable row(s)`] : [];
+  try {
+    budget = budgetStatus(spendOf(run, records, live, now), deps.profiles.forRepo(run.meta.repo).budget);
+  } catch (e) {
+    warnings.push(`budget: ${isCatherdError(e) ? e.message : String(e)}`);
+  }
+  return {
+    id: run.id,
+    title: run.meta.title,
+    repo: run.meta.repo,
+    createdAt: run.meta.createdAt,
+    stateTail: nonBlankLines(runPaths(run.dir).state).slice(-3),
+    live: live.map((d) => ({
+      name: d.admit.name,
+      rung: d.admit.rung,
+      state: d.state,
+      secs: Math.max(0, Math.round((now - Date.parse(d.admit.admittedAt)) / 1000)),
+      dispatchId: d.admit.dispatchId,
+    })),
+    totals: {
+      runs: records.length,
+      ok: records.filter((r) => r.status === "ok").length,
+      notOk: records.filter((r) => r.status !== "ok").map((r) => `${r.name} (${r.status})`),
+      tokens: { input: sum((t) => t.input), cached: sum((t) => t.cached), output: sum((t) => t.output) },
+      costUsd: records.reduce((n, r) => n + (r.costUsd ?? 0), 0),
+      wallMinutes: Math.round((now - Date.parse(run.meta.createdAt)) / 60_000),
+    },
+    agents: {
+      runs: agents.length,
+      totalTokens: agents.reduce((n, a) => n + a.totalTokens, 0),
+      costUsd: agents.reduce((n, a) => n + (a.costUsd ?? 0), 0),
+    },
+    // a jev-kind, lane or default source is a decision Jev did not make whole
+    jev: { decisions: jev.length, fallbacks: jev.filter((j) => j.source !== "jev").length },
+    harness: [...new Set(records.map((r) => r.backend))].sort().map((backend) => {
+      const of = records.filter((r) => r.backend === backend);
+      return {
+        backend,
+        native: of.filter((r) => !r.isolated).length,
+        isolated: of.filter((r) => r.isolated).length,
+      };
+    }),
+    budget,
+    milestones: nonBlankLines(runPaths(run.dir).ledger).slice(1),
+    warnings,
+  };
+}
+
+/** `status(run?)`: that run; without one, every run with a live role, else the newest run. */
+export function status(
+  deps: Deps,
+  runId?: string,
+): { version: string; runs: RunSummary[]; warnings: string[] } {
+  if (runId) return { version: deps.version, runs: [summarizeRun(deps, findRun(runId))], warnings: [] };
+  const { runs, corrupt } = listRuns();
+  const warnings = corrupt.map((c) => `skipped run ${c.id}: ${c.reason}`);
+  const all = runs.map((r) => summarizeRun(deps, r));
+  const live = all.filter((s) => s.live.length > 0);
+  return { version: deps.version, runs: live.length ? live : all.slice(0, 1), warnings };
+}
+
+export interface RungStats {
+  role: string;
+  rung: string;
+  runs: number;
+  ok: number;
+  refusals: number;
+  climbsFrom: number;
+  secs: number;
+  tokens: Tokens;
+}
+
+export interface HarnessCost {
+  backend: string;
+  nativeRuns: number;
+  isolatedRuns: number;
+  nativeMedian: number | null;
+  isolatedMedian: number | null;
+  extraPerRun: number | null;
+}
+
+/** A median in whole tokens: an even count's mean of the two middle values is rounded. */
+function tokenMedian(xs: number[]): number | null {
+  const m = median(xs);
+  return m === null ? null : Math.round(m);
+}
+
+/** Fewer first-turn samples than this on either side, and the harness line gives no figure. */
+const HARNESS_MIN_RUNS = 3;
+
+/**
+ * What the user's own harness setup costs per run: median first-turn input, native against isolated.
+ * No figure below HARNESS_MIN_RUNS a side, nor when native is not dearer: noise, not a cost.
+ */
+function harnessCosts(runs: Run[]): HarnessCost[] {
+  type Row = { backend: string; isolated: boolean; firstTurnInput: number };
+  const rows = runs.flatMap((r) => readJsonl<Row>(runPaths(r.dir).harness).rows);
+  return [...new Set(rows.map((r) => r.backend))].sort().map((backend) => {
+    const pick = (isolated: boolean) =>
+      rows.filter((r) => r.backend === backend && r.isolated === isolated).map((r) => r.firstTurnInput);
+    const nativeRuns = pick(false).length;
+    const isolatedRuns = pick(true).length;
+    const native = tokenMedian(pick(false));
+    const isolated = tokenMedian(pick(true));
+    const enough = nativeRuns >= HARNESS_MIN_RUNS && isolatedRuns >= HARNESS_MIN_RUNS;
+    const extra = native !== null && isolated !== null ? native - isolated : 0;
+    return {
+      backend,
+      nativeRuns,
+      isolatedRuns,
+      nativeMedian: native,
+      isolatedMedian: isolated,
+      extraPerRun: enough && extra > 0 ? extra : null,
+    };
+  });
+}
+
+/** Per role and rung over past runs: runs, ok, refusals, climbs from it, seconds and tokens. Reads only. */
+export function runsSummary(f: { run?: string; repo?: string; role?: string; sinceDays?: number }): {
+  rungs: RungStats[];
+  agents: { rung: string; runs: number; totalTokens: number; secs: number }[];
+  harness: HarnessCost[];
+} {
+  const since = f.sinceDays ? Date.now() - f.sinceDays * 86_400_000 : 0;
+  const repo = f.repo ? resolve(f.repo) : null;
+  const all = listRuns().runs.filter(
+    (r) => (!repo || r.meta.repo === repo) && Date.parse(r.meta.createdAt) >= since,
+  );
+  const runs = all.filter((r) => !f.run || r.id === f.run);
+  // with a run given, the harness line compares like with like: every run of that run's repo
+  const harnessRepo = f.run ? runs[0]?.meta.repo : undefined;
+  const harnessRuns = f.run ? all.filter((r) => r.meta.repo === harnessRepo) : runs;
+  const stats = new Map<string, RungStats>();
+  const at = (role: string, rung: string): RungStats => {
+    const key = `${role} ${rung}`;
+    let s = stats.get(key);
+    if (!s) {
+      s = {
+        role,
+        rung,
+        runs: 0,
+        ok: 0,
+        refusals: 0,
+        climbsFrom: 0,
+        secs: 0,
+        tokens: { input: 0, cached: 0, output: 0 },
+      };
+      stats.set(key, s);
+    }
+    return s;
+  };
+  const agents = new Map<string, { rung: string; runs: number; totalTokens: number; secs: number }>();
+  for (const run of runs) {
+    for (const r of readRecords(run).records) {
+      if (f.role && r.role !== f.role) continue;
+      const s = at(r.role, r.rung);
+      s.runs++;
+      if (r.status === "ok") s.ok++;
+      if (r.replyStatus === "refused" || r.replyStatus === "blocked") s.refusals++;
+      s.secs += r.secs;
+      s.tokens.input += r.tokens.input;
+      s.tokens.cached += r.tokens.cached;
+      s.tokens.output += r.tokens.output;
+    }
+    for (const c of readRoutes(run))
+      if (c.source === "climb" && c.from && (!f.role || c.role === f.role)) at(c.role, c.from).climbsFrom++;
+    for (const a of readAgentRuns(run)) {
+      if (f.role && a.role !== f.role) continue;
+      const s = agents.get(a.rung) ?? { rung: a.rung, runs: 0, totalTokens: 0, secs: 0 };
+      s.runs++;
+      s.totalTokens += a.totalTokens;
+      s.secs += a.secs ?? 0;
+      agents.set(a.rung, s);
+    }
+  }
+  return {
+    rungs: [...stats.values()].sort((a, b) => a.role.localeCompare(b.role) || a.rung.localeCompare(b.rung)),
+    agents: [...agents.values()].sort((a, b) => a.rung.localeCompare(b.rung)),
+    harness: harnessCosts(harnessRuns),
+  };
+}

package/catalog/catalog.json

@@ -1,112 +0,0 @@
-{
-  "version": "2026-09-24",
-  "sources": {
-    "repo_code": "DeepSWE-style resolution of real repository issues, per model and effort, from the published results collected in spec §5 on 2026-09-24",
-    "terminal": "Terminal-Bench-style terminal, ops and environment work, per model and effort, from the same collection",
-    "honesty": "100 minus the rate of failing to disclose a broken tool, from the system cards: gpt-6-luna 71.3, gpt-6-sol 95.1, claude-opus-5-5 not published",
-    "secs_per_task": "median seconds per task in the same benchmark runs",
-    "costRank": "catherd's own order of what one run costs a user on a ChatGPT plan and a Claude plan, lowest first; subscription quota counts"
-  },
-  "notes": {
-    "gpt-6-sol": "context 400000 is unverified for the ChatGPT-login id",
-    "gpt-6-luna": "context 400000 is unverified for the ChatGPT-login id",
-    "claude-opus-5-5": "context 1000000 is unverified",
-    "unscored on purpose": "gpt-6-luna#xhigh was dominated; gpt-6-luna#max took 958 s a task, slower than gpt-6-sol#xhigh at an equal score"
-  },
-  "models": [
-    {
-      "id": "gpt-6-luna",
-      "backend": "codex",
-      "efforts": ["low", "medium", "high", "xhigh", "max"],
-      "capabilities": {
-        "toolCall": true,
-        "imageIn": true,
-        "imageOut": true,
-        "reasoning": true,
-        "context": 400000
-      }
-    },
-    {
-      "id": "gpt-6-sol",
-      "backend": "codex",
-      "efforts": ["low", "medium", "high", "xhigh"],
-      "capabilities": {
-        "toolCall": true,
-        "imageIn": true,
-        "imageOut": true,
-        "reasoning": true,
-        "context": 400000
-      }
-    },
-    {
-      "id": "claude-opus-5-5",
-      "backend": "claude",
-      "efforts": ["low", "medium", "high"],
-      "capabilities": {
-        "toolCall": true,
-        "imageIn": true,
-        "imageOut": false,
-        "reasoning": true,
-        "context": 1000000
-      }
-    }
-  ],
-  "entries": [
-    {
-      "rung": "gpt-6-luna#high",
-      "scores": { "repo_code": 59.3, "terminal": 4.5, "honesty": 71.3, "secs_per_task": 563 },
-      "costRank": 1
-    },
-    {
-      "rung": "gpt-6-sol#medium",
-      "scores": { "repo_code": 56.6, "terminal": 18.7, "honesty": 95.1, "secs_per_task": 264 },
-      "costRank": 2
-    },
-    {
-      "rung": "gpt-6-sol#high",
-      "scores": { "repo_code": 65.3, "terminal": 26.3, "honesty": 95.1, "secs_per_task": 391 },
-      "costRank": 3
-    },
-    {
-      "rung": "gpt-6-sol#xhigh",
-      "scores": { "repo_code": 66.6, "terminal": 30.3, "honesty": 95.1, "secs_per_task": 496 },
-      "costRank": 4
-    },
-    { "rung": "claude-opus-5-5#low", "scores": { "terminal": 31.3 }, "costRank": 5 },
-    { "rung": "claude-opus-5-5#medium", "scores": { "terminal": 52.5 }, "costRank": 6 },
-    { "rung": "claude-opus-5-5#high", "scores": { "terminal": 56.6 }, "costRank": 7 }
-  ],
-  "bars": {
-    "repo_code": {
-      "copy": { "repo_code": 55 },
-      "build": { "repo_code": 55 },
-      "logic": { "repo_code": 55, "terminal": 15 },
-      "hard": { "repo_code": 56, "terminal": 18 }
-    },
-    "terminal": {
-      "copy": { "terminal": 15 },
-      "build": { "terminal": 15 },
-      "logic": { "terminal": 15 },
-      "hard": { "terminal": 15 }
-    },
-    "ui": {
-      "copy": { "repo_code": 55 },
-      "build": { "repo_code": 55 },
-      "logic": { "repo_code": 55, "terminal": 15 },
-      "hard": { "repo_code": 56, "terminal": 18 }
-    },
-    "prose": {
-      "copy": { "repo_code": 55 },
-      "build": { "repo_code": 55 },
-      "logic": { "repo_code": 55, "terminal": 15 },
-      "hard": { "repo_code": 56, "terminal": 18 }
-    },
-    "research": {
-      "copy": { "repo_code": 55 },
-      "build": { "repo_code": 55 },
-      "logic": { "repo_code": 55, "terminal": 15 },
-      "hard": { "repo_code": 56, "terminal": 18 }
-    }
-  },
-  "treatLike": {}
-}