@saadnvd1/agent-os 0.3.471 → 0.3.473

package/lib/tasks/index.ts

@@ -4,6 +4,14 @@
  * drops it.
  */
 
+import {
+  parseOverride,
+  resolveRole,
+  sessionModelRef,
+  type ModelOverride,
+} from "../models/resolve";
+import { effectiveModels } from "../models/config";
+import { ANTHROPIC } from "../models/roles";
 import { processTable, runsSomething } from "../process-table";
 import { randomUUID } from "crypto";
 import { db, queries, type Session } from "../db";
@@ -19,7 +27,6 @@ import {
   isBranchName,
   slugify,
 } from "../git";
-import { resolveModelForAgent } from "../model-catalog";
 import { getProvider } from "../providers";
 import { statusDetector } from "../status-detector";
 import { buildTaskBrief, type StackedOn } from "./brief";
@@ -85,6 +92,10 @@ export interface TaskView {
   hostError: string | null;
   // Merges by itself once every gate passes (orchestrator/armed.ts).
   armed?: boolean;
+  // What it runs on now, "provider/model", and whether it's still in its
+  // plan step (lib/models).
+  model?: string;
+  planning?: boolean;
 }
 
 export async function createTask(opts: {
@@ -92,7 +103,12 @@ export async function createTask(opts: {
   prompt: string;
   // Its name; generated from the prompt when absent.
   name?: string;
-  model?: string;
+  // Its own model: "provider/model" (or an Anthropic alias) for all its
+  // roles, or a map of role to one (lib/models). Its workspace's and
+  // Settings' role models otherwise.
+  model?: ModelOverride;
+  // Plan first on the planning model (Settings' default otherwise).
+  plan?: boolean;
   // Started from this LumifyHub card, which the task then moves.
   cardId?: string;
   // Stacked: cut from another task's pushed branch, at this exact commit.
@@ -169,7 +185,7 @@ async function startTask(
       id,
       prompt,
       name: opts.name,
-      model: opts.model,
+      model: typeof opts.model === "string" ? opts.model : undefined,
       baseBranch: opts.baseBranch,
       view,
     });
@@ -179,6 +195,12 @@ async function startTask(
 
   if (opts.baseBranch !== undefined && !isBranchName(opts.baseBranch))
     throw new Error(`"${opts.baseBranch}" isn't a branch name`);
+  const routing = taskRouting(
+    project.workspace_id,
+    view,
+    opts.model,
+    opts.plan
+  );
   const projectPath = expandHome(project.working_directory);
   const naming = await nameFor(prompt, projectPath, opts.name);
   const feature = `${slugify(naming.name.split(/\s+/).slice(0, 6).join(" "))}-${id.slice(0, 4)}`;
@@ -198,10 +220,7 @@ async function startTask(
   });
 
   const provider = getProvider("claude");
-  const model = resolveModelForAgent(
-    "claude",
-    opts.model || project.default_model
-  );
+  const model = routing.first.model;
   const tmuxName = `${provider.id}-${id}`;
 
   queries
@@ -225,7 +244,8 @@ async function startTask(
     .run(wt.worktreePath, wt.branchName, baseBranch, null, id);
   // Everything the launch needs is on the row, so a restart can resume it.
   db.prepare(
-    `UPDATE sessions SET task_prompt = ?, task_brief = ?, task_status = 'running', setup_status = 'running', name_source = ?, view = ? WHERE id = ?`
+    `UPDATE sessions SET task_prompt = ?, task_brief = ?, task_status = 'running', setup_status = 'running', name_source = ?, view = ?,
+       model_provider = ?, model_overrides = ?, plan_phase = ? WHERE id = ?`
   ).run(
     prompt,
     buildTaskBrief({
@@ -235,6 +255,9 @@ async function startTask(
     }),
     naming.source,
     view,
+    routing.first.provider.id === ANTHROPIC ? null : routing.first.provider.id,
+    routing.override ? JSON.stringify(routing.override) : null,
+    routing.plan ? "planning" : null,
     id
   );
   opts.onCreated?.(id);
@@ -340,6 +363,8 @@ export async function taskView(session: Session): Promise<TaskView> {
     hostName: null,
     hostError: null,
     armed: !!session.merge_armed_at,
+    model: sessionModelRef(session),
+    planning: session.plan_phase === "planning",
   };
 }
 
@@ -365,3 +390,29 @@ export async function listTasks(): Promise<TaskView[]> {
     b.createdAt.localeCompare(a.createdAt)
   );
 }
+
+// The model a task starts on, and whether it plans first: only as a chat,
+// where the handoff can follow the end of its plan turn, and only when
+// planning and implementing are on different models. Both roles resolve
+// now, so a bad override or a provider that's off fails the start.
+export function taskRouting(
+  workspaceId: string | null,
+  view: StartView,
+  model: unknown,
+  plan?: boolean
+) {
+  const override = parseOverride(model);
+  const models = effectiveModels();
+  const opts = { workspaceId, override };
+  const implement = resolveRole("implement", opts, models);
+  const planning = resolveRole("plan", opts, models);
+  const planStep =
+    view === "chat" &&
+    (plan ?? models.planStep) &&
+    planning.ref !== implement.ref;
+  return {
+    override,
+    plan: planStep,
+    first: planStep ? planning : implement,
+  };
+}

package/lib/tasks/queue.ts

@@ -11,6 +11,7 @@
  * is deleted: a started or removed row stays as history.
  */
 
+import type { ModelOverride } from "../models/resolve";
 import { randomUUID } from "crypto";
 import { db, type Session } from "../db";
 import { getProject } from "../projects";
@@ -41,6 +42,8 @@ export interface QueueRow {
   prompt: string;
   name: string | null;
   model: string | null;
+  // A per-role model map (lib/models), when that's what it was given.
+  model_overrides: string | null;
   view: "chat" | "terminal" | null;
   base_branch: string | null;
   host_id: string | null;
@@ -200,7 +203,7 @@ export interface QueueRequest {
   projectId: string;
   prompt: string;
   name?: string;
-  model?: string;
+  model?: ModelOverride;
   view?: "chat" | "terminal";
   baseBranch?: string;
   hostId?: string;
@@ -247,9 +250,9 @@ export function queueIfNeeded(req: QueueRequest): QueuedTaskView | null {
   if (!req.prompt.trim()) throw new Error("Describe the task");
   const id = randomUUID();
   db.prepare(
-    `INSERT INTO task_queue (id, project_id, prompt, name, model, view, base_branch,
+    `INSERT INTO task_queue (id, project_id, prompt, name, model, model_overrides, view, base_branch,
        host_id, after_task, after_any, origin_workspace_id, position)
-     VALUES (@id, @project_id, @prompt, @name, @model, @view, @base_branch,
+     VALUES (@id, @project_id, @prompt, @name, @model, @model_overrides, @view, @base_branch,
        @host_id, @after_task, @after_any, @origin,
        (SELECT COALESCE(MAX(position), 0) + 1 FROM task_queue))`
   ).run({
@@ -257,7 +260,11 @@ export function queueIfNeeded(req: QueueRequest): QueuedTaskView | null {
     project_id: project.id,
     prompt: req.prompt.trim(),
     name: req.name?.trim() || null,
-    model: req.model || null,
+    model: typeof req.model === "string" ? req.model || null : null,
+    model_overrides:
+      req.model && typeof req.model === "object"
+        ? JSON.stringify(req.model)
+        : null,
     view: req.view ?? null,
     base_branch: req.baseBranch || null,
     host_id: req.hostId || null,
@@ -429,7 +436,9 @@ async function startRow(
     projectId: r.project_id,
     prompt: r.prompt,
     name: r.name ?? undefined,
-    model: r.model ?? undefined,
+    model: r.model_overrides
+      ? (JSON.parse(r.model_overrides) as ModelOverride)
+      : (r.model ?? undefined),
     view: r.view ?? undefined,
     baseBranch: r.base_branch ?? undefined,
     hostId: r.host_id ?? undefined,

package/lib/tasks/start.ts

@@ -23,6 +23,7 @@ import { deleteQueued, enqueue, listQueue } from "../chat/queued";
 import { sendQueuedNow } from "../chat/runner";
 import { hasItem } from "../chat/store";
 import { firstMessageId, launchPending } from "./launch-gate";
+import { planPrompt, preparePlanDir } from "../models/plan-step";
 
 export { firstMessageId, launchPending };
 import { expandHome } from "./session";
@@ -136,7 +137,10 @@ export async function finishTaskStart(
       ms: setup.ms,
       error: "Not launched: the task was closed during setup",
     });
-  const prompt = (session.task_prompt ?? "") + setupNote(setup);
+  const task = (session.task_prompt ?? "") + setupNote(setup);
+  const planning = session.plan_phase === "planning";
+  if (planning) preparePlanDir(session.worktree_path);
+  const prompt = planning ? planPrompt(task) : task;
   try {
     if (session.view === "chat") await launchChat(sessionId, prompt);
     else {
@@ -148,6 +152,7 @@ export async function finishTaskStart(
         tmuxName: session.tmux_name,
         cwd: session.worktree_path,
         model: session.model,
+        provider: session.model_provider,
         prompt,
         brief: session.task_brief ?? undefined,
       });
@@ -190,7 +195,7 @@ function earlierSetup(row: SetupColumns): TaskSetup | undefined {
   return undefined;
 }
 
-const killTmux = (name: string) =>
+export const killTmux = (name: string) =>
   execFileAsync("tmux", ["kill-session", "-t", `=${name}`], {
     timeout: 5000,
   }).catch(() => undefined);

package/lib/usage/by-model.ts

@@ -0,0 +1,105 @@
+/**
+ * Spend by model and by role over a window: tokens of each kind, turns,
+ * and the estimated cost from the price table (lib/usage/cost).
+ */
+
+import { estimateCost, rowModel, UNRECORDED, type PricedRow } from "./cost";
+import type { ModelPrice } from "../models/providers";
+
+export interface ModelTurnRow extends PricedRow {
+  at: number;
+  role: string | null;
+}
+
+export interface TokenSpend {
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens: number;
+  cacheWriteTokens: number;
+  turns: number;
+  costUsd: number;
+  // Some of its turns are on a model the price table doesn't list.
+  unpriced: boolean;
+}
+
+export interface ModelSpend extends TokenSpend {
+  provider: string;
+  model: string;
+}
+
+export interface RoleSpend extends TokenSpend {
+  role: string;
+}
+
+export interface SpendByModel {
+  since: number;
+  total: TokenSpend;
+  models: ModelSpend[];
+  roles: RoleSpend[];
+}
+
+export const emptySpend = (): TokenSpend => ({
+  inputTokens: 0,
+  outputTokens: 0,
+  cacheReadTokens: 0,
+  cacheWriteTokens: 0,
+  turns: 0,
+  costUsd: 0,
+  unpriced: false,
+});
+
+export function addRow(
+  s: TokenSpend,
+  r: PricedRow,
+  prices: Record<string, ModelPrice>
+): void {
+  const cost = estimateCost(r, prices);
+  s.inputTokens += r.input_tokens;
+  s.outputTokens += r.output_tokens;
+  s.cacheReadTokens += r.cache_read_tokens;
+  s.cacheWriteTokens += r.cache_write_tokens;
+  s.turns += 1;
+  s.costUsd += cost.usd;
+  s.unpriced ||= !cost.priced;
+}
+
+export function spendByModel(
+  rows: ModelTurnRow[],
+  since: number,
+  prices: Record<string, ModelPrice>
+): SpendByModel {
+  const total = emptySpend();
+  const models = new Map<string, ModelSpend>();
+  const roles = new Map<string, RoleSpend>();
+  for (const r of rows) {
+    if (r.at < since) continue;
+    const { provider, model } = rowModel(r);
+    const mk = `${provider}/${model}`;
+    if (!models.has(mk)) models.set(mk, { ...emptySpend(), provider, model });
+    const role = r.role ?? UNRECORDED;
+    if (!roles.has(role)) roles.set(role, { ...emptySpend(), role });
+    for (const s of [total, models.get(mk)!, roles.get(role)!])
+      addRow(s, r, prices);
+  }
+  const byCost = (a: TokenSpend, b: TokenSpend) => b.costUsd - a.costUsd;
+  return {
+    since,
+    total,
+    models: [...models.values()].sort(byCost),
+    roles: [...roles.values()].sort(byCost),
+  };
+}
+
+export function sumSpend(xs: TokenSpend[]): TokenSpend {
+  const t = emptySpend();
+  for (const x of xs) {
+    t.inputTokens += x.inputTokens;
+    t.outputTokens += x.outputTokens;
+    t.cacheReadTokens += x.cacheReadTokens;
+    t.cacheWriteTokens += x.cacheWriteTokens;
+    t.turns += x.turns;
+    t.costUsd += x.costUsd;
+    t.unpriced ||= x.unpriced;
+  }
+  return t;
+}

package/lib/usage/check-runs.ts

@@ -10,6 +10,8 @@ import { formatTokens } from "../chat/context";
 
 export interface RunUsage {
   model: string | null;
+  // Where the model ran (lib/models); Anthropic when absent.
+  provider?: string;
   inputTokens: number;
   cacheWriteTokens: number;
   cacheReadTokens: number;
@@ -92,9 +94,9 @@ export function recordCheckRun(
   at = Date.now()
 ): void {
   db.prepare(
-    `INSERT INTO check_runs (workspace_id, session_id, label, pr_number, sha, kind, model,
+    `INSERT INTO check_runs (workspace_id, session_id, label, pr_number, sha, kind, model, provider,
        input_tokens, cache_write_tokens, cache_read_tokens, output_tokens, cost_usd, duration_ms, at)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
   ).run(
     target.workspaceId,
     target.sessionId,
@@ -103,6 +105,7 @@ export function recordCheckRun(
     target.sha,
     target.kind,
     u.model,
+    u.provider ?? "anthropic",
     u.inputTokens,
     u.cacheWriteTokens,
     u.cacheReadTokens,

package/lib/usage/compare.ts

@@ -0,0 +1,280 @@
+/**
+ * Claude-only tasks against routed ones, side by side: what each cost
+ * (tokens, dollars at list prices, share of the plan's window, wall time)
+ * and how it went (CI green on the first push, the first review passing,
+ * gate failures, commits after the first review, merged or dropped). Built
+ * on the usage ledgers (chat_turns, and check_runs for the gate's own runs)
+ * and the orchestrator's checks: no record of its own.
+ *
+ * A task's cohort is the provider its own turns (planning and implementing)
+ * ran on: anthropic is the baseline; one that used two is "mixed". Its
+ * reviews and scope checks count toward its cost, whatever they ran on.
+ */
+
+import { db } from "../db";
+import { effectiveModels } from "../models/config";
+import { ANTHROPIC } from "../models/roles";
+import type { ModelPrice } from "../models/providers";
+import { planShares, rowModel, UNRECORDED } from "./cost";
+import {
+  addRow,
+  emptySpend,
+  type ModelTurnRow,
+  type TokenSpend,
+} from "./by-model";
+
+export const SMALL_SAMPLE = 10;
+const CHECK_ROLES = new Set(["review", "scope", "helper"]);
+
+export interface CompareTurn extends ModelTurnRow {
+  session_id: string;
+  duration_ms: number | null;
+  window_pct: number | null;
+}
+
+export interface TaskOutcome {
+  id: string;
+  status: string | null;
+  // The first finished CI run, and review, on its PR: passed or not; null
+  // when it never had one (no PR, or not under an orchestrator).
+  ciFirstGreen: boolean | null;
+  reviewFirstPass: boolean | null;
+  gateFailures: number;
+  // Heads checked after its first review: the rework the review caused.
+  rework: number;
+}
+
+export interface Rate {
+  yes: number;
+  of: number;
+}
+
+export interface Cohort {
+  key: string;
+  baseline: boolean;
+  tasks: number;
+  smallSample: boolean;
+  spend: TokenSpend;
+  // Points of the plan's 5-hour window (an estimate), Anthropic turns only.
+  planPct: number;
+  wallMs: number;
+  ciFirstGreen: Rate;
+  reviewFirstPass: Rate;
+  gateFailures: number;
+  rework: number;
+  merged: Rate;
+  dropped: Rate;
+}
+
+export interface RoleModelRow extends TokenSpend {
+  role: string;
+  provider: string;
+  model: string;
+  planPct: number;
+  wallMs: number;
+}
+
+export interface Comparison {
+  from: number;
+  to: number;
+  cohorts: Cohort[];
+  roles: RoleModelRow[];
+}
+
+const rate = (): Rate => ({ yes: 0, of: 0 });
+const count = (r: Rate, v: boolean | null) => {
+  if (v === null) return;
+  r.of += 1;
+  if (v) r.yes += 1;
+};
+
+function cohortOf(turns: CompareTurn[]): string {
+  const providers = new Set(
+    turns
+      .filter((t) => !CHECK_ROLES.has(t.role ?? ""))
+      .map((t) => rowModel(t).provider)
+  );
+  if (providers.size === 0) return ANTHROPIC;
+  return providers.size === 1 ? [...providers][0] : "mixed";
+}
+
+export function compareRuns(
+  turns: CompareTurn[],
+  outcomes: Map<string, TaskOutcome>,
+  prices: Record<string, ModelPrice>,
+  range: { from: number; to: number }
+): Comparison {
+  const shares = planShares(turns);
+  const share = new Map(turns.map((t, i) => [t, shares[i]]));
+  const inRange = turns.filter((t) => t.at >= range.from && t.at <= range.to);
+
+  const byTask = new Map<string, CompareTurn[]>();
+  for (const t of inRange)
+    if (outcomes.has(t.session_id))
+      byTask.set(t.session_id, [...(byTask.get(t.session_id) ?? []), t]);
+
+  const cohorts = new Map<string, Cohort>();
+  for (const [id, ts] of byTask) {
+    const key = cohortOf(ts);
+    let c = cohorts.get(key);
+    if (!c)
+      cohorts.set(
+        key,
+        (c = {
+          key,
+          baseline: key === ANTHROPIC,
+          tasks: 0,
+          smallSample: true,
+          spend: emptySpend(),
+          planPct: 0,
+          wallMs: 0,
+          ciFirstGreen: rate(),
+          reviewFirstPass: rate(),
+          gateFailures: 0,
+          rework: 0,
+          merged: rate(),
+          dropped: rate(),
+        })
+      );
+    const o = outcomes.get(id)!;
+    c.tasks += 1;
+    for (const t of ts) {
+      addRow(c.spend, t, prices);
+      c.planPct += share.get(t) ?? 0;
+      c.wallMs += t.duration_ms ?? 0;
+    }
+    count(c.ciFirstGreen, o.ciFirstGreen);
+    count(c.reviewFirstPass, o.reviewFirstPass);
+    c.gateFailures += o.gateFailures;
+    c.rework += o.rework;
+    // Still running counts toward neither.
+    const finished = o.status !== "running" && o.status !== null;
+    count(c.merged, finished ? o.status === "merged" : null);
+    count(c.dropped, finished ? o.status === "dropped" : null);
+  }
+  for (const c of cohorts.values()) c.smallSample = c.tasks < SMALL_SAMPLE;
+
+  const roles = new Map<string, RoleModelRow>();
+  for (const t of inRange) {
+    const { provider, model } = rowModel(t);
+    const role = t.role ?? UNRECORDED;
+    const k = `${role} ${provider}/${model}`;
+    let r = roles.get(k);
+    if (!r)
+      roles.set(
+        k,
+        (r = { ...emptySpend(), role, provider, model, planPct: 0, wallMs: 0 })
+      );
+    addRow(r, t, prices);
+    r.planPct += share.get(t) ?? 0;
+    r.wallMs += t.duration_ms ?? 0;
+  }
+
+  return {
+    ...range,
+    // The baseline first, then the most tasks.
+    cohorts: [...cohorts.values()].sort(
+      (a, b) => Number(b.baseline) - Number(a.baseline) || b.tasks - a.tasks
+    ),
+    roles: [...roles.values()].sort(
+      (a, b) => a.role.localeCompare(b.role) || b.costUsd - a.costUsd
+    ),
+  };
+}
+
+// The ledger's turns, with the gate reviews and scope checks recorded in
+// check_runs (lib/usage/check-runs) as turns of their own role.
+export const LEDGER_SQL = `
+  SELECT session_id, at, cost_usd, input_tokens, output_tokens,
+    cache_read_tokens, cache_write_tokens, model, provider, role, duration_ms,
+    window_pct FROM chat_turns WHERE at >= @from AND at <= @to
+  UNION ALL
+  SELECT session_id, at, cost_usd, input_tokens, output_tokens,
+    cache_read_tokens, cache_write_tokens, model, provider, kind AS role,
+    duration_ms, NULL AS window_pct FROM check_runs WHERE at >= @from AND at <= @to
+  ORDER BY at`;
+
+// The outcome of each task with turns in the range.
+export function taskOutcomes(ids: string[]): Map<string, TaskOutcome> {
+  const out = new Map<string, TaskOutcome>();
+  if (!ids.length) return out;
+  const marks = ids.map(() => "?").join(",");
+  const tasks = db
+    .prepare(
+      `SELECT id, task_status FROM sessions
+       WHERE id IN (${marks}) AND task_status IS NOT NULL`
+    )
+    .all(...ids) as { id: string; task_status: string }[];
+  const checks = db
+    .prepare(
+      `SELECT id, session_id, sha, kind, status FROM orchestrator_checks
+       WHERE session_id IN (${marks}) ORDER BY id`
+    )
+    .all(...ids) as {
+    id: number;
+    session_id: string;
+    sha: string;
+    kind: string;
+    status: string;
+  }[];
+  const gates = db
+    .prepare(
+      `SELECT session_id, SUM(count) AS n FROM orchestrator_gate_failures
+       WHERE session_id IN (${marks}) GROUP BY session_id`
+    )
+    .all(...ids) as { session_id: string; n: number }[];
+  for (const t of tasks)
+    out.set(t.id, {
+      id: t.id,
+      status: t.task_status,
+      ciFirstGreen: null,
+      reviewFirstPass: null,
+      gateFailures: 0,
+      rework: 0,
+    });
+  const firstReview = new Map<string, number>();
+  const reworked = new Map<string, Set<string>>();
+  for (const c of checks) {
+    const o = out.get(c.session_id);
+    if (!o) continue;
+    const settled = c.status === "pass" || c.status === "block";
+    if (c.kind === "ci" && settled && o.ciFirstGreen === null)
+      o.ciFirstGreen = c.status === "pass";
+    if (c.kind === "review" && settled && o.reviewFirstPass === null) {
+      o.reviewFirstPass = c.status === "pass";
+      firstReview.set(c.session_id, c.id);
+    }
+    const after = firstReview.get(c.session_id);
+    if (after !== undefined && c.id > after) {
+      const shas = reworked.get(c.session_id) ?? new Set();
+      shas.add(c.sha);
+      reworked.set(c.session_id, shas);
+    }
+  }
+  for (const [id, shas] of reworked) {
+    // The reviewed head itself isn't rework.
+    const o = out.get(id)!;
+    const reviewed = checks.find((c) => c.id === firstReview.get(id))?.sha;
+    o.rework = [...shas].filter((s) => s !== reviewed).length;
+  }
+  for (const g of gates) {
+    const o = out.get(g.session_id);
+    if (o) o.gateFailures = g.n;
+  }
+  return out;
+}
+
+export function comparisonReport(from: number, to: number): Comparison {
+  // From a day earlier, so the first turn in range has a reading before it
+  // to measure its share of the window from.
+  const turns = db
+    .prepare(LEDGER_SQL)
+    .all({ from: from - 24 * 3600 * 1000, to }) as CompareTurn[];
+  const ids = [
+    ...new Set(turns.filter((t) => t.at >= from).map((t) => t.session_id)),
+  ];
+  return compareRuns(turns, taskOutcomes(ids), effectiveModels().prices, {
+    from,
+    to,
+  });
+}