@saadnvd1/agent-os 0.3.499 → 0.3.502

package/lib/usage/calls.ts

@@ -0,0 +1,284 @@
+/**
+ * The per-response usage ledger (usage_calls): one row per Claude API
+ * response, keyed by its message id. Claude Code writes one assistant message
+ * per content block under the same id, with usage growing, so a row is
+ * upserted keeping the highest of each field.
+ */
+
+import { db } from "../db";
+import { ANTHROPIC, sessionRole } from "../models/roles";
+import { ctxOf, weighted, type TokenRow } from "./weights";
+
+export type CallKind = "main" | "sub" | "review" | "gate" | "helper";
+export type CallSource = "live" | "transcript" | "check_run" | "helper";
+
+export interface CallRow {
+  callKey: string;
+  at: number;
+  sessionId: string | null;
+  taskId: string | null;
+  workspaceId: string | null;
+  projectId: string | null;
+  sessionName: string | null;
+  role: string | null;
+  kind: CallKind;
+  agentType: string | null;
+  model: string | null;
+  provider: string;
+  input: number;
+  cacheRead: number;
+  cacheWrite5m: number;
+  cacheWrite1h: number;
+  output: number;
+  ctx: number | null;
+  source: CallSource;
+}
+
+// What a driver reports for one response, before it's tied to a session.
+export type CallUsage = Pick<
+  CallRow,
+  | "callKey"
+  | "at"
+  | "kind"
+  | "agentType"
+  | "model"
+  | "input"
+  | "cacheRead"
+  | "cacheWrite5m"
+  | "cacheWrite1h"
+  | "output"
+>;
+
+interface RawUsage {
+  input_tokens?: number;
+  output_tokens?: number;
+  cache_read_input_tokens?: number;
+  cache_creation_input_tokens?: number;
+  cache_creation?: {
+    ephemeral_5m_input_tokens?: number;
+    ephemeral_1h_input_tokens?: number;
+  } | null;
+}
+
+const n = (v: unknown) => (typeof v === "number" && v > 0 ? v : 0);
+
+// The usage on an assistant message, or null for a synthetic one (an error
+// the CLI wrote itself) or one with no id.
+export function callFromAssistant(
+  message: { id?: string; model?: string; usage?: RawUsage } | undefined,
+  kind: CallKind,
+  agentType: string | null,
+  at = Date.now()
+): CallUsage | null {
+  if (!message?.id || !message.usage) return null;
+  if (!message.model || message.model === "<synthetic>") return null;
+  const u = message.usage;
+  const split = u.cache_creation;
+  const w1h = n(split?.ephemeral_1h_input_tokens);
+  // Without the split, the whole write counts as the 5-minute kind.
+  const w5m = split
+    ? n(split.ephemeral_5m_input_tokens)
+    : n(u.cache_creation_input_tokens);
+  return {
+    callKey: message.id,
+    at,
+    kind,
+    agentType,
+    model: message.model,
+    input: n(u.input_tokens),
+    cacheRead: n(u.cache_read_input_tokens),
+    cacheWrite5m: w5m,
+    cacheWrite1h: w1h,
+    output: n(u.output_tokens),
+  };
+}
+
+export function upsertCall(r: CallRow): void {
+  db.prepare(
+    `INSERT INTO usage_calls (call_key, at, session_id, task_id, workspace_id, project_id,
+       session_name, role, kind, agent_type, model, provider, input_tokens, cache_read_tokens,
+       cache_write_5m_tokens, cache_write_1h_tokens, output_tokens, ctx_tokens, source)
+     VALUES (@callKey, @at, @sessionId, @taskId, @workspaceId, @projectId, @sessionName, @role,
+       @kind, @agentType, @model, @provider, @input, @cacheRead, @cacheWrite5m, @cacheWrite1h,
+       @output, @ctx, @source)
+     ON CONFLICT(call_key) DO UPDATE SET
+       input_tokens = MAX(input_tokens, excluded.input_tokens),
+       cache_read_tokens = MAX(cache_read_tokens, excluded.cache_read_tokens),
+       cache_write_5m_tokens = MAX(cache_write_5m_tokens, excluded.cache_write_5m_tokens),
+       cache_write_1h_tokens = MAX(cache_write_1h_tokens, excluded.cache_write_1h_tokens),
+       output_tokens = MAX(output_tokens, excluded.output_tokens),
+       ctx_tokens = MAX(COALESCE(ctx_tokens, 0), COALESCE(excluded.ctx_tokens, 0)),
+       session_id = COALESCE(session_id, excluded.session_id),
+       task_id = COALESCE(task_id, excluded.task_id),
+       workspace_id = COALESCE(workspace_id, excluded.workspace_id),
+       project_id = COALESCE(project_id, excluded.project_id),
+       session_name = COALESCE(session_name, excluded.session_name),
+       role = COALESCE(role, excluded.role)`
+  ).run(r);
+}
+
+export interface SessionContext {
+  sessionId: string;
+  taskId: string | null;
+  workspaceId: string | null;
+  projectId: string | null;
+  sessionName: string | null;
+  role: string | null;
+  provider: string;
+}
+
+// Who a session's calls belong to. Read per call: a plan becomes
+// implementation, and a name changes.
+export function sessionContext(sessionId: string): SessionContext {
+  const s = db
+    .prepare(
+      `SELECT name, workspace_id, project_id, model_provider, role, task_status,
+         plan_phase, started_by FROM sessions WHERE id = ?`
+    )
+    .get(sessionId) as
+    | {
+        name: string;
+        workspace_id: string | null;
+        project_id: string | null;
+        model_provider: string | null;
+        role: string | null;
+        task_status: string | null;
+        plan_phase: string | null;
+        started_by: string | null;
+      }
+    | undefined;
+  return {
+    sessionId,
+    taskId: s?.task_status ? sessionId : null,
+    workspaceId: s?.workspace_id ?? null,
+    projectId: s?.project_id ?? null,
+    sessionName: s?.name ?? null,
+    role: s ? sessionRole(s) : null,
+    provider: s?.model_provider ?? ANTHROPIC,
+  };
+}
+
+export function recordCall(
+  call: CallUsage,
+  ctx: SessionContext,
+  source: CallSource
+): void {
+  const row: CallRow = {
+    ...call,
+    sessionId: ctx.sessionId,
+    taskId: ctx.taskId,
+    workspaceId: ctx.workspaceId,
+    projectId: ctx.projectId,
+    sessionName: ctx.sessionName,
+    role: ctx.role,
+    provider: ctx.provider,
+    ctx: 0,
+    source,
+  };
+  row.ctx = ctxOf({
+    model: row.model,
+    input_tokens: row.input,
+    cache_read_tokens: row.cacheRead,
+    cache_write_5m_tokens: row.cacheWrite5m,
+    cache_write_1h_tokens: row.cacheWrite1h,
+    output_tokens: row.output,
+  });
+  upsertCall(row);
+}
+
+// A call that never goes through a conversation (a check run).
+export function recordLooseCall(
+  r: Omit<CallRow, "ctx" | "provider"> & { provider?: string }
+): void {
+  upsertCall({ ...r, provider: r.provider ?? ANTHROPIC, ctx: null });
+}
+
+// check_runs rows not yet in the ledger, as calls of their own.
+export function backfillCheckRuns(): number {
+  const rows = db
+    .prepare(
+      `SELECT c.id, c.workspace_id, c.session_id, c.label, c.kind, c.model, c.provider,
+         c.input_tokens, c.cache_write_tokens, c.cache_read_tokens, c.output_tokens, c.at,
+         s.project_id FROM check_runs c LEFT JOIN sessions s ON s.id = c.session_id
+       WHERE NOT EXISTS (SELECT 1 FROM usage_calls u WHERE u.call_key = 'check_run:' || c.id)`
+    )
+    .all() as {
+    id: number;
+    workspace_id: string;
+    session_id: string;
+    label: string;
+    kind: string;
+    model: string | null;
+    provider: string | null;
+    input_tokens: number;
+    cache_write_tokens: number;
+    cache_read_tokens: number;
+    output_tokens: number;
+    at: number;
+    project_id: string | null;
+  }[];
+  db.transaction(() => {
+    for (const c of rows) recordCheckRunCall(c.id, c);
+  })();
+  return rows.length;
+}
+
+export function recordCheckRunCall(
+  id: number,
+  c: {
+    workspace_id: string;
+    session_id: string;
+    label: string;
+    kind: string;
+    model: string | null;
+    provider: string | null;
+    input_tokens: number;
+    cache_write_tokens: number;
+    cache_read_tokens: number;
+    output_tokens: number;
+    at: number;
+    project_id?: string | null;
+  }
+): void {
+  recordLooseCall({
+    callKey: `check_run:${id}`,
+    at: c.at,
+    sessionId: c.session_id,
+    taskId: c.session_id ? sessionContext(c.session_id).taskId : null,
+    workspaceId: c.workspace_id,
+    projectId: c.project_id ?? null,
+    sessionName: c.label,
+    role: `gate-${c.kind}`,
+    kind: "gate",
+    agentType: null,
+    model: c.model,
+    provider: c.provider ?? ANTHROPIC,
+    input: c.input_tokens,
+    cacheRead: c.cache_read_tokens,
+    cacheWrite5m: c.cache_write_tokens,
+    cacheWrite1h: 0,
+    output: c.output_tokens,
+    source: "check_run",
+  });
+}
+
+// What each task cost so far in weighted tokens: its own session, its
+// subagents and its gate reviews. One grouped query for the whole list.
+export function taskCosts(ids: string[]): Map<string, number> {
+  const out = new Map<string, number>();
+  if (!ids.length) return out;
+  const marks = ids.map(() => "?").join(",");
+  const rows = db
+    .prepare(
+      `SELECT task_id, model, provider, SUM(input_tokens) AS input_tokens,
+         SUM(cache_read_tokens) AS cache_read_tokens,
+         SUM(cache_write_5m_tokens) AS cache_write_5m_tokens,
+         SUM(cache_write_1h_tokens) AS cache_write_1h_tokens,
+         SUM(output_tokens) AS output_tokens
+       FROM usage_calls WHERE task_id IN (${marks}) GROUP BY task_id, model, provider`
+    )
+    .all(...ids) as (TokenRow & { task_id: string })[];
+  for (const r of rows)
+    out.set(r.task_id, (out.get(r.task_id) ?? 0) + weighted(r));
+  return out;
+}

package/lib/usage/check-runs.ts

@@ -6,6 +6,7 @@
  */
 
 import { db } from "../db";
+import { recordCheckRunCall } from "./calls";
 import { formatTokens } from "../chat/context";
 
 export interface RunUsage {
@@ -93,27 +94,47 @@ export function recordCheckRun(
   u: RunUsage,
   at = Date.now()
 ): void {
-  db.prepare(
-    `INSERT INTO check_runs (workspace_id, session_id, label, pr_number, sha, kind, model, provider,
+  const info = db
+    .prepare(
+      `INSERT INTO check_runs (workspace_id, session_id, label, pr_number, sha, kind, model, provider,
        input_tokens, cache_write_tokens, cache_read_tokens, output_tokens, cost_usd, duration_ms, at)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
-  ).run(
-    target.workspaceId,
-    target.sessionId,
-    target.label,
-    target.prNumber,
-    target.sha,
-    target.kind,
-    u.model,
-    u.provider ?? "anthropic",
-    u.inputTokens,
-    u.cacheWriteTokens,
-    u.cacheReadTokens,
-    u.outputTokens,
-    u.costUsd,
-    u.durationMs,
-    at
-  );
+    )
+    .run(
+      target.workspaceId,
+      target.sessionId,
+      target.label,
+      target.prNumber,
+      target.sha,
+      target.kind,
+      u.model,
+      u.provider ?? "anthropic",
+      u.inputTokens,
+      u.cacheWriteTokens,
+      u.cacheReadTokens,
+      u.outputTokens,
+      u.costUsd,
+      u.durationMs,
+      at
+    );
+  try {
+    recordCheckRunCall(Number(info.lastInsertRowid), {
+      workspace_id: target.workspaceId,
+      session_id: target.sessionId,
+      label: target.label,
+      kind: target.kind,
+      model: u.model,
+      provider: u.provider ?? "anthropic",
+      input_tokens: u.inputTokens,
+      cache_write_tokens: u.cacheWriteTokens,
+      cache_read_tokens: u.cacheReadTokens,
+      output_tokens: u.outputTokens,
+      at,
+    });
+  } catch (e) {
+    // The nightly backfill picks it up.
+    console.error(`[check-runs] ledger ${target.label}:`, e);
+  }
 }
 
 // A recorder for one check, that never lets a write break the check.

package/lib/usage/config.ts

@@ -0,0 +1,56 @@
+/**
+ * The ratchet's thresholds, kept in the settings table so they can change
+ * without a deploy. Each falls back to its default for a missing or invalid
+ * value.
+ */
+
+import { db } from "../db";
+
+export const USAGE_KEYS = {
+  costPct: "usage.alert_cost_pct",
+  cleanPoints: "usage.alert_clean_points",
+  baselineDays: "usage.baseline_days",
+  avgDays: "usage.avg_days",
+  minMerges: "usage.alert_min_merges",
+  weeklyStaleHours: "usage.weekly_stale_hours",
+} as const;
+
+export interface UsageConfig {
+  // Alert when the 3-day cost per merged PR is this many percent over baseline.
+  costPct: number;
+  // Alert when the clean-merge rate is this many points under baseline.
+  cleanPoints: number;
+  baselineDays: number;
+  avgDays: number;
+  // Fewer merges than this over the average window: too little to judge.
+  minMerges: number;
+  weeklyStaleHours: number;
+}
+
+export const USAGE_DEFAULTS: UsageConfig = {
+  costPct: 15,
+  cleanPoints: 10,
+  baselineDays: 7,
+  avgDays: 3,
+  minMerges: 3,
+  weeklyStaleHours: 6,
+};
+
+const DAY_COUNTS = new Set<keyof UsageConfig>(["baselineDays", "avgDays"]);
+
+export function readUsageConfig(): UsageConfig {
+  const out: UsageConfig = { ...USAGE_DEFAULTS };
+  for (const [name, key] of Object.entries(USAGE_KEYS)) {
+    const row = db
+      .prepare(`SELECT value FROM settings WHERE key = ?`)
+      .get(key) as { value: string } | undefined;
+    const v = row ? Number(row.value) : NaN;
+    const key2 = name as keyof UsageConfig;
+    // A day count is a whole number of days, at least one.
+    const ok = DAY_COUNTS.has(key2)
+      ? Number.isInteger(v) && v >= 1
+      : Number.isFinite(v) && v >= 0;
+    if (ok) out[key2] = v;
+  }
+  return out;
+}

package/lib/usage/nightly.ts

@@ -0,0 +1,157 @@
+/**
+ * The nightly rollup, inside the server, with no model: it imports the
+ * transcripts it couldn't watch, rolls each recent day up, moves the
+ * ratchet's baseline, and alerts once when the last days regressed. A light
+ * pass every half hour keeps today's row (and cost on task cards) current.
+ */
+
+import { db } from "../db";
+import { notifyPhone } from "../notify";
+import { backfillCheckRuns } from "./calls";
+import { readUsageConfig } from "./config";
+import type { Gh } from "./quality";
+import {
+  applyAlertState,
+  evaluateAlerts,
+  loadBaseline,
+  nextAlertState,
+  nextBaseline,
+  openAlerts,
+  saveBaseline,
+} from "./ratchet";
+import { dayKey, dayStart, loadRollups, rollupDay, saveRollup } from "./rollup";
+import { importTranscripts } from "./transcript-import";
+
+const DAY = 24 * 3600 * 1000;
+const LAST_RUN = "usage.last_nightly";
+const TICK_MS = 30 * 60 * 1000;
+// A day's escapes settle three days after it ends.
+const SETTLE_MS = 3 * DAY;
+
+export interface NightlyDeps {
+  now: () => number;
+  gh?: Gh;
+  notify: (text: string) => void;
+  transcriptsRoot?: string;
+}
+
+export const realDeps: NightlyDeps = {
+  now: Date.now,
+  notify: (text) => notifyPhone("usage", text),
+};
+
+const addDays = (day: string, n: number) =>
+  dayKey(dayStart(day) + n * DAY + DAY / 2);
+
+function setting(key: string): string | null {
+  const r = db.prepare(`SELECT value FROM settings WHERE key = ?`).get(key) as
+    | { value: string }
+    | undefined;
+  return r?.value ?? null;
+}
+
+// Imports what's new and keeps today's row current.
+export async function lightPass(deps: NightlyDeps): Promise<void> {
+  const now = deps.now();
+  const seeded = loadRollups(1).length > 0;
+  await importTranscripts(
+    now - (seeded ? 3 : readUsageConfig().baselineDays + 1) * DAY,
+    400,
+    deps.transcriptsRoot
+  );
+  backfillCheckRuns();
+  const today = dayKey(now);
+  saveRollup(await rollupDay(today, now, undefined), false, now);
+}
+
+// The once-a-day pass.
+export async function nightly(deps: NightlyDeps): Promise<void> {
+  const now = deps.now();
+  const cfg = readUsageConfig();
+  const today = dayKey(now);
+  await lightPass(deps);
+  const existing = loadRollups(60);
+  const last = existing.filter((r) => r.day < today).at(-1)?.day;
+  let day = last ? addDays(last, -3) : addDays(today, -cfg.baselineDays);
+  for (; day < today; day = addDays(day, 1)) {
+    const final = dayStart(day) + DAY + SETTLE_MS <= now;
+    saveRollup(await rollupDay(day, now, deps.gh), final, now);
+  }
+  const complete = loadRollups(60).filter((r) => r.day < today);
+  const baseline = nextBaseline(loadBaseline(), complete, cfg, now);
+  if (baseline) saveBaseline(baseline);
+  const state = nextAlertState(
+    openAlerts(),
+    evaluateAlerts(complete, baseline, cfg)
+  );
+  applyAlertState(state, now);
+  for (const a of state.notify) deps.notify(a.message);
+  db.prepare(
+    `INSERT INTO settings (key, value) VALUES (?, ?)
+     ON CONFLICT(key) DO UPDATE SET value = excluded.value`
+  ).run(LAST_RUN, today);
+}
+
+// Whether this server runs the rollup: a dev server only when asked, so it
+// never alerts for a database that isn't real.
+export function usageRollupEnabled(
+  env: Record<string, string | undefined>
+): boolean {
+  const dev = env.AGENTOS_DEV === "1" || env.NODE_ENV !== "production";
+  return dev
+    ? env.AGENTOS_USAGE_ROLLUP === "on"
+    : env.AGENTOS_USAGE_ROLLUP !== "off";
+}
+
+const g = globalThis as unknown as {
+  __agentosUsage?: { stop: () => void };
+  __agentosUsageBusy?: boolean;
+};
+
+// Runs a pass unless one is already running, in this process: the timer and
+// the API route share the flag (on globalThis, as the server and the routes
+// load separate copies of this module). Returns whether it ran.
+export async function exclusive(pass: () => Promise<void>): Promise<boolean> {
+  if (g.__agentosUsageBusy) return false;
+  g.__agentosUsageBusy = true;
+  try {
+    await pass();
+  } finally {
+    g.__agentosUsageBusy = false;
+  }
+  return true;
+}
+
+export const usageBusy = () => !!g.__agentosUsageBusy;
+
+// The due pass: the full one once a day, just after midnight.
+const duePass = (deps: NightlyDeps) =>
+  setting(LAST_RUN) !== dayKey(deps.now()) ? nightly(deps) : lightPass(deps);
+
+export function startUsageRollup(deps: NightlyDeps = realDeps): {
+  stop: () => void;
+} {
+  if (g.__agentosUsage) return g.__agentosUsage;
+  let timer: NodeJS.Timeout | undefined;
+  let stopped = false;
+  const run = () =>
+    exclusive(() => duePass(deps)).catch((error) =>
+      console.error("[usage] rollup failed:", error)
+    );
+  const next = () => {
+    if (stopped) return;
+    timer = setTimeout(() => void run().finally(next), TICK_MS);
+    timer.unref();
+  };
+  // Not at boot: the server is busy resuming sessions then.
+  timer = setTimeout(() => void run().finally(next), 60_000);
+  timer.unref();
+  g.__agentosUsage = {
+    stop: () => {
+      stopped = true;
+      clearTimeout(timer);
+      g.__agentosUsage = undefined;
+    },
+  };
+  return g.__agentosUsage;
+}