@saadnvd1/agent-os 0.3.499 → 0.3.502

package/lib/usage/quality.ts

@@ -0,0 +1,239 @@
+/**
+ * Quality of the work that merged, from data AgentOS already keeps (the
+ * orchestrator's checks, gate failures, asks and brief amendments) and, for
+ * escapes, the merged PR list from gh. No model is involved.
+ */
+
+import { db } from "../db";
+import { run as runCmd } from "../tasks/gh";
+
+export interface MergedTask {
+  id: string;
+  projectId: string | null;
+  prNumber: number | null;
+  mergedAt: string;
+}
+
+export interface TaskQuality {
+  id: string;
+  reviewed: boolean;
+  clean: boolean;
+  // Heads checked after the first reviewed one.
+  fixRounds: number;
+  gateFailures: number;
+}
+
+export interface DayQuality {
+  merged: number;
+  clean: number;
+  fixRounds: number;
+  escalations: number;
+  scopeChanges: number;
+  // Null when gh couldn't say.
+  escapes: number | null;
+}
+
+interface CheckRow {
+  id: number;
+  session_id: string;
+  sha: string;
+  kind: string;
+  status: string;
+}
+
+// Clean: the first reviewed head passed review and CI, nothing failed a
+// gate, and no later head was checked. A merge with no review at all isn't
+// clean: it can't be told apart from one that skipped the gate.
+export function taskQuality(
+  id: string,
+  checks: CheckRow[],
+  gateFailures: number
+): TaskQuality {
+  const mine = checks
+    .filter((c) => c.session_id === id)
+    .sort((a, b) => a.id - b.id);
+  const first = mine.find(
+    (c) => c.kind === "review" && (c.status === "pass" || c.status === "block")
+  );
+  if (!first)
+    return { id, reviewed: false, clean: false, fixRounds: 0, gateFailures };
+  const ci = mine.find((c) => c.kind === "ci" && c.sha === first.sha);
+  const later = new Set(
+    mine.filter((c) => c.id > first.id && c.sha !== first.sha).map((c) => c.sha)
+  );
+  return {
+    id,
+    reviewed: true,
+    clean:
+      first.status === "pass" &&
+      ci?.status === "pass" &&
+      gateFailures === 0 &&
+      later.size === 0,
+    fixRounds: later.size,
+    gateFailures,
+  };
+}
+
+const sqlTime = (ms: number) =>
+  new Date(ms).toISOString().slice(0, 19).replace("T", " ");
+
+export function mergedTasks(from: number, to: number): MergedTask[] {
+  return (
+    db
+      .prepare(
+        `SELECT id, project_id AS projectId, pr_number AS prNumber, merged_at AS mergedAt
+         FROM sessions WHERE task_status = 'merged' AND merged_at >= ? AND merged_at < ?`
+      )
+      .all(sqlTime(from), sqlTime(to)) as MergedTask[]
+  ).filter((t) => t.mergedAt);
+}
+
+export function qualityOf(tasks: MergedTask[]): TaskQuality[] {
+  if (!tasks.length) return [];
+  const marks = tasks.map(() => "?").join(",");
+  const ids = tasks.map((t) => t.id);
+  const checks = db
+    .prepare(
+      `SELECT id, session_id, sha, kind, status FROM orchestrator_checks
+       WHERE session_id IN (${marks})`
+    )
+    .all(...ids) as CheckRow[];
+  const gates = new Map(
+    (
+      db
+        .prepare(
+          `SELECT session_id, SUM(count) AS n FROM orchestrator_gate_failures
+           WHERE session_id IN (${marks}) GROUP BY session_id`
+        )
+        .all(...ids) as { session_id: string; n: number }[]
+    ).map((g) => [g.session_id, g.n])
+  );
+  return tasks.map((t) => taskQuality(t.id, checks, gates.get(t.id) ?? 0));
+}
+
+// Escalations to Saad in the range: gates that gave up on a task, and gate
+// asks, counted once per task.
+export function escalationsIn(from: number, to: number): number {
+  const f = sqlTime(from);
+  const t = sqlTime(to);
+  const rows = db
+    .prepare(
+      `SELECT session_id AS task FROM orchestrator_gate_failures
+         WHERE escalated_at >= ? AND escalated_at < ?
+       UNION
+       SELECT REPLACE(subject, 'task:', '') AS task FROM orchestrator_asks
+         WHERE kind = 'gate' AND created_at >= ? AND created_at < ?`
+    )
+    .all(f, t, f, t);
+  return rows.length;
+}
+
+export function scopeChangesIn(from: number, to: number): number {
+  return (
+    db
+      .prepare(
+        `SELECT COUNT(*) AS n FROM brief_amendments WHERE created_at >= ? AND created_at < ?`
+      )
+      .get(sqlTime(from), sqlTime(to)) as { n: number }
+  ).n;
+}
+
+export interface MergedPR {
+  number: number;
+  title: string;
+  mergedAt: number;
+  files: string[];
+}
+
+const FIX = /^(revert|fix)(\([^)]*\))?!?:|^revert\b/i;
+export const ESCAPE_WINDOW_MS = 3 * 24 * 3600 * 1000;
+
+// PRs in `merged` that a later revert or fix PR touched the same files of,
+// within three days of the merge.
+export function escapesOf(merged: MergedPR[], all: MergedPR[]): MergedPR[] {
+  return merged.filter((p) =>
+    all.some(
+      (q) =>
+        q.number !== p.number &&
+        q.mergedAt > p.mergedAt &&
+        q.mergedAt - p.mergedAt <= ESCAPE_WINDOW_MS &&
+        FIX.test(q.title.trim()) &&
+        q.files.some((f) => p.files.includes(f))
+    )
+  );
+}
+
+export type Gh = (args: string[], cwd: string) => Promise<string>;
+const realGh: Gh = (args, cwd) => runCmd("gh", args, cwd, 60_000);
+
+function repoOf(projectId: string | null): string | null {
+  if (!projectId) return null;
+  const r = db
+    .prepare(
+      `SELECT path FROM project_repositories WHERE project_id = ?
+       ORDER BY is_primary DESC, sort_order LIMIT 1`
+    )
+    .get(projectId) as { path: string } | undefined;
+  return r?.path ?? null;
+}
+
+// Escapes among the tasks merged in [from, to), looking three days past each
+// merge. Null when gh couldn't say for any repo: unknown is not zero.
+export async function escapesIn(
+  tasks: MergedTask[],
+  gh: Gh = realGh
+): Promise<number | null> {
+  const withPR = tasks.filter((t) => t.prNumber !== null);
+  if (!withPR.length) return 0;
+  const byRepo = new Map<string, MergedTask[]>();
+  for (const t of withPR) {
+    const repo = repoOf(t.projectId);
+    if (!repo) return null;
+    byRepo.set(repo, [...(byRepo.get(repo) ?? []), t]);
+  }
+  let total = 0;
+  for (const [repo, ts] of byRepo) {
+    try {
+      const since = new Date(
+        Math.min(
+          ...ts.map((t) => Date.parse(t.mergedAt.replace(" ", "T") + "Z"))
+        ) - 86_400_000
+      )
+        .toISOString()
+        .slice(0, 10);
+      const out = await gh(
+        [
+          "pr",
+          "list",
+          "--state",
+          "merged",
+          "--search",
+          `merged:>=${since}`,
+          "--json",
+          "number,title,mergedAt,files",
+          "--limit",
+          "300",
+        ],
+        repo
+      );
+      const all = (
+        JSON.parse(out) as {
+          number: number;
+          title: string;
+          mergedAt: string;
+          files?: { path: string }[];
+        }[]
+      ).map((p) => ({
+        number: p.number,
+        title: p.title,
+        mergedAt: Date.parse(p.mergedAt),
+        files: (p.files ?? []).map((f) => f.path),
+      }));
+      const mine = all.filter((p) => ts.some((t) => t.prNumber === p.number));
+      total += escapesOf(mine, all).length;
+    } catch {
+      return null;
+    }
+  }
+  return total;
+}

package/lib/usage/ratchet.ts

@@ -0,0 +1,275 @@
+/**
+ * The ratchet. The baseline is the first week of rollups, and from then on
+ * only ever moves to a better value. An alert fires, once, when cost per
+ * merged PR or the clean-merge rate has drifted past the configured margin,
+ * naming the supporting metric that moved most.
+ */
+
+import { db } from "../db";
+import { spanOf, type Rollup, type Span } from "./span";
+import type { UsageConfig } from "./config";
+
+export type Direction = "lower" | "higher";
+
+// The better of the two. A missing or invalid candidate keeps what was.
+export function ratchet(
+  prev: number | null,
+  candidate: number | null,
+  better: Direction
+): number | null {
+  if (candidate === null || !Number.isFinite(candidate)) return prev;
+  if (prev === null || !Number.isFinite(prev)) return candidate;
+  return better === "lower"
+    ? Math.min(prev, candidate)
+    : Math.max(prev, candidate);
+}
+
+// What the baseline is judged on.
+export const HEADLINE = {
+  costPerMerged: "lower",
+  costPerClean: "lower",
+  cleanRate: "higher",
+} as const satisfies Record<string, Direction>;
+export type HeadlineKey = keyof typeof HEADLINE;
+
+// Supporting metrics, each where lower is better, per PR where it is a count.
+export interface Supporting {
+  reviewPerPr: number | null;
+  over200kShare: number | null;
+  orchestratorShare: number | null;
+  opusShare: number | null;
+  fixRoundsPerPr: number | null;
+  escalationsPerPr: number | null;
+  scopeChangesPerPr: number | null;
+  escapesPerPr: number | null;
+}
+
+export const SUPPORTING_LABELS: Record<keyof Supporting, string> = {
+  reviewPerPr: "review cost per PR",
+  over200kShare: "share of calls above 200k context",
+  orchestratorShare: "orchestrator share",
+  opusShare: "Opus share of the model mix",
+  fixRoundsPerPr: "fix rounds per PR",
+  escalationsPerPr: "escalations per PR",
+  scopeChangesPerPr: "scope changes per PR",
+  escapesPerPr: "escapes per PR",
+};
+
+const per = (n: number | null, merged: number) =>
+  n !== null && merged > 0 ? n / merged : null;
+
+export function supportingOf(s: Span): Supporting {
+  return {
+    reviewPerPr: s.reviewPerPr,
+    over200kShare: s.over200kShare,
+    orchestratorShare: s.orchestratorShare,
+    opusShare: s.opusShare,
+    fixRoundsPerPr: s.fixRoundsPerPr,
+    escalationsPerPr: per(s.escalations, s.merged),
+    scopeChangesPerPr: per(s.scopeChanges, s.merged),
+    escapesPerPr: per(s.escapes, s.merged),
+  };
+}
+
+export interface Baseline {
+  values: Record<HeadlineKey, number | null>;
+  supporting: Supporting;
+  setAt: number;
+}
+
+// The next baseline. None until `cfg.baselineDays` complete days exist with
+// at least `cfg.minMerges` merges among them: a quiet week would lock in a
+// baseline nothing can meet, so the seed waits for the first week that has
+// enough work, and a ratchet step skips a week that doesn't. The first is
+// those first days, and each night after it, the trailing week of days is
+// ratcheted in. The supporting snapshot follows cost per merged PR:
+// it is replaced only when that improves.
+export function nextBaseline(
+  prev: Baseline | null,
+  complete: Rollup[],
+  cfg: UsageConfig,
+  now = Date.now()
+): Baseline | null {
+  const days = [...complete].sort((a, b) => a.day.localeCompare(b.day));
+  if (days.length < cfg.baselineDays) return prev;
+  if (!prev) {
+    let s = spanOf(days.slice(0, cfg.baselineDays));
+    if (s.merged < cfg.minMerges) s = spanOf(days.slice(-cfg.baselineDays));
+    if (s.merged < cfg.minMerges) return null;
+    return {
+      values: {
+        costPerMerged: s.costPerMerged,
+        costPerClean: s.costPerClean,
+        cleanRate: s.cleanRate,
+      },
+      supporting: supportingOf(s),
+      setAt: now,
+    };
+  }
+  const s = spanOf(days.slice(-cfg.baselineDays));
+  if (s.merged < cfg.minMerges) return prev;
+  const values = { ...prev.values };
+  for (const k of Object.keys(HEADLINE) as HeadlineKey[])
+    values[k] = ratchet(prev.values[k], s[k], HEADLINE[k]);
+  const improved =
+    s.costPerMerged !== null &&
+    (prev.values.costPerMerged === null ||
+      s.costPerMerged < prev.values.costPerMerged);
+  return {
+    values,
+    supporting: improved ? supportingOf(s) : prev.supporting,
+    setAt: improved ? now : prev.setAt,
+  };
+}
+
+export interface Alert {
+  kind: "cost" | "quality";
+  message: string;
+}
+
+const round = (x: number) => Math.round(x * 1e6) / 1e6;
+
+// The supporting metric that moved furthest the wrong way since the baseline.
+export function biggestMover(
+  base: Supporting,
+  now: Supporting
+): { label: string; change: string } | null {
+  let best: { key: keyof Supporting; score: number } | null = null;
+  for (const key of Object.keys(base) as (keyof Supporting)[]) {
+    const b = base[key];
+    const c = now[key];
+    if (b === null || c === null) continue;
+    const score = b > 0 ? (c - b) / b : c > 0 ? 1 : 0;
+    if (score > 0 && (!best || score > best.score)) best = { key, score };
+  }
+  if (!best) return null;
+  const b = base[best.key] as number;
+  return {
+    label: SUPPORTING_LABELS[best.key],
+    change: b > 0 ? `+${Math.round(best.score * 100)}%` : "up from none",
+  };
+}
+
+// Conditions that hold now, over the last `avgDays` complete days.
+export function evaluateAlerts(
+  complete: Rollup[],
+  baseline: Baseline | null,
+  cfg: UsageConfig
+): Alert[] {
+  if (!baseline) return [];
+  const days = [...complete]
+    .sort((a, b) => a.day.localeCompare(b.day))
+    .slice(-cfg.avgDays);
+  const s = spanOf(days);
+  if (days.length < cfg.avgDays || s.merged < cfg.minMerges) return [];
+  const mover = biggestMover(baseline.supporting, supportingOf(s));
+  const moved = mover ? ` Biggest mover: ${mover.label} ${mover.change}.` : "";
+  const out: Alert[] = [];
+  const baseCost = baseline.values.costPerMerged;
+  if (s.costPerMerged !== null && baseCost !== null && baseCost > 0) {
+    const over = round(((s.costPerMerged - baseCost) / baseCost) * 100);
+    if (over > cfg.costPct)
+      out.push({
+        kind: "cost",
+        message: `Usage: cost per merged PR, ${cfg.avgDays}-day average, is ${Math.round(over)}% over baseline.${moved}`,
+      });
+  }
+  const baseClean = baseline.values.cleanRate;
+  if (s.cleanRate !== null && baseClean !== null) {
+    const under = round((baseClean - s.cleanRate) * 100);
+    if (under > cfg.cleanPoints)
+      out.push({
+        kind: "quality",
+        message: `Usage: clean-merge rate, ${cfg.avgDays}-day average, is ${Math.round(under)} points under baseline.${moved}`,
+      });
+  }
+  return out;
+}
+
+// Which firing alerts are new (to send), and which open ones have cleared.
+export function nextAlertState(
+  open: string[],
+  firing: Alert[]
+): { notify: Alert[]; clear: string[] } {
+  const now = new Set(firing.map((a) => a.kind));
+  return {
+    notify: firing.filter((a) => !open.includes(a.kind)),
+    clear: open.filter((k) => !now.has(k as Alert["kind"])),
+  };
+}
+
+export function loadBaseline(): Baseline | null {
+  const rows = db
+    .prepare(`SELECT metric, value, set_at, supporting FROM usage_baseline`)
+    .all() as {
+    metric: string;
+    value: number;
+    set_at: number;
+    supporting: string | null;
+  }[];
+  if (!rows.length) return null;
+  const values: Baseline["values"] = {
+    costPerMerged: null,
+    costPerClean: null,
+    cleanRate: null,
+  };
+  let supporting: Supporting | null = null;
+  let setAt = 0;
+  for (const r of rows) {
+    if (r.metric in values) values[r.metric as HeadlineKey] = r.value;
+    if (r.metric === "costPerMerged" && r.supporting)
+      supporting = JSON.parse(r.supporting) as Supporting;
+    setAt = Math.max(setAt, r.set_at);
+  }
+  return {
+    values,
+    supporting: supporting ?? supportingOf(spanOf([])),
+    setAt,
+  };
+}
+
+export function saveBaseline(b: Baseline): void {
+  db.transaction(() => {
+    for (const k of Object.keys(HEADLINE) as HeadlineKey[]) {
+      const v = b.values[k];
+      if (v === null) continue;
+      db.prepare(
+        `INSERT INTO usage_baseline (metric, value, set_at, supporting) VALUES (?, ?, ?, ?)
+         ON CONFLICT(metric) DO UPDATE SET value = excluded.value, set_at = excluded.set_at,
+           supporting = excluded.supporting`
+      ).run(
+        k,
+        v,
+        b.setAt,
+        k === "costPerMerged" ? JSON.stringify(b.supporting) : null
+      );
+    }
+  })();
+}
+
+export function openAlerts(): string[] {
+  return (
+    db
+      .prepare(`SELECT kind FROM usage_alerts WHERE cleared_at IS NULL`)
+      .all() as { kind: string }[]
+  ).map((r) => r.kind);
+}
+
+export function applyAlertState(
+  state: { notify: Alert[]; clear: string[] },
+  now = Date.now()
+): void {
+  db.transaction(() => {
+    for (const a of state.notify)
+      db.prepare(
+        `INSERT INTO usage_alerts (kind, opened_at, cleared_at, message) VALUES (?, ?, NULL, ?)
+         ON CONFLICT(kind) DO UPDATE SET opened_at = excluded.opened_at, cleared_at = NULL,
+           message = excluded.message`
+      ).run(a.kind, now, a.message);
+    for (const k of state.clear)
+      db.prepare(`UPDATE usage_alerts SET cleared_at = ? WHERE kind = ?`).run(
+        now,
+        k
+      );
+  })();
+}

package/lib/usage/rate-limits.ts

@@ -0,0 +1,192 @@
+/**
+ * The account's limit windows as Claude Code reports them: its
+ * rate_limit_event carries what the anthropic-ratelimit-unified-* headers
+ * said on a response, for the window that matters most at that moment. Each
+ * reading is kept with its time; between readings the weekly figure is an
+ * estimate from recorded usage, and labelled as one.
+ */
+
+import { db } from "../db";
+import { weighted, type TokenRow } from "./weights";
+import { readUsageConfig } from "./config";
+
+export type LimitWindow = "five_hour" | "seven_day";
+
+export interface Reading {
+  window: LimitWindow;
+  // Percent used, 0-100.
+  utilization: number;
+  raw: number;
+  // Epoch ms.
+  resetsAt: number | null;
+  status: string | null;
+}
+
+const WINDOWS: LimitWindow[] = ["five_hour", "seven_day"];
+const num = (v: unknown) =>
+  typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;
+
+// A rate_limit_event carries what the headers said: utilization as a
+// fraction (0.42) and the reset in epoch seconds. A plan's per-window object
+// carries a percent (42) and an ISO time. Each source has one unit.
+const eventPercent = (fraction: number) => fraction * 100;
+
+// Readings in what the CLI emits: a rate_limit_event's info, or a plan's
+// per-window object. Anything else gives none.
+export function parseRateLimit(info: unknown): Reading[] {
+  if (!info || typeof info !== "object") return [];
+  const i = info as Record<string, unknown>;
+  const out: Reading[] = [];
+  const status = typeof i.status === "string" ? i.status : null;
+  const type = i.rateLimitType;
+  const raw = num(i.utilization);
+  if ((type === "five_hour" || type === "seven_day") && raw !== null) {
+    const reset = num(i.resetsAt);
+    out.push({
+      window: type,
+      utilization: eventPercent(raw),
+      raw,
+      resetsAt: reset === null ? null : reset * 1000,
+      status,
+    });
+  }
+  for (const w of WINDOWS) {
+    const o = i[w] as Record<string, unknown> | null | undefined;
+    const u = o && num(o.utilization);
+    if (!o || u === null || u === undefined) continue;
+    const reset =
+      typeof o.resets_at === "string" ? Date.parse(o.resets_at) : NaN;
+    out.push({
+      window: w,
+      utilization: u,
+      raw: u,
+      resetsAt: Number.isFinite(reset) ? reset : null,
+      status,
+    });
+  }
+  return out;
+}
+
+// Stores a reading only when it differs from that window's last, so a busy
+// day adds a handful of rows, not one per response.
+export function recordReadings(
+  readings: Reading[],
+  source: string,
+  sessionId: string | null = null,
+  at = Date.now()
+): number {
+  let added = 0;
+  for (const r of readings) {
+    const last = db
+      .prepare(
+        `SELECT utilization, resets_at, status FROM rate_limit_readings
+         WHERE window = ? ORDER BY id DESC LIMIT 1`
+      )
+      .get(r.window) as
+      | { utilization: number; resets_at: number | null; status: string | null }
+      | undefined;
+    if (
+      last &&
+      last.utilization === r.utilization &&
+      last.resets_at === r.resetsAt &&
+      last.status === r.status
+    )
+      continue;
+    db.prepare(
+      `INSERT INTO rate_limit_readings (at, window, utilization, raw, resets_at, status, session_id, source)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+    ).run(
+      at,
+      r.window,
+      r.utilization,
+      r.raw,
+      r.resetsAt,
+      r.status,
+      sessionId,
+      source
+    );
+    added++;
+  }
+  return added;
+}
+
+export interface WindowReading {
+  pct: number;
+  resetsAt: number | null;
+  // When it was read (or, for an estimate, worked out).
+  at: number;
+  estimated: boolean;
+}
+
+export function lastReading(
+  window: LimitWindow,
+  before = Number.MAX_SAFE_INTEGER
+): { at: number; utilization: number; resetsAt: number | null } | null {
+  const r = db
+    .prepare(
+      `SELECT at, utilization, resets_at FROM rate_limit_readings
+       WHERE window = ? AND at <= ? ORDER BY at DESC, id DESC LIMIT 1`
+    )
+    .get(window, before) as
+    | { at: number; utilization: number; resets_at: number | null }
+    | undefined;
+  return r
+    ? { at: r.at, utilization: r.utilization, resetsAt: r.resets_at }
+    : null;
+}
+
+const WEEK = 7 * 24 * 3600 * 1000;
+
+export function weightedBetween(from: number, to: number): number {
+  const rows = db
+    .prepare(
+      `SELECT model, provider, input_tokens, cache_read_tokens, cache_write_5m_tokens,
+         cache_write_1h_tokens, output_tokens FROM usage_calls WHERE at > ? AND at <= ?`
+    )
+    .all(from, to) as TokenRow[];
+  let t = 0;
+  for (const r of rows) t += weighted(r);
+  return t;
+}
+
+// The weekly percent now, worked out from the last real reading: the usage
+// recorded in the week up to that reading stands for its percent, so the
+// week up to now stands for this one. Null before any reading.
+export function estimateWeekly(now: number): WindowReading | null {
+  const last = lastReading("seven_day", now);
+  if (!last || last.utilization <= 0) return null;
+  const then = weightedBetween(last.at - WEEK, last.at);
+  if (then <= 0) return null;
+  const pct = (weightedBetween(now - WEEK, now) / then) * last.utilization;
+  return {
+    pct: Math.round(pct * 10) / 10,
+    resetsAt: last.resetsAt,
+    at: now,
+    estimated: true,
+  };
+}
+
+// The window from a reading when it's fresh enough, else (weekly only) the
+// estimate, labelled.
+export function currentWindow(
+  window: LimitWindow,
+  now = Date.now()
+): WindowReading | null {
+  const last = lastReading(window, now);
+  const maxAge =
+    window === "five_hour"
+      ? 30 * 60_000
+      : readUsageConfig().weeklyStaleHours * 3_600_000;
+  // A reading from before its window reset describes the old window.
+  const live =
+    last && now - last.at <= maxAge && (!last.resetsAt || last.resetsAt > now);
+  if (last && live)
+    return {
+      pct: last.utilization,
+      resetsAt: last.resetsAt,
+      at: last.at,
+      estimated: false,
+    };
+  if (window === "seven_day") return estimateWeekly(now);
+  return null;
+}