@saadnvd1/agent-os 0.3.499 → 0.3.501

package/lib/usage/nightly.ts

@@ -0,0 +1,157 @@
+/**
+ * The nightly rollup, inside the server, with no model: it imports the
+ * transcripts it couldn't watch, rolls each recent day up, moves the
+ * ratchet's baseline, and alerts once when the last days regressed. A light
+ * pass every half hour keeps today's row (and cost on task cards) current.
+ */
+
+import { db } from "../db";
+import { notifyPhone } from "../notify";
+import { backfillCheckRuns } from "./calls";
+import { readUsageConfig } from "./config";
+import type { Gh } from "./quality";
+import {
+  applyAlertState,
+  evaluateAlerts,
+  loadBaseline,
+  nextAlertState,
+  nextBaseline,
+  openAlerts,
+  saveBaseline,
+} from "./ratchet";
+import { dayKey, dayStart, loadRollups, rollupDay, saveRollup } from "./rollup";
+import { importTranscripts } from "./transcript-import";
+
+const DAY = 24 * 3600 * 1000;
+const LAST_RUN = "usage.last_nightly";
+const TICK_MS = 30 * 60 * 1000;
+// A day's escapes settle three days after it ends.
+const SETTLE_MS = 3 * DAY;
+
+export interface NightlyDeps {
+  now: () => number;
+  gh?: Gh;
+  notify: (text: string) => void;
+  transcriptsRoot?: string;
+}
+
+export const realDeps: NightlyDeps = {
+  now: Date.now,
+  notify: (text) => notifyPhone("usage", text),
+};
+
+const addDays = (day: string, n: number) =>
+  dayKey(dayStart(day) + n * DAY + DAY / 2);
+
+function setting(key: string): string | null {
+  const r = db.prepare(`SELECT value FROM settings WHERE key = ?`).get(key) as
+    | { value: string }
+    | undefined;
+  return r?.value ?? null;
+}
+
+// Imports what's new and keeps today's row current.
+export async function lightPass(deps: NightlyDeps): Promise<void> {
+  const now = deps.now();
+  const seeded = loadRollups(1).length > 0;
+  await importTranscripts(
+    now - (seeded ? 3 : readUsageConfig().baselineDays + 1) * DAY,
+    400,
+    deps.transcriptsRoot
+  );
+  backfillCheckRuns();
+  const today = dayKey(now);
+  saveRollup(await rollupDay(today, now, undefined), false, now);
+}
+
+// The once-a-day pass.
+export async function nightly(deps: NightlyDeps): Promise<void> {
+  const now = deps.now();
+  const cfg = readUsageConfig();
+  const today = dayKey(now);
+  await lightPass(deps);
+  const existing = loadRollups(60);
+  const last = existing.filter((r) => r.day < today).at(-1)?.day;
+  let day = last ? addDays(last, -3) : addDays(today, -cfg.baselineDays);
+  for (; day < today; day = addDays(day, 1)) {
+    const final = dayStart(day) + DAY + SETTLE_MS <= now;
+    saveRollup(await rollupDay(day, now, deps.gh), final, now);
+  }
+  const complete = loadRollups(60).filter((r) => r.day < today);
+  const baseline = nextBaseline(loadBaseline(), complete, cfg, now);
+  if (baseline) saveBaseline(baseline);
+  const state = nextAlertState(
+    openAlerts(),
+    evaluateAlerts(complete, baseline, cfg)
+  );
+  applyAlertState(state, now);
+  for (const a of state.notify) deps.notify(a.message);
+  db.prepare(
+    `INSERT INTO settings (key, value) VALUES (?, ?)
+     ON CONFLICT(key) DO UPDATE SET value = excluded.value`
+  ).run(LAST_RUN, today);
+}
+
+// Whether this server runs the rollup: a dev server only when asked, so it
+// never alerts for a database that isn't real.
+export function usageRollupEnabled(
+  env: Record<string, string | undefined>
+): boolean {
+  const dev = env.AGENTOS_DEV === "1" || env.NODE_ENV !== "production";
+  return dev
+    ? env.AGENTOS_USAGE_ROLLUP === "on"
+    : env.AGENTOS_USAGE_ROLLUP !== "off";
+}
+
+const g = globalThis as unknown as {
+  __agentosUsage?: { stop: () => void };
+  __agentosUsageBusy?: boolean;
+};
+
+// Runs a pass unless one is already running, in this process: the timer and
+// the API route share the flag (on globalThis, as the server and the routes
+// load separate copies of this module). Returns whether it ran.
+export async function exclusive(pass: () => Promise<void>): Promise<boolean> {
+  if (g.__agentosUsageBusy) return false;
+  g.__agentosUsageBusy = true;
+  try {
+    await pass();
+  } finally {
+    g.__agentosUsageBusy = false;
+  }
+  return true;
+}
+
+export const usageBusy = () => !!g.__agentosUsageBusy;
+
+// The due pass: the full one once a day, just after midnight.
+const duePass = (deps: NightlyDeps) =>
+  setting(LAST_RUN) !== dayKey(deps.now()) ? nightly(deps) : lightPass(deps);
+
+export function startUsageRollup(deps: NightlyDeps = realDeps): {
+  stop: () => void;
+} {
+  if (g.__agentosUsage) return g.__agentosUsage;
+  let timer: NodeJS.Timeout | undefined;
+  let stopped = false;
+  const run = () =>
+    exclusive(() => duePass(deps)).catch((error) =>
+      console.error("[usage] rollup failed:", error)
+    );
+  const next = () => {
+    if (stopped) return;
+    timer = setTimeout(() => void run().finally(next), TICK_MS);
+    timer.unref();
+  };
+  // Not at boot: the server is busy resuming sessions then.
+  timer = setTimeout(() => void run().finally(next), 60_000);
+  timer.unref();
+  g.__agentosUsage = {
+    stop: () => {
+      stopped = true;
+      clearTimeout(timer);
+      g.__agentosUsage = undefined;
+    },
+  };
+  return g.__agentosUsage;
+}

package/lib/usage/quality.ts

@@ -0,0 +1,239 @@
+/**
+ * Quality of the work that merged, from data AgentOS already keeps (the
+ * orchestrator's checks, gate failures, asks and brief amendments) and, for
+ * escapes, the merged PR list from gh. No model is involved.
+ */
+
+import { db } from "../db";
+import { run as runCmd } from "../tasks/gh";
+
+export interface MergedTask {
+  id: string;
+  projectId: string | null;
+  prNumber: number | null;
+  mergedAt: string;
+}
+
+export interface TaskQuality {
+  id: string;
+  reviewed: boolean;
+  clean: boolean;
+  // Heads checked after the first reviewed one.
+  fixRounds: number;
+  gateFailures: number;
+}
+
+export interface DayQuality {
+  merged: number;
+  clean: number;
+  fixRounds: number;
+  escalations: number;
+  scopeChanges: number;
+  // Null when gh couldn't say.
+  escapes: number | null;
+}
+
+interface CheckRow {
+  id: number;
+  session_id: string;
+  sha: string;
+  kind: string;
+  status: string;
+}
+
+// Clean: the first reviewed head passed review and CI, nothing failed a
+// gate, and no later head was checked. A merge with no review at all isn't
+// clean: it can't be told apart from one that skipped the gate.
+export function taskQuality(
+  id: string,
+  checks: CheckRow[],
+  gateFailures: number
+): TaskQuality {
+  const mine = checks
+    .filter((c) => c.session_id === id)
+    .sort((a, b) => a.id - b.id);
+  const first = mine.find(
+    (c) => c.kind === "review" && (c.status === "pass" || c.status === "block")
+  );
+  if (!first)
+    return { id, reviewed: false, clean: false, fixRounds: 0, gateFailures };
+  const ci = mine.find((c) => c.kind === "ci" && c.sha === first.sha);
+  const later = new Set(
+    mine.filter((c) => c.id > first.id && c.sha !== first.sha).map((c) => c.sha)
+  );
+  return {
+    id,
+    reviewed: true,
+    clean:
+      first.status === "pass" &&
+      ci?.status === "pass" &&
+      gateFailures === 0 &&
+      later.size === 0,
+    fixRounds: later.size,
+    gateFailures,
+  };
+}
+
+const sqlTime = (ms: number) =>
+  new Date(ms).toISOString().slice(0, 19).replace("T", " ");
+
+export function mergedTasks(from: number, to: number): MergedTask[] {
+  return (
+    db
+      .prepare(
+        `SELECT id, project_id AS projectId, pr_number AS prNumber, merged_at AS mergedAt
+         FROM sessions WHERE task_status = 'merged' AND merged_at >= ? AND merged_at < ?`
+      )
+      .all(sqlTime(from), sqlTime(to)) as MergedTask[]
+  ).filter((t) => t.mergedAt);
+}
+
+export function qualityOf(tasks: MergedTask[]): TaskQuality[] {
+  if (!tasks.length) return [];
+  const marks = tasks.map(() => "?").join(",");
+  const ids = tasks.map((t) => t.id);
+  const checks = db
+    .prepare(
+      `SELECT id, session_id, sha, kind, status FROM orchestrator_checks
+       WHERE session_id IN (${marks})`
+    )
+    .all(...ids) as CheckRow[];
+  const gates = new Map(
+    (
+      db
+        .prepare(
+          `SELECT session_id, SUM(count) AS n FROM orchestrator_gate_failures
+           WHERE session_id IN (${marks}) GROUP BY session_id`
+        )
+        .all(...ids) as { session_id: string; n: number }[]
+    ).map((g) => [g.session_id, g.n])
+  );
+  return tasks.map((t) => taskQuality(t.id, checks, gates.get(t.id) ?? 0));
+}
+
+// Escalations to Saad in the range: gates that gave up on a task, and gate
+// asks, counted once per task.
+export function escalationsIn(from: number, to: number): number {
+  const f = sqlTime(from);
+  const t = sqlTime(to);
+  const rows = db
+    .prepare(
+      `SELECT session_id AS task FROM orchestrator_gate_failures
+         WHERE escalated_at >= ? AND escalated_at < ?
+       UNION
+       SELECT REPLACE(subject, 'task:', '') AS task FROM orchestrator_asks
+         WHERE kind = 'gate' AND created_at >= ? AND created_at < ?`
+    )
+    .all(f, t, f, t);
+  return rows.length;
+}
+
+export function scopeChangesIn(from: number, to: number): number {
+  return (
+    db
+      .prepare(
+        `SELECT COUNT(*) AS n FROM brief_amendments WHERE created_at >= ? AND created_at < ?`
+      )
+      .get(sqlTime(from), sqlTime(to)) as { n: number }
+  ).n;
+}
+
+export interface MergedPR {
+  number: number;
+  title: string;
+  mergedAt: number;
+  files: string[];
+}
+
+const FIX = /^(revert|fix)(\([^)]*\))?!?:|^revert\b/i;
+export const ESCAPE_WINDOW_MS = 3 * 24 * 3600 * 1000;
+
+// PRs in `merged` that a later revert or fix PR touched the same files of,
+// within three days of the merge.
+export function escapesOf(merged: MergedPR[], all: MergedPR[]): MergedPR[] {
+  return merged.filter((p) =>
+    all.some(
+      (q) =>
+        q.number !== p.number &&
+        q.mergedAt > p.mergedAt &&
+        q.mergedAt - p.mergedAt <= ESCAPE_WINDOW_MS &&
+        FIX.test(q.title.trim()) &&
+        q.files.some((f) => p.files.includes(f))
+    )
+  );
+}
+
+export type Gh = (args: string[], cwd: string) => Promise<string>;
+const realGh: Gh = (args, cwd) => runCmd("gh", args, cwd, 60_000);
+
+function repoOf(projectId: string | null): string | null {
+  if (!projectId) return null;
+  const r = db
+    .prepare(
+      `SELECT path FROM project_repositories WHERE project_id = ?
+       ORDER BY is_primary DESC, sort_order LIMIT 1`
+    )
+    .get(projectId) as { path: string } | undefined;
+  return r?.path ?? null;
+}
+
+// Escapes among the tasks merged in [from, to), looking three days past each
+// merge. Null when gh couldn't say for any repo: unknown is not zero.
+export async function escapesIn(
+  tasks: MergedTask[],
+  gh: Gh = realGh
+): Promise<number | null> {
+  const withPR = tasks.filter((t) => t.prNumber !== null);
+  if (!withPR.length) return 0;
+  const byRepo = new Map<string, MergedTask[]>();
+  for (const t of withPR) {
+    const repo = repoOf(t.projectId);
+    if (!repo) return null;
+    byRepo.set(repo, [...(byRepo.get(repo) ?? []), t]);
+  }
+  let total = 0;
+  for (const [repo, ts] of byRepo) {
+    try {
+      const since = new Date(
+        Math.min(
+          ...ts.map((t) => Date.parse(t.mergedAt.replace(" ", "T") + "Z"))
+        ) - 86_400_000
+      )
+        .toISOString()
+        .slice(0, 10);
+      const out = await gh(
+        [
+          "pr",
+          "list",
+          "--state",
+          "merged",
+          "--search",
+          `merged:>=${since}`,
+          "--json",
+          "number,title,mergedAt,files",
+          "--limit",
+          "300",
+        ],
+        repo
+      );
+      const all = (
+        JSON.parse(out) as {
+          number: number;
+          title: string;
+          mergedAt: string;
+          files?: { path: string }[];
+        }[]
+      ).map((p) => ({
+        number: p.number,
+        title: p.title,
+        mergedAt: Date.parse(p.mergedAt),
+        files: (p.files ?? []).map((f) => f.path),
+      }));
+      const mine = all.filter((p) => ts.some((t) => t.prNumber === p.number));
+      total += escapesOf(mine, all).length;
+    } catch {
+      return null;
+    }
+  }
+  return total;
+}

package/lib/usage/ratchet.ts

@@ -0,0 +1,275 @@
+/**
+ * The ratchet. The baseline is the first week of rollups, and from then on
+ * only ever moves to a better value. An alert fires, once, when cost per
+ * merged PR or the clean-merge rate has drifted past the configured margin,
+ * naming the supporting metric that moved most.
+ */
+
+import { db } from "../db";
+import { spanOf, type Rollup, type Span } from "./span";
+import type { UsageConfig } from "./config";
+
+export type Direction = "lower" | "higher";
+
+// The better of the two. A missing or invalid candidate keeps what was.
+export function ratchet(
+  prev: number | null,
+  candidate: number | null,
+  better: Direction
+): number | null {
+  if (candidate === null || !Number.isFinite(candidate)) return prev;
+  if (prev === null || !Number.isFinite(prev)) return candidate;
+  return better === "lower"
+    ? Math.min(prev, candidate)
+    : Math.max(prev, candidate);
+}
+
+// What the baseline is judged on.
+export const HEADLINE = {
+  costPerMerged: "lower",
+  costPerClean: "lower",
+  cleanRate: "higher",
+} as const satisfies Record<string, Direction>;
+export type HeadlineKey = keyof typeof HEADLINE;
+
+// Supporting metrics, each where lower is better, per PR where it is a count.
+export interface Supporting {
+  reviewPerPr: number | null;
+  over200kShare: number | null;
+  orchestratorShare: number | null;
+  opusShare: number | null;
+  fixRoundsPerPr: number | null;
+  escalationsPerPr: number | null;
+  scopeChangesPerPr: number | null;
+  escapesPerPr: number | null;
+}
+
+export const SUPPORTING_LABELS: Record<keyof Supporting, string> = {
+  reviewPerPr: "review cost per PR",
+  over200kShare: "share of calls above 200k context",
+  orchestratorShare: "orchestrator share",
+  opusShare: "Opus share of the model mix",
+  fixRoundsPerPr: "fix rounds per PR",
+  escalationsPerPr: "escalations per PR",
+  scopeChangesPerPr: "scope changes per PR",
+  escapesPerPr: "escapes per PR",
+};
+
+const per = (n: number | null, merged: number) =>
+  n !== null && merged > 0 ? n / merged : null;
+
+export function supportingOf(s: Span): Supporting {
+  return {
+    reviewPerPr: s.reviewPerPr,
+    over200kShare: s.over200kShare,
+    orchestratorShare: s.orchestratorShare,
+    opusShare: s.opusShare,
+    fixRoundsPerPr: s.fixRoundsPerPr,
+    escalationsPerPr: per(s.escalations, s.merged),
+    scopeChangesPerPr: per(s.scopeChanges, s.merged),
+    escapesPerPr: per(s.escapes, s.merged),
+  };
+}
+
+export interface Baseline {
+  values: Record<HeadlineKey, number | null>;
+  supporting: Supporting;
+  setAt: number;
+}
+
+// The next baseline. None until `cfg.baselineDays` complete days exist with
+// at least `cfg.minMerges` merges among them: a quiet week would lock in a
+// baseline nothing can meet, so the seed waits for the first week that has
+// enough work, and a ratchet step skips a week that doesn't. The first is
+// those first days, and each night after it, the trailing week of days is
+// ratcheted in. The supporting snapshot follows cost per merged PR:
+// it is replaced only when that improves.
+export function nextBaseline(
+  prev: Baseline | null,
+  complete: Rollup[],
+  cfg: UsageConfig,
+  now = Date.now()
+): Baseline | null {
+  const days = [...complete].sort((a, b) => a.day.localeCompare(b.day));
+  if (days.length < cfg.baselineDays) return prev;
+  if (!prev) {
+    let s = spanOf(days.slice(0, cfg.baselineDays));
+    if (s.merged < cfg.minMerges) s = spanOf(days.slice(-cfg.baselineDays));
+    if (s.merged < cfg.minMerges) return null;
+    return {
+      values: {
+        costPerMerged: s.costPerMerged,
+        costPerClean: s.costPerClean,
+        cleanRate: s.cleanRate,
+      },
+      supporting: supportingOf(s),
+      setAt: now,
+    };
+  }
+  const s = spanOf(days.slice(-cfg.baselineDays));
+  if (s.merged < cfg.minMerges) return prev;
+  const values = { ...prev.values };
+  for (const k of Object.keys(HEADLINE) as HeadlineKey[])
+    values[k] = ratchet(prev.values[k], s[k], HEADLINE[k]);
+  const improved =
+    s.costPerMerged !== null &&
+    (prev.values.costPerMerged === null ||
+      s.costPerMerged < prev.values.costPerMerged);
+  return {
+    values,
+    supporting: improved ? supportingOf(s) : prev.supporting,
+    setAt: improved ? now : prev.setAt,
+  };
+}
+
+export interface Alert {
+  kind: "cost" | "quality";
+  message: string;
+}
+
+const round = (x: number) => Math.round(x * 1e6) / 1e6;
+
+// The supporting metric that moved furthest the wrong way since the baseline.
+export function biggestMover(
+  base: Supporting,
+  now: Supporting
+): { label: string; change: string } | null {
+  let best: { key: keyof Supporting; score: number } | null = null;
+  for (const key of Object.keys(base) as (keyof Supporting)[]) {
+    const b = base[key];
+    const c = now[key];
+    if (b === null || c === null) continue;
+    const score = b > 0 ? (c - b) / b : c > 0 ? 1 : 0;
+    if (score > 0 && (!best || score > best.score)) best = { key, score };
+  }
+  if (!best) return null;
+  const b = base[best.key] as number;
+  return {
+    label: SUPPORTING_LABELS[best.key],
+    change: b > 0 ? `+${Math.round(best.score * 100)}%` : "up from none",
+  };
+}
+
+// Conditions that hold now, over the last `avgDays` complete days.
+export function evaluateAlerts(
+  complete: Rollup[],
+  baseline: Baseline | null,
+  cfg: UsageConfig
+): Alert[] {
+  if (!baseline) return [];
+  const days = [...complete]
+    .sort((a, b) => a.day.localeCompare(b.day))
+    .slice(-cfg.avgDays);
+  const s = spanOf(days);
+  if (days.length < cfg.avgDays || s.merged < cfg.minMerges) return [];
+  const mover = biggestMover(baseline.supporting, supportingOf(s));
+  const moved = mover ? ` Biggest mover: ${mover.label} ${mover.change}.` : "";
+  const out: Alert[] = [];
+  const baseCost = baseline.values.costPerMerged;
+  if (s.costPerMerged !== null && baseCost !== null && baseCost > 0) {
+    const over = round(((s.costPerMerged - baseCost) / baseCost) * 100);
+    if (over > cfg.costPct)
+      out.push({
+        kind: "cost",
+        message: `Usage: cost per merged PR, ${cfg.avgDays}-day average, is ${Math.round(over)}% over baseline.${moved}`,
+      });
+  }
+  const baseClean = baseline.values.cleanRate;
+  if (s.cleanRate !== null && baseClean !== null) {
+    const under = round((baseClean - s.cleanRate) * 100);
+    if (under > cfg.cleanPoints)
+      out.push({
+        kind: "quality",
+        message: `Usage: clean-merge rate, ${cfg.avgDays}-day average, is ${Math.round(under)} points under baseline.${moved}`,
+      });
+  }
+  return out;
+}
+
+// Which firing alerts are new (to send), and which open ones have cleared.
+export function nextAlertState(
+  open: string[],
+  firing: Alert[]
+): { notify: Alert[]; clear: string[] } {
+  const now = new Set(firing.map((a) => a.kind));
+  return {
+    notify: firing.filter((a) => !open.includes(a.kind)),
+    clear: open.filter((k) => !now.has(k as Alert["kind"])),
+  };
+}
+
+export function loadBaseline(): Baseline | null {
+  const rows = db
+    .prepare(`SELECT metric, value, set_at, supporting FROM usage_baseline`)
+    .all() as {
+    metric: string;
+    value: number;
+    set_at: number;
+    supporting: string | null;
+  }[];
+  if (!rows.length) return null;
+  const values: Baseline["values"] = {
+    costPerMerged: null,
+    costPerClean: null,
+    cleanRate: null,
+  };
+  let supporting: Supporting | null = null;
+  let setAt = 0;
+  for (const r of rows) {
+    if (r.metric in values) values[r.metric as HeadlineKey] = r.value;
+    if (r.metric === "costPerMerged" && r.supporting)
+      supporting = JSON.parse(r.supporting) as Supporting;
+    setAt = Math.max(setAt, r.set_at);
+  }
+  return {
+    values,
+    supporting: supporting ?? supportingOf(spanOf([])),
+    setAt,
+  };
+}
+
+export function saveBaseline(b: Baseline): void {
+  db.transaction(() => {
+    for (const k of Object.keys(HEADLINE) as HeadlineKey[]) {
+      const v = b.values[k];
+      if (v === null) continue;
+      db.prepare(
+        `INSERT INTO usage_baseline (metric, value, set_at, supporting) VALUES (?, ?, ?, ?)
+         ON CONFLICT(metric) DO UPDATE SET value = excluded.value, set_at = excluded.set_at,
+           supporting = excluded.supporting`
+      ).run(
+        k,
+        v,
+        b.setAt,
+        k === "costPerMerged" ? JSON.stringify(b.supporting) : null
+      );
+    }
+  })();
+}
+
+export function openAlerts(): string[] {
+  return (
+    db
+      .prepare(`SELECT kind FROM usage_alerts WHERE cleared_at IS NULL`)
+      .all() as { kind: string }[]
+  ).map((r) => r.kind);
+}
+
+export function applyAlertState(
+  state: { notify: Alert[]; clear: string[] },
+  now = Date.now()
+): void {
+  db.transaction(() => {
+    for (const a of state.notify)
+      db.prepare(
+        `INSERT INTO usage_alerts (kind, opened_at, cleared_at, message) VALUES (?, ?, NULL, ?)
+         ON CONFLICT(kind) DO UPDATE SET opened_at = excluded.opened_at, cleared_at = NULL,
+           message = excluded.message`
+      ).run(a.kind, now, a.message);
+    for (const k of state.clear)
+      db.prepare(`UPDATE usage_alerts SET cleared_at = ? WHERE kind = ?`).run(
+        now,
+        k
+      );
+  })();
+}