@saadnvd1/agent-os 0.3.498 → 0.3.501

package/lib/usage/check-runs.ts

@@ -6,6 +6,7 @@
  */
 
 import { db } from "../db";
+import { recordCheckRunCall } from "./calls";
 import { formatTokens } from "../chat/context";
 
 export interface RunUsage {
@@ -93,27 +94,47 @@ export function recordCheckRun(
   u: RunUsage,
   at = Date.now()
 ): void {
-  db.prepare(
-    `INSERT INTO check_runs (workspace_id, session_id, label, pr_number, sha, kind, model, provider,
+  const info = db
+    .prepare(
+      `INSERT INTO check_runs (workspace_id, session_id, label, pr_number, sha, kind, model, provider,
        input_tokens, cache_write_tokens, cache_read_tokens, output_tokens, cost_usd, duration_ms, at)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
-  ).run(
-    target.workspaceId,
-    target.sessionId,
-    target.label,
-    target.prNumber,
-    target.sha,
-    target.kind,
-    u.model,
-    u.provider ?? "anthropic",
-    u.inputTokens,
-    u.cacheWriteTokens,
-    u.cacheReadTokens,
-    u.outputTokens,
-    u.costUsd,
-    u.durationMs,
-    at
-  );
+    )
+    .run(
+      target.workspaceId,
+      target.sessionId,
+      target.label,
+      target.prNumber,
+      target.sha,
+      target.kind,
+      u.model,
+      u.provider ?? "anthropic",
+      u.inputTokens,
+      u.cacheWriteTokens,
+      u.cacheReadTokens,
+      u.outputTokens,
+      u.costUsd,
+      u.durationMs,
+      at
+    );
+  try {
+    recordCheckRunCall(Number(info.lastInsertRowid), {
+      workspace_id: target.workspaceId,
+      session_id: target.sessionId,
+      label: target.label,
+      kind: target.kind,
+      model: u.model,
+      provider: u.provider ?? "anthropic",
+      input_tokens: u.inputTokens,
+      cache_write_tokens: u.cacheWriteTokens,
+      cache_read_tokens: u.cacheReadTokens,
+      output_tokens: u.outputTokens,
+      at,
+    });
+  } catch (e) {
+    // The nightly backfill picks it up.
+    console.error(`[check-runs] ledger ${target.label}:`, e);
+  }
 }
 
 // A recorder for one check, that never lets a write break the check.

package/lib/usage/config.ts

@@ -0,0 +1,56 @@
+/**
+ * The ratchet's thresholds, kept in the settings table so they can change
+ * without a deploy. Each falls back to its default for a missing or invalid
+ * value.
+ */
+
+import { db } from "../db";
+
+export const USAGE_KEYS = {
+  costPct: "usage.alert_cost_pct",
+  cleanPoints: "usage.alert_clean_points",
+  baselineDays: "usage.baseline_days",
+  avgDays: "usage.avg_days",
+  minMerges: "usage.alert_min_merges",
+  weeklyStaleHours: "usage.weekly_stale_hours",
+} as const;
+
+export interface UsageConfig {
+  // Alert when the 3-day cost per merged PR is this many percent over baseline.
+  costPct: number;
+  // Alert when the clean-merge rate is this many points under baseline.
+  cleanPoints: number;
+  baselineDays: number;
+  avgDays: number;
+  // Fewer merges than this over the average window: too little to judge.
+  minMerges: number;
+  weeklyStaleHours: number;
+}
+
+export const USAGE_DEFAULTS: UsageConfig = {
+  costPct: 15,
+  cleanPoints: 10,
+  baselineDays: 7,
+  avgDays: 3,
+  minMerges: 3,
+  weeklyStaleHours: 6,
+};
+
+const DAY_COUNTS = new Set<keyof UsageConfig>(["baselineDays", "avgDays"]);
+
+export function readUsageConfig(): UsageConfig {
+  const out: UsageConfig = { ...USAGE_DEFAULTS };
+  for (const [name, key] of Object.entries(USAGE_KEYS)) {
+    const row = db
+      .prepare(`SELECT value FROM settings WHERE key = ?`)
+      .get(key) as { value: string } | undefined;
+    const v = row ? Number(row.value) : NaN;
+    const key2 = name as keyof UsageConfig;
+    // A day count is a whole number of days, at least one.
+    const ok = DAY_COUNTS.has(key2)
+      ? Number.isInteger(v) && v >= 1
+      : Number.isFinite(v) && v >= 0;
+    if (ok) out[key2] = v;
+  }
+  return out;
+}

package/lib/usage/nightly.ts

@@ -0,0 +1,157 @@
+/**
+ * The nightly rollup, inside the server, with no model: it imports the
+ * transcripts it couldn't watch, rolls each recent day up, moves the
+ * ratchet's baseline, and alerts once when the last days regressed. A light
+ * pass every half hour keeps today's row (and cost on task cards) current.
+ */
+
+import { db } from "../db";
+import { notifyPhone } from "../notify";
+import { backfillCheckRuns } from "./calls";
+import { readUsageConfig } from "./config";
+import type { Gh } from "./quality";
+import {
+  applyAlertState,
+  evaluateAlerts,
+  loadBaseline,
+  nextAlertState,
+  nextBaseline,
+  openAlerts,
+  saveBaseline,
+} from "./ratchet";
+import { dayKey, dayStart, loadRollups, rollupDay, saveRollup } from "./rollup";
+import { importTranscripts } from "./transcript-import";
+
+const DAY = 24 * 3600 * 1000;
+const LAST_RUN = "usage.last_nightly";
+const TICK_MS = 30 * 60 * 1000;
+// A day's escapes settle three days after it ends.
+const SETTLE_MS = 3 * DAY;
+
+export interface NightlyDeps {
+  now: () => number;
+  gh?: Gh;
+  notify: (text: string) => void;
+  transcriptsRoot?: string;
+}
+
+export const realDeps: NightlyDeps = {
+  now: Date.now,
+  notify: (text) => notifyPhone("usage", text),
+};
+
+const addDays = (day: string, n: number) =>
+  dayKey(dayStart(day) + n * DAY + DAY / 2);
+
+function setting(key: string): string | null {
+  const r = db.prepare(`SELECT value FROM settings WHERE key = ?`).get(key) as
+    | { value: string }
+    | undefined;
+  return r?.value ?? null;
+}
+
+// Imports what's new and keeps today's row current.
+export async function lightPass(deps: NightlyDeps): Promise<void> {
+  const now = deps.now();
+  const seeded = loadRollups(1).length > 0;
+  await importTranscripts(
+    now - (seeded ? 3 : readUsageConfig().baselineDays + 1) * DAY,
+    400,
+    deps.transcriptsRoot
+  );
+  backfillCheckRuns();
+  const today = dayKey(now);
+  saveRollup(await rollupDay(today, now, undefined), false, now);
+}
+
+// The once-a-day pass.
+export async function nightly(deps: NightlyDeps): Promise<void> {
+  const now = deps.now();
+  const cfg = readUsageConfig();
+  const today = dayKey(now);
+  await lightPass(deps);
+  const existing = loadRollups(60);
+  const last = existing.filter((r) => r.day < today).at(-1)?.day;
+  let day = last ? addDays(last, -3) : addDays(today, -cfg.baselineDays);
+  for (; day < today; day = addDays(day, 1)) {
+    const final = dayStart(day) + DAY + SETTLE_MS <= now;
+    saveRollup(await rollupDay(day, now, deps.gh), final, now);
+  }
+  const complete = loadRollups(60).filter((r) => r.day < today);
+  const baseline = nextBaseline(loadBaseline(), complete, cfg, now);
+  if (baseline) saveBaseline(baseline);
+  const state = nextAlertState(
+    openAlerts(),
+    evaluateAlerts(complete, baseline, cfg)
+  );
+  applyAlertState(state, now);
+  for (const a of state.notify) deps.notify(a.message);
+  db.prepare(
+    `INSERT INTO settings (key, value) VALUES (?, ?)
+     ON CONFLICT(key) DO UPDATE SET value = excluded.value`
+  ).run(LAST_RUN, today);
+}
+
+// Whether this server runs the rollup: a dev server only when asked, so it
+// never alerts for a database that isn't real.
+export function usageRollupEnabled(
+  env: Record<string, string | undefined>
+): boolean {
+  const dev = env.AGENTOS_DEV === "1" || env.NODE_ENV !== "production";
+  return dev
+    ? env.AGENTOS_USAGE_ROLLUP === "on"
+    : env.AGENTOS_USAGE_ROLLUP !== "off";
+}
+
+const g = globalThis as unknown as {
+  __agentosUsage?: { stop: () => void };
+  __agentosUsageBusy?: boolean;
+};
+
+// Runs a pass unless one is already running, in this process: the timer and
+// the API route share the flag (on globalThis, as the server and the routes
+// load separate copies of this module). Returns whether it ran.
+export async function exclusive(pass: () => Promise<void>): Promise<boolean> {
+  if (g.__agentosUsageBusy) return false;
+  g.__agentosUsageBusy = true;
+  try {
+    await pass();
+  } finally {
+    g.__agentosUsageBusy = false;
+  }
+  return true;
+}
+
+export const usageBusy = () => !!g.__agentosUsageBusy;
+
+// The due pass: the full one once a day, just after midnight.
+const duePass = (deps: NightlyDeps) =>
+  setting(LAST_RUN) !== dayKey(deps.now()) ? nightly(deps) : lightPass(deps);
+
+export function startUsageRollup(deps: NightlyDeps = realDeps): {
+  stop: () => void;
+} {
+  if (g.__agentosUsage) return g.__agentosUsage;
+  let timer: NodeJS.Timeout | undefined;
+  let stopped = false;
+  const run = () =>
+    exclusive(() => duePass(deps)).catch((error) =>
+      console.error("[usage] rollup failed:", error)
+    );
+  const next = () => {
+    if (stopped) return;
+    timer = setTimeout(() => void run().finally(next), TICK_MS);
+    timer.unref();
+  };
+  // Not at boot: the server is busy resuming sessions then.
+  timer = setTimeout(() => void run().finally(next), 60_000);
+  timer.unref();
+  g.__agentosUsage = {
+    stop: () => {
+      stopped = true;
+      clearTimeout(timer);
+      g.__agentosUsage = undefined;
+    },
+  };
+  return g.__agentosUsage;
+}

package/lib/usage/quality.ts

@@ -0,0 +1,239 @@
+/**
+ * Quality of the work that merged, from data AgentOS already keeps (the
+ * orchestrator's checks, gate failures, asks and brief amendments) and, for
+ * escapes, the merged PR list from gh. No model is involved.
+ */
+
+import { db } from "../db";
+import { run as runCmd } from "../tasks/gh";
+
+export interface MergedTask {
+  id: string;
+  projectId: string | null;
+  prNumber: number | null;
+  mergedAt: string;
+}
+
+export interface TaskQuality {
+  id: string;
+  reviewed: boolean;
+  clean: boolean;
+  // Heads checked after the first reviewed one.
+  fixRounds: number;
+  gateFailures: number;
+}
+
+export interface DayQuality {
+  merged: number;
+  clean: number;
+  fixRounds: number;
+  escalations: number;
+  scopeChanges: number;
+  // Null when gh couldn't say.
+  escapes: number | null;
+}
+
+interface CheckRow {
+  id: number;
+  session_id: string;
+  sha: string;
+  kind: string;
+  status: string;
+}
+
+// Clean: the first reviewed head passed review and CI, nothing failed a
+// gate, and no later head was checked. A merge with no review at all isn't
+// clean: it can't be told apart from one that skipped the gate.
+export function taskQuality(
+  id: string,
+  checks: CheckRow[],
+  gateFailures: number
+): TaskQuality {
+  const mine = checks
+    .filter((c) => c.session_id === id)
+    .sort((a, b) => a.id - b.id);
+  const first = mine.find(
+    (c) => c.kind === "review" && (c.status === "pass" || c.status === "block")
+  );
+  if (!first)
+    return { id, reviewed: false, clean: false, fixRounds: 0, gateFailures };
+  const ci = mine.find((c) => c.kind === "ci" && c.sha === first.sha);
+  const later = new Set(
+    mine.filter((c) => c.id > first.id && c.sha !== first.sha).map((c) => c.sha)
+  );
+  return {
+    id,
+    reviewed: true,
+    clean:
+      first.status === "pass" &&
+      ci?.status === "pass" &&
+      gateFailures === 0 &&
+      later.size === 0,
+    fixRounds: later.size,
+    gateFailures,
+  };
+}
+
+const sqlTime = (ms: number) =>
+  new Date(ms).toISOString().slice(0, 19).replace("T", " ");
+
+export function mergedTasks(from: number, to: number): MergedTask[] {
+  return (
+    db
+      .prepare(
+        `SELECT id, project_id AS projectId, pr_number AS prNumber, merged_at AS mergedAt
+         FROM sessions WHERE task_status = 'merged' AND merged_at >= ? AND merged_at < ?`
+      )
+      .all(sqlTime(from), sqlTime(to)) as MergedTask[]
+  ).filter((t) => t.mergedAt);
+}
+
+export function qualityOf(tasks: MergedTask[]): TaskQuality[] {
+  if (!tasks.length) return [];
+  const marks = tasks.map(() => "?").join(",");
+  const ids = tasks.map((t) => t.id);
+  const checks = db
+    .prepare(
+      `SELECT id, session_id, sha, kind, status FROM orchestrator_checks
+       WHERE session_id IN (${marks})`
+    )
+    .all(...ids) as CheckRow[];
+  const gates = new Map(
+    (
+      db
+        .prepare(
+          `SELECT session_id, SUM(count) AS n FROM orchestrator_gate_failures
+           WHERE session_id IN (${marks}) GROUP BY session_id`
+        )
+        .all(...ids) as { session_id: string; n: number }[]
+    ).map((g) => [g.session_id, g.n])
+  );
+  return tasks.map((t) => taskQuality(t.id, checks, gates.get(t.id) ?? 0));
+}
+
+// Escalations to Saad in the range: gates that gave up on a task, and gate
+// asks, counted once per task.
+export function escalationsIn(from: number, to: number): number {
+  const f = sqlTime(from);
+  const t = sqlTime(to);
+  const rows = db
+    .prepare(
+      `SELECT session_id AS task FROM orchestrator_gate_failures
+         WHERE escalated_at >= ? AND escalated_at < ?
+       UNION
+       SELECT REPLACE(subject, 'task:', '') AS task FROM orchestrator_asks
+         WHERE kind = 'gate' AND created_at >= ? AND created_at < ?`
+    )
+    .all(f, t, f, t);
+  return rows.length;
+}
+
+export function scopeChangesIn(from: number, to: number): number {
+  return (
+    db
+      .prepare(
+        `SELECT COUNT(*) AS n FROM brief_amendments WHERE created_at >= ? AND created_at < ?`
+      )
+      .get(sqlTime(from), sqlTime(to)) as { n: number }
+  ).n;
+}
+
+export interface MergedPR {
+  number: number;
+  title: string;
+  mergedAt: number;
+  files: string[];
+}
+
+const FIX = /^(revert|fix)(\([^)]*\))?!?:|^revert\b/i;
+export const ESCAPE_WINDOW_MS = 3 * 24 * 3600 * 1000;
+
+// PRs in `merged` that a later revert or fix PR touched the same files of,
+// within three days of the merge.
+export function escapesOf(merged: MergedPR[], all: MergedPR[]): MergedPR[] {
+  return merged.filter((p) =>
+    all.some(
+      (q) =>
+        q.number !== p.number &&
+        q.mergedAt > p.mergedAt &&
+        q.mergedAt - p.mergedAt <= ESCAPE_WINDOW_MS &&
+        FIX.test(q.title.trim()) &&
+        q.files.some((f) => p.files.includes(f))
+    )
+  );
+}
+
+export type Gh = (args: string[], cwd: string) => Promise<string>;
+const realGh: Gh = (args, cwd) => runCmd("gh", args, cwd, 60_000);
+
+function repoOf(projectId: string | null): string | null {
+  if (!projectId) return null;
+  const r = db
+    .prepare(
+      `SELECT path FROM project_repositories WHERE project_id = ?
+       ORDER BY is_primary DESC, sort_order LIMIT 1`
+    )
+    .get(projectId) as { path: string } | undefined;
+  return r?.path ?? null;
+}
+
+// Escapes among the tasks merged in [from, to), looking three days past each
+// merge. Null when gh couldn't say for any repo: unknown is not zero.
+export async function escapesIn(
+  tasks: MergedTask[],
+  gh: Gh = realGh
+): Promise<number | null> {
+  const withPR = tasks.filter((t) => t.prNumber !== null);
+  if (!withPR.length) return 0;
+  const byRepo = new Map<string, MergedTask[]>();
+  for (const t of withPR) {
+    const repo = repoOf(t.projectId);
+    if (!repo) return null;
+    byRepo.set(repo, [...(byRepo.get(repo) ?? []), t]);
+  }
+  let total = 0;
+  for (const [repo, ts] of byRepo) {
+    try {
+      const since = new Date(
+        Math.min(
+          ...ts.map((t) => Date.parse(t.mergedAt.replace(" ", "T") + "Z"))
+        ) - 86_400_000
+      )
+        .toISOString()
+        .slice(0, 10);
+      const out = await gh(
+        [
+          "pr",
+          "list",
+          "--state",
+          "merged",
+          "--search",
+          `merged:>=${since}`,
+          "--json",
+          "number,title,mergedAt,files",
+          "--limit",
+          "300",
+        ],
+        repo
+      );
+      const all = (
+        JSON.parse(out) as {
+          number: number;
+          title: string;
+          mergedAt: string;
+          files?: { path: string }[];
+        }[]
+      ).map((p) => ({
+        number: p.number,
+        title: p.title,
+        mergedAt: Date.parse(p.mergedAt),
+        files: (p.files ?? []).map((f) => f.path),
+      }));
+      const mine = all.filter((p) => ts.some((t) => t.prNumber === p.number));
+      total += escapesOf(mine, all).length;
+    } catch {
+      return null;
+    }
+  }
+  return total;
+}