@saadnvd1/agent-os 0.3.499 → 0.3.501

package/lib/usage/rate-limits.ts

@@ -0,0 +1,192 @@
+/**
+ * The account's limit windows as Claude Code reports them: its
+ * rate_limit_event carries what the anthropic-ratelimit-unified-* headers
+ * said on a response, for the window that matters most at that moment. Each
+ * reading is kept with its time; between readings the weekly figure is an
+ * estimate from recorded usage, and labelled as one.
+ */
+
+import { db } from "../db";
+import { weighted, type TokenRow } from "./weights";
+import { readUsageConfig } from "./config";
+
+export type LimitWindow = "five_hour" | "seven_day";
+
+export interface Reading {
+  window: LimitWindow;
+  // Percent used, 0-100.
+  utilization: number;
+  raw: number;
+  // Epoch ms.
+  resetsAt: number | null;
+  status: string | null;
+}
+
+const WINDOWS: LimitWindow[] = ["five_hour", "seven_day"];
+const num = (v: unknown) =>
+  typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;
+
+// A rate_limit_event carries what the headers said: utilization as a
+// fraction (0.42) and the reset in epoch seconds. A plan's per-window object
+// carries a percent (42) and an ISO time. Each source has one unit.
+const eventPercent = (fraction: number) => fraction * 100;
+
+// Readings in what the CLI emits: a rate_limit_event's info, or a plan's
+// per-window object. Anything else gives none.
+export function parseRateLimit(info: unknown): Reading[] {
+  if (!info || typeof info !== "object") return [];
+  const i = info as Record<string, unknown>;
+  const out: Reading[] = [];
+  const status = typeof i.status === "string" ? i.status : null;
+  const type = i.rateLimitType;
+  const raw = num(i.utilization);
+  if ((type === "five_hour" || type === "seven_day") && raw !== null) {
+    const reset = num(i.resetsAt);
+    out.push({
+      window: type,
+      utilization: eventPercent(raw),
+      raw,
+      resetsAt: reset === null ? null : reset * 1000,
+      status,
+    });
+  }
+  for (const w of WINDOWS) {
+    const o = i[w] as Record<string, unknown> | null | undefined;
+    const u = o && num(o.utilization);
+    if (!o || u === null || u === undefined) continue;
+    const reset =
+      typeof o.resets_at === "string" ? Date.parse(o.resets_at) : NaN;
+    out.push({
+      window: w,
+      utilization: u,
+      raw: u,
+      resetsAt: Number.isFinite(reset) ? reset : null,
+      status,
+    });
+  }
+  return out;
+}
+
+// Stores a reading only when it differs from that window's last, so a busy
+// day adds a handful of rows, not one per response.
+export function recordReadings(
+  readings: Reading[],
+  source: string,
+  sessionId: string | null = null,
+  at = Date.now()
+): number {
+  let added = 0;
+  for (const r of readings) {
+    const last = db
+      .prepare(
+        `SELECT utilization, resets_at, status FROM rate_limit_readings
+         WHERE window = ? ORDER BY id DESC LIMIT 1`
+      )
+      .get(r.window) as
+      | { utilization: number; resets_at: number | null; status: string | null }
+      | undefined;
+    if (
+      last &&
+      last.utilization === r.utilization &&
+      last.resets_at === r.resetsAt &&
+      last.status === r.status
+    )
+      continue;
+    db.prepare(
+      `INSERT INTO rate_limit_readings (at, window, utilization, raw, resets_at, status, session_id, source)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+    ).run(
+      at,
+      r.window,
+      r.utilization,
+      r.raw,
+      r.resetsAt,
+      r.status,
+      sessionId,
+      source
+    );
+    added++;
+  }
+  return added;
+}
+
+export interface WindowReading {
+  pct: number;
+  resetsAt: number | null;
+  // When it was read (or, for an estimate, worked out).
+  at: number;
+  estimated: boolean;
+}
+
+export function lastReading(
+  window: LimitWindow,
+  before = Number.MAX_SAFE_INTEGER
+): { at: number; utilization: number; resetsAt: number | null } | null {
+  const r = db
+    .prepare(
+      `SELECT at, utilization, resets_at FROM rate_limit_readings
+       WHERE window = ? AND at <= ? ORDER BY at DESC, id DESC LIMIT 1`
+    )
+    .get(window, before) as
+    | { at: number; utilization: number; resets_at: number | null }
+    | undefined;
+  return r
+    ? { at: r.at, utilization: r.utilization, resetsAt: r.resets_at }
+    : null;
+}
+
+const WEEK = 7 * 24 * 3600 * 1000;
+
+export function weightedBetween(from: number, to: number): number {
+  const rows = db
+    .prepare(
+      `SELECT model, provider, input_tokens, cache_read_tokens, cache_write_5m_tokens,
+         cache_write_1h_tokens, output_tokens FROM usage_calls WHERE at > ? AND at <= ?`
+    )
+    .all(from, to) as TokenRow[];
+  let t = 0;
+  for (const r of rows) t += weighted(r);
+  return t;
+}
+
+// The weekly percent now, worked out from the last real reading: the usage
+// recorded in the week up to that reading stands for its percent, so the
+// week up to now stands for this one. Null before any reading.
+export function estimateWeekly(now: number): WindowReading | null {
+  const last = lastReading("seven_day", now);
+  if (!last || last.utilization <= 0) return null;
+  const then = weightedBetween(last.at - WEEK, last.at);
+  if (then <= 0) return null;
+  const pct = (weightedBetween(now - WEEK, now) / then) * last.utilization;
+  return {
+    pct: Math.round(pct * 10) / 10,
+    resetsAt: last.resetsAt,
+    at: now,
+    estimated: true,
+  };
+}
+
+// The window from a reading when it's fresh enough, else (weekly only) the
+// estimate, labelled.
+export function currentWindow(
+  window: LimitWindow,
+  now = Date.now()
+): WindowReading | null {
+  const last = lastReading(window, now);
+  const maxAge =
+    window === "five_hour"
+      ? 30 * 60_000
+      : readUsageConfig().weeklyStaleHours * 3_600_000;
+  // A reading from before its window reset describes the old window.
+  const live =
+    last && now - last.at <= maxAge && (!last.resetsAt || last.resetsAt > now);
+  if (last && live)
+    return {
+      pct: last.utilization,
+      resetsAt: last.resetsAt,
+      at: last.at,
+      estimated: false,
+    };
+  if (window === "seven_day") return estimateWeekly(now);
+  return null;
+}

package/lib/usage/rollup.ts

@@ -0,0 +1,90 @@
+/**
+ * One day of usage and quality, from the ledger and the quality readers.
+ * Rows keep their sums as well as their ratios, so any span of days
+ * aggregates as a ratio of sums: three days' cost per PR is their total cost
+ * over their total PRs, never a mean of three ratios.
+ */
+
+import { db } from "../db";
+import { currentWindow } from "./rate-limits";
+import {
+  escalationsIn,
+  escapesIn,
+  mergedTasks,
+  qualityOf,
+  scopeChangesIn,
+  type Gh,
+} from "./quality";
+
+export * from "./span";
+import { computeRollup, type CallGroup, type Rollup } from "./span";
+
+export const dayKey = (ms: number) => {
+  const d = new Date(ms);
+  return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
+};
+export const dayStart = (day: string) => {
+  const [y, m, d] = day.split("-").map(Number);
+  return new Date(y, m - 1, d).getTime();
+};
+export const dayEnd = (day: string) => {
+  const [y, m, d] = day.split("-").map(Number);
+  return new Date(y, m - 1, d + 1).getTime();
+};
+
+function groupsIn(from: number, to: number): CallGroup[] {
+  return db
+    .prepare(
+      `SELECT kind, role, model, provider, COUNT(*) AS calls,
+         SUM(input_tokens) AS input_tokens, SUM(cache_read_tokens) AS cache_read_tokens,
+         SUM(cache_write_5m_tokens) AS cache_write_5m_tokens,
+         SUM(cache_write_1h_tokens) AS cache_write_1h_tokens,
+         SUM(output_tokens) AS output_tokens,
+         SUM(ctx_tokens IS NOT NULL) AS ctxKnown, SUM(ctx_tokens > 200000) AS over200k
+       FROM usage_calls WHERE at >= ? AND at < ?
+       GROUP BY kind, role, model, provider`
+    )
+    .all(from, to) as CallGroup[];
+}
+
+export async function rollupDay(
+  day: string,
+  now = Date.now(),
+  gh?: Gh
+): Promise<Rollup> {
+  const from = dayStart(day);
+  const to = dayEnd(day);
+  const tasks = mergedTasks(from, to);
+  const q = qualityOf(tasks);
+  const weekly = currentWindow("seven_day", Math.min(now, to));
+  return computeRollup({
+    day,
+    groups: groupsIn(from, to),
+    merged: tasks.length,
+    clean: q.filter((t) => t.clean).length,
+    fixRounds: q.reduce((t, x) => t + x.fixRounds, 0),
+    escalations: escalationsIn(from, to),
+    scopeChanges: scopeChangesIn(from, to),
+    escapes: await escapesIn(tasks, gh),
+    weeklyPct: weekly?.pct ?? null,
+    weeklyEstimated: weekly?.estimated ?? false,
+  });
+}
+
+export function saveRollup(r: Rollup, final: boolean, now = Date.now()): void {
+  db.prepare(
+    `INSERT INTO usage_rollups (day, data, final, computed_at) VALUES (?, ?, ?, ?)
+     ON CONFLICT(day) DO UPDATE SET data = excluded.data, final = excluded.final,
+       computed_at = excluded.computed_at`
+  ).run(r.day, JSON.stringify(r), final ? 1 : 0, now);
+}
+
+export function loadRollups(limit = 60): Rollup[] {
+  return (
+    db
+      .prepare(`SELECT data FROM usage_rollups ORDER BY day DESC LIMIT ?`)
+      .all(limit) as { data: string }[]
+  )
+    .map((r) => JSON.parse(r.data) as Rollup)
+    .reverse();
+}

package/lib/usage/span.ts

@@ -0,0 +1,152 @@
+/**
+ * The pure half of the daily rollup: one day from its inputs, and any span
+ * of days as ratios of sums. No database, so the Usage view can use it too.
+ * A day keeps its sums as well as its ratios, so three days' cost per PR is
+ * their total cost over their total PRs, never a mean of three ratios.
+ */
+
+import { weighted, family, type TokenRow } from "./weights";
+
+export interface CallGroup extends TokenRow {
+  kind: string;
+  role: string | null;
+  calls: number;
+  // Calls with a known context size, and those of them above 200k.
+  ctxKnown: number;
+  over200k: number;
+}
+
+export interface Rollup {
+  day: string;
+  weighted: number;
+  // Everything but a person's own sessions.
+  work: number;
+  review: number;
+  orchestrator: number;
+  mix: { opus: number; sonnet: number; haiku: number };
+  calls: number;
+  ctxKnown: number;
+  over200k: number;
+  merged: number;
+  clean: number;
+  fixRounds: number;
+  escalations: number;
+  scopeChanges: number;
+  escapes: number | null;
+  weeklyPct: number | null;
+  weeklyEstimated: boolean;
+}
+
+export interface DayInputs {
+  day: string;
+  groups: CallGroup[];
+  merged: number;
+  clean: number;
+  fixRounds: number;
+  escalations: number;
+  scopeChanges: number;
+  escapes: number | null;
+  weeklyPct: number | null;
+  weeklyEstimated: boolean;
+}
+
+export function computeRollup(i: DayInputs): Rollup {
+  const r: Rollup = {
+    day: i.day,
+    weighted: 0,
+    work: 0,
+    review: 0,
+    orchestrator: 0,
+    mix: { opus: 0, sonnet: 0, haiku: 0 },
+    calls: 0,
+    ctxKnown: 0,
+    over200k: 0,
+    merged: i.merged,
+    clean: i.clean,
+    fixRounds: i.fixRounds,
+    escalations: i.escalations,
+    scopeChanges: i.scopeChanges,
+    escapes: i.escapes,
+    weeklyPct: i.weeklyPct,
+    weeklyEstimated: i.weeklyEstimated,
+  };
+  for (const g of i.groups) {
+    const w = weighted(g);
+    if (w <= 0 && g.provider && g.provider !== "anthropic") continue;
+    r.weighted += w;
+    r.mix[family(g.model)] += w;
+    r.calls += g.calls;
+    r.ctxKnown += g.ctxKnown;
+    r.over200k += g.over200k;
+    if (g.role !== "interactive") r.work += w;
+    if (g.kind === "review" || g.kind === "gate") r.review += w;
+    if (g.role === "orchestrator") r.orchestrator += w;
+  }
+  return r;
+}
+
+// A span of days, as ratios of sums.
+export interface Span {
+  days: number;
+  weighted: number;
+  work: number;
+  merged: number;
+  clean: number;
+  costPerMerged: number | null;
+  costPerClean: number | null;
+  cleanRate: number | null;
+  reviewPerPr: number | null;
+  over200kShare: number | null;
+  orchestratorShare: number | null;
+  opusShare: number | null;
+  fixRoundsPerPr: number | null;
+  escalations: number;
+  scopeChanges: number;
+  escapes: number | null;
+}
+
+const ratio = (a: number, b: number) => (b > 0 ? a / b : null);
+
+export function spanOf(rows: Rollup[]): Span {
+  const sum = (f: (r: Rollup) => number) => rows.reduce((t, r) => t + f(r), 0);
+  const work = sum((r) => r.work);
+  const merged = sum((r) => r.merged);
+  const clean = sum((r) => r.clean);
+  const weightedTotal = sum((r) => r.weighted);
+  const known = rows.filter((r) => r.escapes !== null);
+  return {
+    days: rows.length,
+    weighted: weightedTotal,
+    work,
+    merged,
+    clean,
+    costPerMerged: ratio(work, merged),
+    costPerClean: ratio(work, clean),
+    cleanRate: ratio(clean, merged),
+    reviewPerPr: ratio(
+      sum((r) => r.review),
+      merged
+    ),
+    over200kShare: ratio(
+      sum((r) => r.over200k),
+      sum((r) => r.ctxKnown)
+    ),
+    orchestratorShare: ratio(
+      sum((r) => r.orchestrator),
+      weightedTotal
+    ),
+    opusShare: ratio(
+      sum((r) => r.mix.opus),
+      weightedTotal
+    ),
+    fixRoundsPerPr: ratio(
+      sum((r) => r.fixRounds),
+      merged
+    ),
+    escalations: sum((r) => r.escalations),
+    scopeChanges: sum((r) => r.scopeChanges),
+    escapes: known.length
+      ? known.reduce((t, r) => t + (r.escapes ?? 0), 0)
+      : null,
+  };
+}

package/lib/usage/transcript-import.ts

@@ -0,0 +1,226 @@
+/**
+ * Usage from Claude's own transcripts, for the sessions AgentOS runs that it
+ * can't watch live (a terminal session) and for the days before live capture.
+ * Messages are deduped by id against what the workers recorded, and each file
+ * is read from where the last pass stopped.
+ */
+
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { db } from "../db";
+import {
+  callFromAssistant,
+  recordCall,
+  sessionContext,
+  type CallKind,
+} from "./calls";
+
+const claudeDir = () =>
+  process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), ".claude");
+
+interface TranscriptFile {
+  file: string;
+  sessionId: string;
+  // Set for a subagent's file.
+  agentType: string | null;
+  sidechain: boolean;
+}
+
+// Every transcript of a session AgentOS knows, modified since `since`.
+export async function transcriptFiles(
+  since: number,
+  root = path.join(claudeDir(), "projects")
+): Promise<TranscriptFile[]> {
+  const known = new Map<string, string>();
+  const rows = db
+    .prepare(
+      `SELECT id, claude_session_id FROM sessions WHERE claude_session_id IS NOT NULL`
+    )
+    .all() as { id: string; claude_session_id: string }[];
+  for (const r of rows) known.set(r.claude_session_id, r.id);
+  const out: TranscriptFile[] = [];
+  const dirs = await fs.promises.readdir(root).catch(() => [] as string[]);
+  for (const d of dirs) {
+    const dir = path.join(root, d);
+    const entries = await fs.promises
+      .readdir(dir, { withFileTypes: true })
+      .catch(() => []);
+    for (const e of entries) {
+      if (e.isFile() && e.name.endsWith(".jsonl")) {
+        const claudeId = e.name.slice(0, -".jsonl".length);
+        const sid = known.get(claudeId);
+        if (!sid || !(await newer(path.join(dir, e.name), since))) continue;
+        out.push({
+          file: path.join(dir, e.name),
+          sessionId: sid,
+          agentType: null,
+          sidechain: false,
+        });
+      } else if (e.isDirectory() && known.has(e.name)) {
+        const sid = known.get(e.name)!;
+        const sub = path.join(dir, e.name, "subagents");
+        const files = await fs.promises
+          .readdir(sub)
+          .catch(() => [] as string[]);
+        for (const f of files) {
+          if (!f.endsWith(".jsonl")) continue;
+          const file = path.join(sub, f);
+          if (!(await newer(file, since))) continue;
+          out.push({
+            file,
+            sessionId: sid,
+            agentType: await agentTypeOf(file),
+            sidechain: true,
+          });
+        }
+      }
+    }
+  }
+  return out;
+}
+
+async function newer(file: string, since: number): Promise<boolean> {
+  const st = await fs.promises.stat(file).catch(() => null);
+  return !!st && st.mtimeMs >= since;
+}
+
+async function agentTypeOf(file: string): Promise<string | null> {
+  try {
+    const meta = JSON.parse(
+      await fs.promises.readFile(file.replace(/\.jsonl$/, ".meta.json"), "utf8")
+    ) as { agentType?: unknown };
+    return typeof meta.agentType === "string" ? meta.agentType : null;
+  } catch {
+    return null;
+  }
+}
+
+// Each line of the file from `start` that ends in a newline, as bytes; a last
+// line still being written is left for the next pass.
+async function* completeLines(
+  file: string,
+  start: number,
+  end: number
+): AsyncGenerator<Buffer> {
+  if (end <= start) return;
+  let rest: Buffer = Buffer.alloc(0);
+  // `end` is inclusive, so a line appended during the read waits too.
+  const stream = fs.createReadStream(file, { start, end: end - 1 });
+  for await (const chunk of stream) {
+    rest = rest.length
+      ? Buffer.concat([rest, chunk as Buffer])
+      : (chunk as Buffer);
+    let from = 0;
+    for (let i = rest.indexOf(10, from); i !== -1; i = rest.indexOf(10, from)) {
+      yield rest.subarray(from, i);
+      from = i + 1;
+    }
+    rest = rest.subarray(from);
+  }
+}
+
+interface Line {
+  type?: string;
+  timestamp?: string;
+  attributionAgent?: string;
+  attributionSkill?: string;
+  message?: Parameters<typeof callFromAssistant>[0];
+}
+
+// Imports one file from its stored offset. Returns the calls it saw.
+export async function importFile(
+  f: TranscriptFile,
+  since: number
+): Promise<number> {
+  const st = await fs.promises.stat(f.file);
+  const saved = db
+    .prepare(`SELECT offset, mtime FROM usage_import_offsets WHERE path = ?`)
+    .get(f.file) as { offset: number; mtime: number } | undefined;
+  // A file shorter than where we stopped was rewritten: start again.
+  const start = saved && saved.offset <= st.size ? saved.offset : 0;
+  if (
+    saved &&
+    saved.offset === st.size &&
+    saved.mtime === Math.floor(st.mtimeMs)
+  )
+    return 0;
+  const ctx = sessionContext(f.sessionId);
+  let offset = start;
+  let n = 0;
+  const pending: (() => void)[] = [];
+  for await (const raw of completeLines(f.file, start, st.size)) {
+    // Counted in bytes as they sit in the file, whatever ends the line.
+    offset += raw.length + 1;
+    const line = raw.toString("utf8");
+    if (!line.includes('"assistant"')) continue;
+    let d: Line;
+    try {
+      d = JSON.parse(line) as Line;
+    } catch {
+      continue;
+    }
+    if (d.type !== "assistant") continue;
+    const at = d.timestamp ? Date.parse(d.timestamp) : NaN;
+    if (!Number.isFinite(at) || at < since) continue;
+    const agent = f.agentType ?? d.attributionAgent ?? null;
+    const kind: CallKind = !f.sidechain
+      ? "main"
+      : agent?.startsWith("review-") || d.attributionSkill === "do-code-review"
+        ? "review"
+        : "sub";
+    const call = callFromAssistant(
+      d.message,
+      kind,
+      f.sidechain ? agent : null,
+      at
+    );
+    if (!call) continue;
+    pending.push(() => recordCall(call, ctx, "transcript"));
+    n++;
+  }
+  db.transaction(() => {
+    for (const w of pending) w();
+    db.prepare(
+      `INSERT INTO usage_import_offsets (path, offset, mtime) VALUES (?, ?, ?)
+       ON CONFLICT(path) DO UPDATE SET offset = excluded.offset, mtime = excluded.mtime`
+    ).run(f.file, offset, Math.floor(st.mtimeMs));
+  })();
+  return n;
+}
+
+// Whether a file is as the last pass left it.
+function unchanged(f: TranscriptFile, st: fs.Stats): boolean {
+  const saved = db
+    .prepare(`SELECT offset, mtime FROM usage_import_offsets WHERE path = ?`)
+    .get(f.file) as { offset: number; mtime: number } | undefined;
+  return (
+    !!saved &&
+    saved.offset === st.size &&
+    saved.mtime === Math.floor(st.mtimeMs)
+  );
+}
+
+// One pass over everything modified since `since`, yielding between files so
+// the server keeps answering. Files already up to date don't count against
+// the cap, and the rest go oldest first, so a backlog drains over passes
+// instead of the same first files being taken every time.
+export async function importTranscripts(
+  since: number,
+  maxFiles = 400,
+  root?: string
+): Promise<{ files: number; calls: number }> {
+  const todo: { f: TranscriptFile; mtime: number }[] = [];
+  for (const f of await transcriptFiles(since, root)) {
+    const st = await fs.promises.stat(f.file).catch(() => null);
+    if (st && !unchanged(f, st)) todo.push({ f, mtime: st.mtimeMs });
+  }
+  todo.sort((a, b) => a.mtime - b.mtime);
+  let calls = 0;
+  const batch = todo.slice(0, maxFiles);
+  for (const { f } of batch) {
+    calls += await importFile(f, since).catch(() => 0);
+    await new Promise((r) => setImmediate(r));
+  }
+  return { files: batch.length, calls };
+}