@saadnvd1/agent-os 0.3.498 → 0.3.501

package/lib/usage/transcript-import.ts

@@ -0,0 +1,226 @@
+/**
+ * Usage from Claude's own transcripts, for the sessions AgentOS runs that it
+ * can't watch live (a terminal session) and for the days before live capture.
+ * Messages are deduped by id against what the workers recorded, and each file
+ * is read from where the last pass stopped.
+ */
+
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { db } from "../db";
+import {
+  callFromAssistant,
+  recordCall,
+  sessionContext,
+  type CallKind,
+} from "./calls";
+
+const claudeDir = () =>
+  process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), ".claude");
+
+interface TranscriptFile {
+  file: string;
+  sessionId: string;
+  // Set for a subagent's file.
+  agentType: string | null;
+  sidechain: boolean;
+}
+
+// Every transcript of a session AgentOS knows, modified since `since`.
+export async function transcriptFiles(
+  since: number,
+  root = path.join(claudeDir(), "projects")
+): Promise<TranscriptFile[]> {
+  const known = new Map<string, string>();
+  const rows = db
+    .prepare(
+      `SELECT id, claude_session_id FROM sessions WHERE claude_session_id IS NOT NULL`
+    )
+    .all() as { id: string; claude_session_id: string }[];
+  for (const r of rows) known.set(r.claude_session_id, r.id);
+  const out: TranscriptFile[] = [];
+  const dirs = await fs.promises.readdir(root).catch(() => [] as string[]);
+  for (const d of dirs) {
+    const dir = path.join(root, d);
+    const entries = await fs.promises
+      .readdir(dir, { withFileTypes: true })
+      .catch(() => []);
+    for (const e of entries) {
+      if (e.isFile() && e.name.endsWith(".jsonl")) {
+        const claudeId = e.name.slice(0, -".jsonl".length);
+        const sid = known.get(claudeId);
+        if (!sid || !(await newer(path.join(dir, e.name), since))) continue;
+        out.push({
+          file: path.join(dir, e.name),
+          sessionId: sid,
+          agentType: null,
+          sidechain: false,
+        });
+      } else if (e.isDirectory() && known.has(e.name)) {
+        const sid = known.get(e.name)!;
+        const sub = path.join(dir, e.name, "subagents");
+        const files = await fs.promises
+          .readdir(sub)
+          .catch(() => [] as string[]);
+        for (const f of files) {
+          if (!f.endsWith(".jsonl")) continue;
+          const file = path.join(sub, f);
+          if (!(await newer(file, since))) continue;
+          out.push({
+            file,
+            sessionId: sid,
+            agentType: await agentTypeOf(file),
+            sidechain: true,
+          });
+        }
+      }
+    }
+  }
+  return out;
+}
+
+async function newer(file: string, since: number): Promise<boolean> {
+  const st = await fs.promises.stat(file).catch(() => null);
+  return !!st && st.mtimeMs >= since;
+}
+
+async function agentTypeOf(file: string): Promise<string | null> {
+  try {
+    const meta = JSON.parse(
+      await fs.promises.readFile(file.replace(/\.jsonl$/, ".meta.json"), "utf8")
+    ) as { agentType?: unknown };
+    return typeof meta.agentType === "string" ? meta.agentType : null;
+  } catch {
+    return null;
+  }
+}
+
+// Each line of the file from `start` that ends in a newline, as bytes; a last
+// line still being written is left for the next pass.
+async function* completeLines(
+  file: string,
+  start: number,
+  end: number
+): AsyncGenerator<Buffer> {
+  if (end <= start) return;
+  let rest: Buffer = Buffer.alloc(0);
+  // `end` is inclusive, so a line appended during the read waits too.
+  const stream = fs.createReadStream(file, { start, end: end - 1 });
+  for await (const chunk of stream) {
+    rest = rest.length
+      ? Buffer.concat([rest, chunk as Buffer])
+      : (chunk as Buffer);
+    let from = 0;
+    for (let i = rest.indexOf(10, from); i !== -1; i = rest.indexOf(10, from)) {
+      yield rest.subarray(from, i);
+      from = i + 1;
+    }
+    rest = rest.subarray(from);
+  }
+}
+
+interface Line {
+  type?: string;
+  timestamp?: string;
+  attributionAgent?: string;
+  attributionSkill?: string;
+  message?: Parameters<typeof callFromAssistant>[0];
+}
+
+// Imports one file from its stored offset. Returns the calls it saw.
+export async function importFile(
+  f: TranscriptFile,
+  since: number
+): Promise<number> {
+  const st = await fs.promises.stat(f.file);
+  const saved = db
+    .prepare(`SELECT offset, mtime FROM usage_import_offsets WHERE path = ?`)
+    .get(f.file) as { offset: number; mtime: number } | undefined;
+  // A file shorter than where we stopped was rewritten: start again.
+  const start = saved && saved.offset <= st.size ? saved.offset : 0;
+  if (
+    saved &&
+    saved.offset === st.size &&
+    saved.mtime === Math.floor(st.mtimeMs)
+  )
+    return 0;
+  const ctx = sessionContext(f.sessionId);
+  let offset = start;
+  let n = 0;
+  const pending: (() => void)[] = [];
+  for await (const raw of completeLines(f.file, start, st.size)) {
+    // Counted in bytes as they sit in the file, whatever ends the line.
+    offset += raw.length + 1;
+    const line = raw.toString("utf8");
+    if (!line.includes('"assistant"')) continue;
+    let d: Line;
+    try {
+      d = JSON.parse(line) as Line;
+    } catch {
+      continue;
+    }
+    if (d.type !== "assistant") continue;
+    const at = d.timestamp ? Date.parse(d.timestamp) : NaN;
+    if (!Number.isFinite(at) || at < since) continue;
+    const agent = f.agentType ?? d.attributionAgent ?? null;
+    const kind: CallKind = !f.sidechain
+      ? "main"
+      : agent?.startsWith("review-") || d.attributionSkill === "do-code-review"
+        ? "review"
+        : "sub";
+    const call = callFromAssistant(
+      d.message,
+      kind,
+      f.sidechain ? agent : null,
+      at
+    );
+    if (!call) continue;
+    pending.push(() => recordCall(call, ctx, "transcript"));
+    n++;
+  }
+  db.transaction(() => {
+    for (const w of pending) w();
+    db.prepare(
+      `INSERT INTO usage_import_offsets (path, offset, mtime) VALUES (?, ?, ?)
+       ON CONFLICT(path) DO UPDATE SET offset = excluded.offset, mtime = excluded.mtime`
+    ).run(f.file, offset, Math.floor(st.mtimeMs));
+  })();
+  return n;
+}
+
+// Whether a file is as the last pass left it.
+function unchanged(f: TranscriptFile, st: fs.Stats): boolean {
+  const saved = db
+    .prepare(`SELECT offset, mtime FROM usage_import_offsets WHERE path = ?`)
+    .get(f.file) as { offset: number; mtime: number } | undefined;
+  return (
+    !!saved &&
+    saved.offset === st.size &&
+    saved.mtime === Math.floor(st.mtimeMs)
+  );
+}
+
+// One pass over everything modified since `since`, yielding between files so
+// the server keeps answering. Files already up to date don't count against
+// the cap, and the rest go oldest first, so a backlog drains over passes
+// instead of the same first files being taken every time.
+export async function importTranscripts(
+  since: number,
+  maxFiles = 400,
+  root?: string
+): Promise<{ files: number; calls: number }> {
+  const todo: { f: TranscriptFile; mtime: number }[] = [];
+  for (const f of await transcriptFiles(since, root)) {
+    const st = await fs.promises.stat(f.file).catch(() => null);
+    if (st && !unchanged(f, st)) todo.push({ f, mtime: st.mtimeMs });
+  }
+  todo.sort((a, b) => a.mtime - b.mtime);
+  let calls = 0;
+  const batch = todo.slice(0, maxFiles);
+  for (const { f } of batch) {
+    calls += await importFile(f, since).catch(() => 0);
+    await new Promise((r) => setImmediate(r));
+  }
+  return { files: batch.length, calls };
+}

package/lib/usage/weights.ts

@@ -0,0 +1,64 @@
+/**
+ * Weighted tokens: the API's price ratios by token type and model family, as
+ * a proxy for how much of the plan's limits a call uses. Opus input = 1.
+ * An estimate, not what the plan charges.
+ */
+
+import { ANTHROPIC } from "../models/roles";
+
+export type Family = "opus" | "sonnet" | "haiku";
+
+export interface TokenRow {
+  model: string | null;
+  provider?: string | null;
+  input_tokens: number;
+  cache_read_tokens: number;
+  cache_write_5m_tokens: number;
+  cache_write_1h_tokens: number;
+  output_tokens: number;
+}
+
+// Unknown Claude ids weigh as Opus.
+export const family = (model: string | null | undefined): Family => {
+  const m = (model ?? "").toLowerCase();
+  return m.includes("sonnet")
+    ? "sonnet"
+    : m.includes("haiku")
+      ? "haiku"
+      : "opus";
+};
+
+const MULT: Record<Family, number> = { opus: 1, sonnet: 0.5, haiku: 0.25 };
+const READ: Record<Family, number> = { opus: 0.1, sonnet: 0.1, haiku: 0.05 };
+
+export const WEIGHTS = {
+  input: 1,
+  cacheWrite5m: 1.25,
+  cacheWrite1h: 2,
+  output: 5,
+} as const;
+
+export function weighted(r: TokenRow): number {
+  if (r.provider && r.provider !== ANTHROPIC) return 0;
+  const f = family(r.model);
+  return (
+    MULT[f] *
+      (WEIGHTS.input * r.input_tokens +
+        WEIGHTS.cacheWrite5m * r.cache_write_5m_tokens +
+        WEIGHTS.cacheWrite1h * r.cache_write_1h_tokens +
+        WEIGHTS.output * r.output_tokens) +
+    READ[f] * r.cache_read_tokens
+  );
+}
+
+export const CTX_LIMIT = 200_000;
+
+// What the call sent: fresh input, cache writes and cache reads.
+export const ctxOf = (r: TokenRow): number =>
+  r.input_tokens +
+  r.cache_read_tokens +
+  r.cache_write_5m_tokens +
+  r.cache_write_1h_tokens;
+
+export const overLimit = (ctx: number | null) =>
+  ctx !== null && ctx > CTX_LIMIT;

package/lib/usage/windows.ts

@@ -9,6 +9,7 @@ import os from "os";
 import { query, type SDKUserMessage } from "@anthropic-ai/claude-agent-sdk";
 import { InputQueue } from "../chat/queue";
 import { readUsage } from "../orchestrator/usage";
+import { currentWindow, type WindowReading } from "./rate-limits";
 
 export interface WindowView {
   pct: number;
@@ -17,6 +18,8 @@ export interface WindowView {
   // Seconds until it runs out at the current rate, when before it resets.
   capsIn?: number | null;
   burnPerMin?: number | null;
+  // Worked out from recorded usage, not read from the account.
+  estimated?: boolean;
 }
 
 export interface UsageWindows {
@@ -108,8 +111,30 @@ export function fromPlan(
   };
 }
 
+const fromReading = (
+  r: WindowReading | null,
+  nowMs: number
+): WindowView | null =>
+  r
+    ? {
+        pct: Math.round(r.pct),
+        resetsIn: r.resetsAt
+          ? Math.max(0, Math.round((r.resetsAt - nowMs) / 1000))
+          : null,
+        estimated: r.estimated || undefined,
+      }
+    : null;
+
 export async function usageWindows(): Promise<UsageWindows> {
   const sampled = readUsage();
+  const read = (w: "five_hour" | "seven_day") => {
+    try {
+      const now = Date.now();
+      return fromReading(currentWindow(w, now), now);
+    } catch {
+      return null;
+    }
+  };
   const plan = await planLimits();
   const fiveHour =
     "window" in sampled && sampled.window
@@ -119,8 +144,8 @@ export async function usageWindows(): Promise<UsageWindows> {
           capsIn: sampled.window.capsIn,
           burnPerMin: sampled.window.burnPerMin,
         }
-      : fromPlan(plan.limits?.five_hour);
-  const sevenDay = fromPlan(plan.limits?.seven_day);
+      : (fromPlan(plan.limits?.five_hour) ?? read("five_hour"));
+  const sevenDay = fromPlan(plan.limits?.seven_day) ?? read("seven_day");
   const note =
     !sevenDay || !fiveHour
       ? (plan.error ??

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saadnvd1/agent-os",
-  "version": "0.3.498",
+  "version": "0.3.501",
   "description": "Self-hosted web UI for managing Claude Code sessions",
   "bin": {
     "agent-os": "scripts/agent-os"
@@ -12,8 +12,7 @@
     "data/",
     "hooks/",
     "lib/",
-    "!lib/connect/test-pki.ts",
-    "!lib/connect/test-relay.ts",
+    "!lib/security/test-pki.ts",
     "!lib/**/*.test.ts",
     "mcp/",
     "public/",
@@ -121,7 +120,6 @@
     "@xterm/addon-webgl": "^0.19.0",
     "@xterm/headless": "6.0.0",
     "@xterm/xterm": "^6.0.0",
-    "acme-client": "^5.4.0",
     "better-sqlite3": "^13.0.3",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",

package/server.ts

@@ -13,6 +13,7 @@ import { stackStartGate } from "./lib/orchestrator/brakes";
 import { buildId } from "./lib/build";
 import { startOrchestratorWatcher } from "./lib/orchestrator/watcher";
 import { realDeps, schedulesEnabled, startScheduler } from "./lib/schedules";
+import { startUsageRollup, usageRollupEnabled } from "./lib/usage/nightly";
 import { resumePhoneOutbox } from "./lib/notify";
 import {
   bindAddresses,
@@ -39,6 +40,7 @@ import {
 } from "./lib/security/next-internals";
 import { lanEnabled } from "./lib/security/network-settings";
 import { startConnect } from "./lib/connect/serve";
+import { serverPort } from "./lib/connect/config";
 import { startTailnetHttps } from "./lib/security/tailnet-https";
 import {
   setStatusSource,
@@ -73,9 +75,7 @@ const dev = process.env.NODE_ENV !== "production";
 const hostname = "127.0.0.1";
 
 // Support: npm run dev -- -p 3012
-const pFlagIndex = process.argv.indexOf("-p");
-const portArg = pFlagIndex !== -1 ? process.argv[pFlagIndex + 1] : undefined;
-const port = parseInt(portArg || process.env.PORT || "3011", 10);
+const port = serverPort();
 claimAgentosEnv(port);
 // Demo mode: seeded data, nothing that runs code (lib/security/demo).
 const demo = demoMode();
@@ -313,7 +313,8 @@ app.prepare().then(async () => {
         : []),
     ];
   };
-  // AgentOS Connect: reachable at <id>.<machine domain> through the relay.
+  // AgentOS Connect: reachable at m-<16 hex>.runagentos.com through a
+  // Cloudflare tunnel, served on a listener of its own (lib/connect/serve).
   const connect = demo
     ? { hostname: () => null }
     : startConnect({ onRequest, onUpgrade }, port);
@@ -393,6 +394,8 @@ app.prepare().then(async () => {
   // Schedules tick here, once a minute, and nowhere else.
   if (schedulesEnabled(process.env)) startScheduler(realDeps);
   resumePhoneOutbox();
+  // The usage ledger's daily rollup and its ratchet alerts.
+  if (usageRollupEnabled(process.env)) startUsageRollup();
   // Machine load: a gauge, notes on heavy commands, one alert when red.
   if (loadEnabled()) startLoadMonitor();
 });

package/stores/usageUi.ts

@@ -1,14 +1,14 @@
 import { proxy } from "valtio";
 
-export type UsageTab = "spend" | "models" | "compare";
+export type UsageTab = "daily" | "spend" | "models" | "compare";
 
 export const usageUi = proxy<{ open: boolean; tab: UsageTab }>({
   open: false,
-  tab: "spend",
+  tab: "daily",
 });
 
 export const usageUiActions = {
-  open: (tab: UsageTab = "spend") => {
+  open: (tab: UsageTab = "daily") => {
     usageUi.tab = tab;
     usageUi.open = true;
   },

package/lib/connect/cert-check.ts

@@ -1,32 +0,0 @@
-import { X509Certificate } from "crypto";
-
-export interface CertStatus {
-  expiresAt: string;
-  daysLeft: number;
-  /** Set when it needs renewing, or can't be read. */
-  warning?: string;
-}
-
-const RENEW_AT_DAYS = 30;
-
-export function certStatus(pem: string, now = Date.now()): CertStatus {
-  try {
-    const expires = new Date(new X509Certificate(pem).validTo);
-    const daysLeft = Math.floor((expires.getTime() - now) / 86_400_000);
-    return {
-      expiresAt: expires.toISOString(),
-      daysLeft,
-      warning:
-        daysLeft <= RENEW_AT_DAYS
-          ? `The Connect certificate expires in ${daysLeft} days. Run agent-os connect --renew.`
-          : undefined,
-    };
-  } catch {
-    return {
-      expiresAt: "",
-      daysLeft: 0,
-      warning:
-        "The Connect certificate can't be read. Run agent-os connect --renew.",
-    };
-  }
-}

package/lib/connect/cloudflare-dns.ts

@@ -1,56 +0,0 @@
-/**
- * Dev and operator path only: write ACME TXT records straight to Cloudflare
- * with a zone-scoped token. Machines in the wild will go through the control
- * plane's broker instead, which only lets a machine touch its own name.
- */
-
-import type { DnsChallenge } from "./enrol";
-
-const API = "https://api.cloudflare.com/client/v4";
-
-export function cloudflareDns(token: string, zone: string): DnsChallenge {
-  const call = async (path: string, init: RequestInit = {}) => {
-    const res = await fetch(`${API}${path}`, {
-      ...init,
-      headers: {
-        Authorization: `Bearer ${token}`,
-        "Content-Type": "application/json",
-      },
-    });
-    const body = (await res.json()) as {
-      success: boolean;
-      result: unknown;
-      errors: unknown;
-    };
-    if (!body.success)
-      throw new Error(`cloudflare: ${JSON.stringify(body.errors)}`);
-    return body.result;
-  };
-  let zoneId: string | null = null;
-  const zid = async () =>
-    (zoneId ??=
-      ((await call(`/zones?name=${zone}`)) as { id: string }[])[0]?.id ?? null);
-
-  return {
-    async set(name, value) {
-      await call(`/zones/${await zid()}/dns_records`, {
-        method: "POST",
-        body: JSON.stringify({ type: "TXT", name, content: value, ttl: 60 }),
-      });
-      // Give the authoritative servers a moment before Let's Encrypt looks.
-      await new Promise((r) => setTimeout(r, 15_000));
-    },
-    async clear(name, value) {
-      const records = (await call(
-        `/zones/${await zid()}/dns_records?type=TXT&name=${encodeURIComponent(name)}`
-      )) as { id: string; content: string }[];
-      for (const r of records.filter(
-        (r) => r.content.replace(/"/g, "") === value
-      )) {
-        await call(`/zones/${await zid()}/dns_records/${r.id}`, {
-          method: "DELETE",
-        });
-      }
-    },
-  };
-}

package/lib/connect/frames.ts

@@ -1,72 +0,0 @@
-/**
- * The Connect tunnel's wire format: one WebSocket from a machine to the
- * relay, carrying many TCP streams. Each binary message is one frame:
- *
- *   type (u8) | stream id (u32 BE) | payload
- *
- * The relay only ever moves the TLS bytes of user connections; it never
- * holds a key that could read them.
- */
-
-export const Frame = {
-  /** machine → relay: {machineId, ts, nonce, sig} */
-  HELLO: 0,
-  /** relay → machine: a phone connected; payload {remote} */
-  OPEN: 1,
-  DATA: 2,
-  /** either side: this stream is done */
-  CLOSE: 3,
-  /** relay → machine: {hostname} once HELLO checks out */
-  READY: 4,
-  /** either side: stop / start sending DATA for this stream */
-  PAUSE: 5,
-  RESUME: 6,
-  /** relay → machine: refused, payload {error}; the socket closes after */
-  DENIED: 7,
-} as const;
-
-export type FrameType = (typeof Frame)[keyof typeof Frame];
-
-export interface Decoded {
-  type: FrameType;
-  stream: number;
-  payload: Buffer;
-}
-
-export function encode(
-  type: FrameType,
-  stream = 0,
-  payload?: Buffer | string | object
-): Buffer {
-  const body =
-    payload === undefined
-      ? Buffer.alloc(0)
-      : Buffer.isBuffer(payload)
-        ? payload
-        : Buffer.from(
-            typeof payload === "string" ? payload : JSON.stringify(payload)
-          );
-  const head = Buffer.alloc(5);
-  head.writeUInt8(type, 0);
-  head.writeUInt32BE(stream, 1);
-  return Buffer.concat([head, body]);
-}
-
-export function decode(data: Buffer): Decoded | null {
-  if (data.length < 5) return null;
-  const type = data.readUInt8(0);
-  if (type > Frame.DENIED) return null;
-  return {
-    type: type as FrameType,
-    stream: data.readUInt32BE(1),
-    payload: data.subarray(5),
-  };
-}
-
-export const json = <T>(payload: Buffer): T | null => {
-  try {
-    return JSON.parse(payload.toString("utf8")) as T;
-  } catch {
-    return null;
-  }
-};

package/lib/connect/identity.ts

@@ -1,98 +0,0 @@
-/**
- * A machine proves who it is to the relay with an Ed25519 key that never
- * leaves it. HELLO signs (machine id, time, nonce); the relay checks the
- * signature against the public key registered for that id, the time
- * against its clock, and refuses a nonce it has already seen.
- */
-
-import crypto from "crypto";
-
-const SKEW_MS = 60_000;
-const CONTEXT = "agentos-connect-hello";
-
-export interface Hello {
-  machineId: string;
-  ts: number;
-  nonce: string;
-  sig: string;
-}
-
-export function generateMachineKey(): {
-  publicKey: string;
-  privateKey: string;
-} {
-  const { publicKey, privateKey } = crypto.generateKeyPairSync("ed25519");
-  return {
-    publicKey: publicKey.export({ type: "spki", format: "pem" }).toString(),
-    privateKey: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
-  };
-}
-
-const message = (h: Omit<Hello, "sig">) =>
-  Buffer.from(`${CONTEXT}:${h.machineId}:${h.ts}:${h.nonce}`);
-
-export function signHello(
-  machineId: string,
-  privateKeyPem: string,
-  now = Date.now()
-): Hello {
-  const h = {
-    machineId,
-    ts: now,
-    nonce: crypto.randomBytes(16).toString("base64url"),
-  };
-  const sig = crypto
-    .sign(null, message(h), privateKeyPem)
-    .toString("base64url");
-  return { ...h, sig };
-}
-
-export class NonceCache {
-  private seen = new Map<string, number>();
-  /** False if this nonce was already used inside the window. */
-  remember(nonce: string, now = Date.now()): boolean {
-    for (const [n, at] of this.seen)
-      if (at < now - 2 * SKEW_MS) this.seen.delete(n);
-    if (this.seen.has(nonce)) return false;
-    this.seen.set(nonce, now);
-    return true;
-  }
-}
-
-export type HelloCheck =
-  | { ok: true; machineId: string }
-  | { ok: false; error: string };
-
-export function verifyHello(
-  hello: Partial<Hello> | null,
-  publicKeyPem: string | null,
-  nonces: NonceCache,
-  now = Date.now()
-): HelloCheck {
-  if (
-    !hello?.machineId ||
-    !hello.sig ||
-    !hello.nonce ||
-    typeof hello.ts !== "number"
-  ) {
-    return { ok: false, error: "malformed hello" };
-  }
-  if (!publicKeyPem) return { ok: false, error: "unknown machine" };
-  if (Math.abs(now - hello.ts) > SKEW_MS)
-    return { ok: false, error: "clock skew" };
-  let valid = false;
-  try {
-    valid = crypto.verify(
-      null,
-      message(hello as Hello),
-      publicKeyPem,
-      Buffer.from(hello.sig, "base64url")
-    );
-  } catch {
-    valid = false;
-  }
-  if (!valid) return { ok: false, error: "bad signature" };
-  if (!nonces.remember(hello.nonce, now))
-    return { ok: false, error: "replayed" };
-  return { ok: true, machineId: hello.machineId };
-}