@saadnvd1/agent-os 0.3.499 → 0.3.502

package/lib/connect/supervise.ts

@@ -0,0 +1,281 @@
+/**
+ * Keeps `cloudflared tunnel run` going while Connect is on. Each start asks
+ * the Connect service for a fresh connector token, which goes to the child
+ * in TUNNEL_TOKEN only: never argv, never a file, never a log line. The
+ * child exits → it's started again with backoff; stop() ends it.
+ */
+
+import { execFile, spawn as nodeSpawn } from "child_process";
+import fs from "fs";
+import path from "path";
+import readline from "readline";
+import type { Readable } from "stream";
+import { ApiError, clean } from "./control-api";
+
+export type TunnelState = "connecting" | "up" | "denied" | "error" | "stopped";
+
+export interface Prepared {
+  bin: string;
+  token: string;
+}
+
+interface Child {
+  pid?: number;
+  stdout: Readable | null;
+  stderr: Readable | null;
+  kill(signal?: NodeJS.Signals): boolean;
+  on(event: "exit", fn: (code: number | null) => void): unknown;
+  on(event: "error", fn: (err: Error) => void): unknown;
+}
+
+export type Spawn = (
+  bin: string,
+  args: string[],
+  opts: { env: Record<string, string>; stdio: ["ignore", "pipe", "pipe"] }
+) => Child;
+
+export interface SuperviseOptions {
+  /** Finds the binary and fetches a fresh token; throws to retry later. */
+  prepare: () => Promise<Prepared>;
+  /** The Connect folder: cloudflared's config, home and pid file live here. */
+  dir: string;
+  log?: (line: string) => void;
+  spawn?: Spawn;
+  delays?: { min: number; max: number; setupMax: number; denied: number };
+}
+
+const DELAYS = {
+  min: 1000,
+  max: 60_000,
+  setupMax: 5 * 60_000,
+  denied: 5 * 60_000,
+};
+// cloudflared can be chatty while the network flaps.
+const LOG_LINES_PER_MINUTE = 20;
+
+/** A cloudflared left running by an AgentOS that died without stopping it. */
+const commandOf = (pid: number) =>
+  new Promise<string>((resolve) =>
+    execFile(
+      "ps",
+      ["-o", "command=", "-p", String(pid)],
+      { encoding: "utf8", timeout: 5000 },
+      (err, out) => resolve(err ? "" : out)
+    )
+  );
+
+async function stopStale(pidFile: string, binDir: string) {
+  let pid = 0;
+  try {
+    pid = Number(fs.readFileSync(pidFile, "utf8")) || 0;
+  } catch {
+    return;
+  }
+  if (pid > 1) {
+    const command = await commandOf(pid);
+    try {
+      if (command.startsWith(binDir + path.sep)) process.kill(pid, "SIGTERM");
+    } catch {}
+  }
+  fs.rmSync(pidFile, { force: true });
+}
+
+export function superviseTunnel(opts: SuperviseOptions) {
+  const log = opts.log ?? (() => {});
+  const spawn = opts.spawn ?? (nodeSpawn as unknown as Spawn);
+  const delays = opts.delays ?? DELAYS;
+  const pidFile = path.join(opts.dir, "cloudflared.pid");
+  const home = path.join(opts.dir, "cloudflared-home");
+  const configFile = path.join(opts.dir, "cloudflared.yml");
+
+  let state: TunnelState = "connecting";
+  let reason: string | undefined;
+  let stopped = false;
+  let child: Child | null = null;
+  let timer: NodeJS.Timeout | null = null;
+  let delay = delays.min;
+  const connections = new Set<string>();
+  let logged = { at: 0, n: 0 };
+
+  const say = (line: string) => {
+    const now = Date.now();
+    if (now - logged.at > 60_000) logged = { at: now, n: 0 };
+    if (++logged.n <= LOG_LINES_PER_MINUTE) log(line);
+  };
+
+  const retry = (wait: number) => {
+    if (stopped) return;
+    timer = setTimeout(() => {
+      timer = null;
+      tick();
+    }, wait);
+    timer.unref?.();
+  };
+
+  // Nothing here may reject: an unhandled rejection takes AgentOS down.
+  const tick = () =>
+    void attempt().catch(() => {
+      state = "error";
+      retry(delays.max);
+    });
+
+  const attempt = async () => {
+    if (stopped) return;
+    state = "connecting";
+    try {
+      const prepared = await opts.prepare();
+      if (stopped) return;
+      await run(prepared);
+    } catch (err) {
+      if (stopped) return;
+      const status = err instanceof ApiError ? err.status : 0;
+      reason = clean((err as Error)?.message ?? String(err));
+      log(`connect: ${reason}`);
+      if (status === 401 || status === 403) {
+        // Revoked or disabled: it won't fix itself in seconds.
+        state = "denied";
+        retry(delays.denied);
+      } else {
+        state = "error";
+        retry(delay);
+        delay = Math.min(delay * 2, delays.setupMax);
+      }
+    }
+  };
+
+  const run = async ({ bin, token }: Prepared) => {
+    const redact = (line: string) => clean(line.split(token).join("[token]"));
+    await stopStale(pidFile, path.dirname(bin));
+    if (stopped) return;
+    fs.mkdirSync(home, { recursive: true, mode: 0o700 });
+    // Its own config and home, so no config.yml elsewhere on the machine
+    // changes what it does.
+    fs.writeFileSync(configFile, "no-autoupdate: true\n", { mode: 0o600 });
+    let proc: Child;
+    try {
+      proc = spawn(
+        bin,
+        [
+          "tunnel",
+          "--config",
+          configFile,
+          "--no-autoupdate",
+          // Never debug: it logs every request's headers, device tokens included.
+          "--loglevel",
+          "info",
+          "--output",
+          "json",
+          "run",
+        ],
+        {
+          env: {
+            PATH: process.env.PATH ?? "/usr/bin:/bin",
+            HOME: home,
+            TUNNEL_TOKEN: token,
+          },
+          stdio: ["ignore", "pipe", "pipe"],
+        }
+      );
+    } catch (err) {
+      state = "error";
+      reason = `cloudflared could not start: ${redact((err as Error).message)}`;
+      log(`connect: ${reason}`);
+      retry(delay);
+      delay = Math.min(delay * 2, delays.max);
+      return;
+    }
+    child = proc;
+    if (proc.pid) {
+      try {
+        fs.writeFileSync(pidFile, String(proc.pid), { mode: 0o600 });
+      } catch {}
+    }
+
+    const onLine = (line: string) => {
+      if (!line.trim()) return;
+      let entry: {
+        level?: string;
+        message?: string;
+        error?: string;
+        connIndex?: number;
+      } | null = null;
+      if (line.startsWith("{")) {
+        try {
+          entry = JSON.parse(line);
+        } catch {}
+      }
+      if (!entry) return say(`connect: cloudflared: ${redact(line)}`);
+      const message = String(entry.message ?? "");
+      const conn = String(entry.connIndex ?? "0");
+      if (message.startsWith("Registered tunnel connection")) {
+        connections.add(conn);
+        if (state !== "up") log("connect: tunnel up");
+        state = "up";
+        reason = undefined;
+        delay = delays.min;
+      } else if (
+        message.startsWith("Unregistered tunnel connection") ||
+        ((entry.level === "error" || entry.level === "fatal") &&
+          entry.connIndex !== undefined)
+      ) {
+        connections.delete(conn);
+        if (state === "up" && connections.size === 0) state = "connecting";
+      }
+      if (["warn", "error", "fatal"].includes(entry.level ?? "")) {
+        const text = entry.error ? `${message}: ${entry.error}` : message;
+        reason = redact(text);
+        say(`connect: cloudflared: ${reason}`);
+      }
+    };
+    for (const stream of [proc.stdout, proc.stderr]) {
+      if (!stream) continue;
+      stream.on("error", () => {});
+      readline.createInterface({ input: stream }).on("line", onLine);
+    }
+
+    let ended = false;
+    const end = (why: string) => {
+      if (ended) return;
+      ended = true;
+      if (child === proc) child = null;
+      connections.clear();
+      try {
+        if (fs.readFileSync(pidFile, "utf8") === String(proc.pid))
+          fs.rmSync(pidFile, { force: true });
+      } catch {}
+      if (stopped) return;
+      state = "connecting";
+      reason = why;
+      log(`connect: ${why}; starting it again`);
+      retry(delay);
+      delay = Math.min(delay * 2, delays.max);
+    };
+    proc.on("error", (err) =>
+      end(`cloudflared failed: ${redact(err.message)}`)
+    );
+    proc.on("exit", (code) => end(`cloudflared exited (${code ?? "signal"})`));
+  };
+
+  const killNow = () => child?.kill("SIGTERM");
+  process.once("exit", killNow);
+  tick();
+
+  return {
+    state: () => state,
+    reason: () => reason,
+    stop() {
+      if (stopped) return;
+      stopped = true;
+      state = "stopped";
+      process.removeListener("exit", killNow);
+      if (timer) clearTimeout(timer);
+      const proc = child;
+      if (!proc) return;
+      proc.kill("SIGTERM");
+      // It drains connections for a while on SIGTERM; don't wait forever.
+      const hard = setTimeout(() => proc.kill("SIGKILL"), 5000);
+      hard.unref?.();
+      proc.on("exit", () => clearTimeout(hard));
+    },
+  };
+}

package/lib/db/migrations.ts

@@ -1521,6 +1521,78 @@ const migrations: Migration[] = [
         );
     },
   },
+  {
+    id: 68,
+    name: "usage_calls_and_rollups",
+    up: (db) => {
+      // One row per Claude response (or per one-shot run), the daily rollups
+      // computed from them, the ratchet's baseline and the limit readings.
+      db.transaction(() => {
+        db.exec(`
+          CREATE TABLE IF NOT EXISTS usage_calls (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            call_key TEXT NOT NULL UNIQUE,
+            at INTEGER NOT NULL,
+            session_id TEXT,
+            task_id TEXT,
+            workspace_id TEXT,
+            project_id TEXT,
+            session_name TEXT,
+            role TEXT,
+            kind TEXT NOT NULL,
+            agent_type TEXT,
+            model TEXT,
+            provider TEXT,
+            input_tokens INTEGER NOT NULL DEFAULT 0,
+            cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+            cache_write_5m_tokens INTEGER NOT NULL DEFAULT 0,
+            cache_write_1h_tokens INTEGER NOT NULL DEFAULT 0,
+            output_tokens INTEGER NOT NULL DEFAULT 0,
+            ctx_tokens INTEGER,
+            source TEXT NOT NULL
+          );
+          CREATE INDEX IF NOT EXISTS idx_usage_calls_at ON usage_calls(at);
+          CREATE INDEX IF NOT EXISTS idx_usage_calls_task ON usage_calls(task_id);
+          CREATE INDEX IF NOT EXISTS idx_usage_calls_session ON usage_calls(session_id);
+          CREATE TABLE IF NOT EXISTS rate_limit_readings (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            at INTEGER NOT NULL,
+            window TEXT NOT NULL,
+            utilization REAL NOT NULL,
+            raw REAL,
+            resets_at INTEGER,
+            status TEXT,
+            session_id TEXT,
+            source TEXT NOT NULL
+          );
+          CREATE INDEX IF NOT EXISTS idx_rate_limit_readings ON rate_limit_readings(window, at);
+          CREATE TABLE IF NOT EXISTS usage_rollups (
+            day TEXT PRIMARY KEY,
+            data TEXT NOT NULL,
+            final INTEGER NOT NULL DEFAULT 0,
+            computed_at INTEGER NOT NULL
+          );
+          CREATE TABLE IF NOT EXISTS usage_baseline (
+            metric TEXT PRIMARY KEY,
+            value REAL NOT NULL,
+            set_at INTEGER NOT NULL,
+            supporting TEXT
+          );
+          CREATE TABLE IF NOT EXISTS usage_alerts (
+            kind TEXT PRIMARY KEY,
+            opened_at INTEGER NOT NULL,
+            cleared_at INTEGER,
+            message TEXT NOT NULL
+          );
+          CREATE TABLE IF NOT EXISTS usage_import_offsets (
+            path TEXT PRIMARY KEY,
+            offset INTEGER NOT NULL,
+            mtime INTEGER NOT NULL
+          );
+        `);
+      })();
+    },
+  },
 ];
 
 // `upTo`: stop after this id (tests that start from an older database).

package/lib/orchestrator/describe.ts

@@ -25,7 +25,7 @@ export function ciWord(pr: TaskPR): string {
 export const shortId = (id: string) => id.slice(0, 8);
 
 function sessionLine(f: SessionFacts): string {
-  const head = `- ${f.name} (${f.project ?? "no project"}, ${f.quick ? "your quick change" : f.view}, id ${shortId(f.id)}): ${STATUS_WORD[f.status]}`;
+  const head = `- ${f.name} (${f.project ?? "no project"}, ${f.quick ? "your quick change" : f.view}, id ${shortId(f.id)}): ${f.compacting ? "compacting context" : STATUS_WORD[f.status]}`;
   const parts = [
     f.activity ? `${head}, ${untrusted(f.name, f.activity)}` : head,
   ];

package/lib/orchestrator/facts.ts

@@ -9,7 +9,7 @@ import { db, type Session, type StackItemStatus } from "../db";
 import { statusDetector } from "../status-detector";
 import { hostLink } from "../hosts/remote-api";
 import { peerStatus } from "../hosts/peer-sessions";
-import { chatState } from "../chat/runner";
+import { chatCompacting, chatState } from "../chat/runner";
 import { chatActivityLine, chatStall, type Stall } from "../chat/activity";
 import { lastUserTask } from "../chat/store";
 import { needsYou } from "../needs-you";
@@ -35,6 +35,8 @@ export interface SessionFacts {
   needsInput: boolean;
   // A chat turn silent past the stall threshold (lib/chat/stall).
   stalled?: Stall | null;
+  // Its agent is summarising the context: not stuck, not plain work.
+  compacting?: boolean;
   // Last sign of life, in ms.
   lastActive: number;
   // Work that should end in a PR: a task, or a session on its own branch.
@@ -274,6 +276,8 @@ export async function sessionFacts(
         needsInput,
         stalled:
           s.view === "chat" && !hostLink(s.host_id) ? chatStall(s.id) : null,
+        compacting:
+          s.view === "chat" && !hostLink(s.host_id) && !!chatCompacting(s.id),
         lastActive: Math.max(
           sqliteMs(s.updated_at),
           statusDetector.getTimestamp(s.tmux_name) * 1000,

package/lib/security/auth.ts

@@ -2,6 +2,11 @@
  * Who may use AgentOS once a request has passed the host/origin checks in
  * net.ts. Loopback and the tailnet are trusted as before; anything else
  * (Wi-Fi, a reverse proxy, Connect) needs a paired device's token.
+ *
+ * Connect's tunnel arrives from cloudflared on loopback, so its listener
+ * marks every socket it accepts: a marked socket is never loopback, never
+ * the tailnet and never let in by AGENTOS_AUTH=off, whatever its address,
+ * Host or headers say.
  */
 
 import type { IncomingHttpHeaders } from "http";
@@ -23,8 +28,18 @@ export interface AuthRequest {
   headers: IncomingHttpHeaders;
   remoteAddress?: string;
   localAddress?: string;
+  /** Came through Connect's tunnel listener: a paired device or nothing. */
+  tunnel?: boolean;
 }
 
+const TUNNEL_SOCKETS = new WeakSet<object>();
+/** Called by the tunnel listener for every socket, before any request. */
+export const markTunnelSocket = (socket: object) => {
+  TUNNEL_SOCKETS.add(socket);
+};
+export const isTunnelSocket = (socket: object | null | undefined) =>
+  !!socket && TUNNEL_SOCKETS.has(socket);
+
 export interface AuthPolicy {
   /** This machine's own Tailscale addresses. */
   tailnet: string[];
@@ -146,13 +161,14 @@ export function authorize(req: AuthRequest, policy: AuthPolicy): AuthResult {
 }
 
 function decide(req: AuthRequest, policy: AuthPolicy): AuthResult {
-  const trusted = networkTrust(req, policy);
+  const trusted = req.tunnel ? null : networkTrust(req, policy);
   if (trusted) return { ok: true, via: trusted };
   const token = readToken(req.headers);
   const device = token ? policy.lookup(token) : null;
   if (device) return { ok: true, via: "device", deviceId: device.id };
-  // AGENTOS_AUTH=off is for a login proxy on this side of the network. A
-  // Connect stream has no socket address and must never be let in by it.
-  if (policy.off && req.remoteAddress) return { ok: true, via: "open" };
+  // AGENTOS_AUTH=off is for a login proxy on this side of the network.
+  // Connect's tunnel must never be let in by it.
+  if (policy.off && !req.tunnel && req.remoteAddress)
+    return { ok: true, via: "open" };
   return { ok: false };
 }

package/lib/security/gate.ts

@@ -9,6 +9,7 @@ import {
   authorize,
   cookieToken,
   isPublicPath,
+  isTunnelSocket,
   plainAddress,
   DEVICE_HEADER,
   TRUST_HEADER,
@@ -44,22 +45,24 @@ export function authPolicy(env = process.env): AuthPolicy {
   };
 }
 
-// Connect streams have no address. Give them a fixed label so rate limits
-// can't be steered by headers the caller chose.
-export const sourceLabel = (remote?: string) =>
-  plainAddress(remote) || "connect";
+// Everything through Connect arrives from cloudflared on loopback. Give it
+// a fixed label so rate limits can't be steered by headers the caller chose.
+export const sourceLabel = (remote?: string, tunnel = false) =>
+  tunnel ? "connect" : plainAddress(remote) || "unknown";
 
 function decide(req: IncomingMessage, policy: AuthPolicy): AuthResult {
   // Never trust a client-supplied verdict.
   delete req.headers[DEVICE_HEADER];
   delete req.headers[TRUST_HEADER];
-  req.headers[REMOTE_HEADER] = sourceLabel(req.socket.remoteAddress);
+  const tunnel = isTunnelSocket(req.socket);
+  req.headers[REMOTE_HEADER] = sourceLabel(req.socket.remoteAddress, tunnel);
   const result = authorize(
     {
       url: req.url,
       headers: req.headers,
       remoteAddress: req.socket.remoteAddress,
       localAddress: req.socket.localAddress,
+      tunnel,
     },
     policy
   );

package/lib/security/reach.ts

@@ -44,7 +44,7 @@ export async function reachableAt(): Promise<Reach[]> {
     await tailscaleStatus(),
     lanEnabled() ? lanAddresses() : [],
     port,
-    loadConnect()?.config.hostname ?? null,
+    loadConnect()?.config.hostname || null,
     tailnetHttpsUrl()
   );
 }

package/lib/sidebar/shelves.ts

@@ -30,6 +30,8 @@ export interface RowStatus {
   need?: SessionNeed | null;
   unread?: boolean;
   stalled?: boolean;
+  // A chat summarising its context (status stays "running").
+  compacting?: { since: number; before?: number };
   asks?: number;
   // A program's own message (OSC 7501): untrusted, plain text only.
   detail?: string | null;
@@ -41,6 +43,8 @@ export interface SidebarRow {
   need: SessionNeed | null;
   unread: boolean;
   working: boolean;
+  // Working, but summarising its context, not on the task.
+  compacting?: boolean;
   // Orchestration workers, nested under their conductor.
   workers: SidebarRow[];
   // Its task's state, when it's a task.
@@ -120,6 +124,7 @@ function toRow(input: ShelfInput, session: Session): SidebarRow {
     need: needOf(session, status, input.tasks[session.id]),
     unread: !!status?.unread,
     working: status?.status === "running",
+    compacting: !!status?.compacting,
     workers: [],
     task: input.tasks[session.id],
     quick: [],
@@ -256,6 +261,8 @@ function sameStatus(a: RowStatus | undefined, b: RowStatus | undefined) {
     (a.need ?? null) === (b.need ?? null) &&
     !!a.unread === !!b.unread &&
     !!a.stalled === !!b.stalled &&
+    (a.compacting?.since ?? null) === (b.compacting?.since ?? null) &&
+    (a.compacting?.before ?? null) === (b.compacting?.before ?? null) &&
     a.asks === b.asks &&
     (a.detail ?? null) === (b.detail ?? null)
   );

package/lib/status/collect.ts

@@ -20,7 +20,7 @@ import {
   getSessionIdFromName,
 } from "../providers/registry";
 import { getDb, type Session } from "../db";
-import { chatState } from "../chat/runner";
+import { chatCompacting, chatState } from "../chat/runner";
 import { chatActivityLine, chatStall } from "../chat/activity";
 import {
   chatNeed,
@@ -30,6 +30,7 @@ import {
   terminalStatus,
 } from "../needs-you";
 import type { SessionNeed } from "../sidebar/shelves";
+import type { Compacting } from "../chat/events";
 import { openAskCount } from "../orchestrator/asks";
 import { lastUserTask } from "../chat/store";
 import { hostExecFile, isRemoteHost } from "../hosts";
@@ -62,6 +63,8 @@ export interface SessionStatusResponse {
   unread?: boolean;
   // A chat turn silent past the stall threshold (lib/chat/stall).
   stalled?: boolean;
+  // A chat whose agent is summarising its context ("status" stays running).
+  compacting?: Compacting;
   // A program's own message (OSC 7501): untrusted, plain text only.
   detail?: string | null;
   progress?: number | null;
@@ -467,6 +470,9 @@ async function collect(): Promise<StatusSnapshot> {
       need: chatNeed(session, state, asks),
       unread: isUnread(session, state),
       ...(chatStall(session.id) && { stalled: true }),
+      ...(chatCompacting(session.id) && {
+        compacting: chatCompacting(session.id)!,
+      }),
       ...(session.role === "orchestrator" && { asks }),
     };
   }

package/lib/tasks/index.ts

@@ -4,6 +4,7 @@
  * drops it.
  */
 
+import { taskCosts } from "../usage/calls";
 import {
   parseOverride,
   resolveRole,
@@ -100,6 +101,8 @@ export interface TaskView {
   plan?: { planned: boolean; reason: string | null } | null;
   // A quick change: the orchestrator session that makes it (no agent).
   quickOf?: string | null;
+  // What it has cost so far in weighted tokens (lib/usage/weights).
+  weighted?: number;
 }
 
 export async function createTask(opts: {
@@ -415,6 +418,11 @@ export async function listTasks(): Promise<TaskView[]> {
     Promise.all(sessions.filter((s) => !isMirror(s)).map(taskView)),
     remoteTaskViews(sessions.filter(isMirror)),
   ]);
+  const costs = taskCosts(local.map((t) => t.id));
+  for (const t of local) {
+    const w = costs.get(t.id);
+    if (w) t.weighted = w;
+  }
   return [...local, ...remote].sort((a, b) =>
     b.createdAt.localeCompare(a.createdAt)
   );