@saadnvd1/agent-os 0.3.697 → 0.3.700

package/lib/bus/waits/store.ts

@@ -0,0 +1,196 @@
+/**
+ * Waits: session A waiting on session B, kept in `bus_waits`. A change of
+ * state is claimed with a conditional UPDATE and acted on only by the caller
+ * whose update changed a row, so overlapping sweeps and restarts never
+ * wake anyone twice.
+ */
+
+import { db, type Session } from "../../db";
+import { isHeldRemote } from "../../tasks/contract/held-calls";
+import { notifySessionsChanged } from "../../status/hub";
+
+export type WaitKind = "reply" | "idle" | "merged";
+export type WaitOutcome =
+  "replied" | "idle" | "merged" | "timeout" | "ended" | "cancelled";
+
+export interface WaitRow {
+  id: number;
+  waiter_id: string;
+  target_id: string;
+  kind: WaitKind;
+  message_id: number | null;
+  request: string | null;
+  deadline: number;
+  created_at: number;
+  seen_busy_at: number | null;
+  nudged_at: number | null;
+  closed_at: number | null;
+  outcome: WaitOutcome | null;
+  reply_id: number | null;
+}
+
+export const MAX_OPEN_WAITS = 10;
+export const DEFAULT_TIMEOUT_MS = 30 * 60_000;
+export const MIN_TIMEOUT_MS = 60_000;
+export const MAX_TIMEOUT_MS = 24 * 60 * 60_000;
+
+// A request shown in a line: one line, cut.
+export function excerpt(body: string, max = 100): string {
+  const one = body.replace(/\s+/g, " ").trim().replace(/"/g, "'");
+  return one.length <= max ? one : `${one.slice(0, max - 1)}…`;
+}
+
+export function openWait(w: {
+  waiterId: string;
+  targetId: string;
+  kind: WaitKind;
+  messageId?: number | null;
+  request?: string | null;
+  deadline: number;
+  now?: number;
+}): number {
+  const now = w.now ?? Date.now();
+  const open = db
+    .prepare(
+      `SELECT id FROM bus_waits WHERE waiter_id = ? AND target_id = ? AND kind = ? AND closed_at IS NULL`
+    )
+    .get(w.waiterId, w.targetId, w.kind) as { id: number } | undefined;
+  let id: number;
+  if (open) {
+    // The same wait again: the newer request and deadline stand.
+    db.prepare(
+      `UPDATE bus_waits SET message_id = ?, request = ?, deadline = ?, nudged_at = NULL WHERE id = ?`
+    ).run(w.messageId ?? null, w.request ?? null, w.deadline, open.id);
+    id = open.id;
+  } else {
+    const count = (
+      db
+        .prepare(
+          `SELECT COUNT(*) AS n FROM bus_waits WHERE waiter_id = ? AND closed_at IS NULL`
+        )
+        .get(w.waiterId) as { n: number }
+    ).n;
+    if (count >= MAX_OPEN_WAITS)
+      throw new Error(
+        `You already have ${MAX_OPEN_WAITS} waits open; wait for one to finish`
+      );
+    id = Number(
+      db
+        .prepare(
+          `INSERT INTO bus_waits (waiter_id, target_id, kind, message_id, request, deadline, created_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?)`
+        )
+        .run(
+          w.waiterId,
+          w.targetId,
+          w.kind,
+          w.messageId ?? null,
+          w.request ?? null,
+          w.deadline,
+          now
+        ).lastInsertRowid
+    );
+  }
+  notifySessionsChanged();
+  return id;
+}
+
+// True for the one caller that closed it.
+export function claimClose(
+  id: number,
+  outcome: WaitOutcome,
+  replyId: number | null = null,
+  now = Date.now()
+): boolean {
+  const changed =
+    db
+      .prepare(
+        `UPDATE bus_waits SET closed_at = ?, outcome = ?, reply_id = ? WHERE id = ? AND closed_at IS NULL`
+      )
+      .run(now, outcome, replyId, id).changes === 1;
+  if (changed) notifySessionsChanged();
+  return changed;
+}
+
+export const claimNudge = (id: number, now = Date.now()): boolean =>
+  db
+    .prepare(
+      `UPDATE bus_waits SET nudged_at = ? WHERE id = ? AND closed_at IS NULL AND nudged_at IS NULL`
+    )
+    .run(now, id).changes === 1;
+
+export function markSeenBusy(id: number, now = Date.now()): void {
+  db.prepare(
+    `UPDATE bus_waits SET seen_busy_at = ? WHERE id = ? AND seen_busy_at IS NULL`
+  ).run(now, id);
+}
+
+export const openWaits = (): WaitRow[] =>
+  db
+    .prepare(`SELECT * FROM bus_waits WHERE closed_at IS NULL ORDER BY id`)
+    .all() as WaitRow[];
+
+// A reply from `targetId` to `waiterId` settles these.
+export const replyWaitsFor = (waiterId: string, targetId: string): WaitRow[] =>
+  db
+    .prepare(
+      `SELECT * FROM bus_waits WHERE waiter_id = ? AND target_id = ? AND kind = 'reply' AND closed_at IS NULL ORDER BY id`
+    )
+    .all(waiterId, targetId) as WaitRow[];
+
+export interface WaitBadges {
+  on: { id: string; name: string; kind: WaitKind; deadline: number }[];
+  by: { id: string; name: string }[];
+}
+
+// Who each session waits on, and who waits on it, for the open waits.
+export function waitBadges(): Map<string, WaitBadges> {
+  const rows = db
+    .prepare(
+      `SELECT w.waiter_id, w.target_id, w.kind, w.deadline,
+              COALESCE(a.name, 'a session') AS waiter_name,
+              COALESCE(b.name, 'a session') AS target_name
+       FROM bus_waits w
+       LEFT JOIN sessions a ON a.id = w.waiter_id
+       LEFT JOIN sessions b ON b.id = w.target_id
+       WHERE w.closed_at IS NULL ORDER BY w.id`
+    )
+    .all() as {
+    waiter_id: string;
+    target_id: string;
+    kind: WaitKind;
+    deadline: number;
+    waiter_name: string;
+    target_name: string;
+  }[];
+  const out = new Map<string, WaitBadges>();
+  const get = (id: string) => {
+    let b = out.get(id);
+    if (!b) out.set(id, (b = { on: [], by: [] }));
+    return b;
+  };
+  for (const r of rows) {
+    get(r.waiter_id).on.push({
+      id: r.target_id,
+      name: r.target_name,
+      kind: r.kind,
+      deadline: r.deadline,
+    });
+    get(r.target_id).by.push({ id: r.waiter_id, name: r.waiter_name });
+  }
+  return out;
+}
+
+// "B", "B and C", "B, C +2": for a sidebar chip.
+export function namesLine(names: string[]): string {
+  const unique = [...new Set(names)];
+  if (unique.length <= 2) return unique.join(" and ");
+  return `${unique.slice(0, 2).join(", ")} +${unique.length - 2}`;
+}
+
+// Why a wait on this session can't be kept, or null.
+export function cantWaitOn(to: Session): string | null {
+  if (isHeldRemote(to) || (to.peer_mirror && !to.task_prompt))
+    return "it runs on a linked machine, which can't tell this one when it answers";
+  return null;
+}

package/lib/bus/waits/tick.ts

@@ -0,0 +1,192 @@
+/**
+ * Settles open waits: nudges a target that went idle without answering,
+ * wakes a waiter whose condition holds, whose deadline passed or whose
+ * target ended. Replies settle in sendMessage; this covers the rest. It
+ * reads status through statusOf, the same idle test the orchestrator uses.
+ */
+
+import type { Session } from "../../db";
+import { db } from "../../db";
+import { statusDetector } from "../../status-detector";
+import { statusOf } from "../../orchestrator/facts";
+import { queueEventOnce } from "../../orchestrator/events";
+import { sessionWorkspace } from "../../schedules/store";
+import { sendMessage } from "../index";
+import { onKick } from "./kick";
+import {
+  claimClose,
+  claimNudge,
+  markSeenBusy,
+  openWaits,
+  type WaitRow,
+} from "./store";
+import {
+  closedPrLine,
+  endedLine,
+  nudgeLine,
+  orchestratorTimeoutLine,
+  satisfiedLine,
+  timeoutLine,
+} from "./lines";
+
+// A target that was never seen working is idle only this long after the
+// wait opened: a chat turn can start and end between two looks, and a
+// message may not have been picked up yet.
+export const IDLE_GRACE_MS = 20_000;
+// A terminal reads dead for a moment when tmux hiccups: twice in a row is gone.
+const DEAD_LOOKS = 2;
+const EVERY_MS = 5000;
+
+const dead = new Map<string, number>();
+
+const row = (id: string) =>
+  db.prepare(`SELECT * FROM session_rows WHERE id = ?`).get(id) as
+    Session | undefined;
+
+// Why this session can no longer act on a wait, or null.
+function endedHow(s: Session | undefined): string | null {
+  if (!s) return "deleted";
+  if (s.archived_at) return "archived";
+  if (s.task_status === "moved" || s.task_status === "moving") return "moved";
+  if (s.task_status && s.task_status !== "running") return s.task_status;
+  return null;
+}
+
+async function tell(to: Session, line: string): Promise<void> {
+  try {
+    await sendMessage({
+      fromId: null,
+      fromLabel: "AgentOS",
+      to: to.id,
+      body: line,
+      line,
+    });
+  } catch (error) {
+    console.error(`[waits] couldn't tell ${to.name}:`, error);
+  }
+}
+
+async function settle(w: WaitRow, waiter: Session, now: number): Promise<void> {
+  const target = row(w.target_id);
+  const end = endedHow(target);
+  const prMerged =
+    target?.pr_status === "merged" || target?.task_status === "merged";
+
+  if (w.kind === "merged" && prMerged) {
+    if (claimClose(w.id, "merged", null, now))
+      await tell(waiter, satisfiedLine(target!, "merged", target!.pr_number));
+    return;
+  }
+  if (w.kind === "merged" && target?.pr_status === "closed") {
+    if (claimClose(w.id, "ended", null, now))
+      await tell(waiter, closedPrLine(target, target.pr_number));
+    return;
+  }
+  if (end || !target) {
+    if (claimClose(w.id, "ended", null, now))
+      await tell(
+        waiter,
+        endedLine(
+          target ?? { id: w.target_id, name: "the session" },
+          end ?? "deleted",
+          w.request
+        )
+      );
+    return;
+  }
+
+  const { status } = await statusOf(target);
+  if (status === "dead") {
+    const n = (dead.get(target.id) ?? 0) + 1;
+    dead.set(target.id, n);
+    if (n >= DEAD_LOOKS) {
+      if (claimClose(w.id, "ended", null, now))
+        await tell(
+          waiter,
+          endedLine(target, "its terminal is gone", w.request)
+        );
+      return;
+    }
+  } else dead.delete(target.id);
+
+  if (status === "running") markSeenBusy(w.id, now);
+  const idle =
+    status === "idle" &&
+    (w.seen_busy_at != null || now - w.created_at >= IDLE_GRACE_MS);
+
+  if (w.kind === "idle" && idle) {
+    if (claimClose(w.id, "idle", null, now))
+      await tell(waiter, satisfiedLine(target, "idle"));
+    return;
+  }
+
+  if (now >= w.deadline) {
+    if (!claimClose(w.id, "timeout", null, now)) return;
+    const is = status === "running" ? "still working" : status;
+    await tell(
+      waiter,
+      timeoutLine(target, w.kind, w.deadline - w.created_at, w.request, is)
+    );
+    const workspace = sessionWorkspace(waiter);
+    if (workspace && waiter.role !== "orchestrator")
+      queueEventOnce(
+        workspace,
+        `wait-timeout:${w.id}`,
+        waiter.id,
+        orchestratorTimeoutLine(
+          waiter.name,
+          target.name,
+          w.deadline - w.created_at,
+          w.request
+        )
+      );
+    return;
+  }
+
+  if (w.kind === "reply" && idle && !w.nudged_at && claimNudge(w.id, now))
+    await tell(target, nudgeLine(waiter, w.request, w.message_id));
+}
+
+let running = false;
+
+export async function waitsTick(now = Date.now()): Promise<void> {
+  if (running) return;
+  running = true;
+  try {
+    const waits = openWaits();
+    if (!waits.length) return;
+    await statusDetector.refreshCache();
+    for (const w of waits) {
+      try {
+        const waiter = row(w.waiter_id);
+        // A waiter that is gone has no one to tell.
+        if (!waiter || endedHow(waiter)) {
+          claimClose(w.id, "cancelled", null, now);
+          continue;
+        }
+        await settle(w, waiter, now);
+      } catch (error) {
+        console.error(`[waits] wait ${w.id}:`, error);
+      }
+    }
+  } finally {
+    running = false;
+  }
+}
+
+let timer: NodeJS.Timeout | null = null;
+let kicked = false;
+
+export function startWaitWatcher(): void {
+  if (timer) return;
+  timer = setInterval(() => void waitsTick(), EVERY_MS);
+  timer.unref();
+  onKick(() => {
+    if (kicked) return;
+    kicked = true;
+    setImmediate(() => {
+      kicked = false;
+      void waitsTick();
+    });
+  });
+}

package/lib/connect/control-api.ts

@@ -214,17 +214,123 @@ export async function requestTunnel(
   return { hostname: hostnameFrom(r), token: tokenFrom(r) };
 }
 
+/** What the account's plan allows; absent from an older Connect service. */
+export interface Entitlements {
+  plan: string;
+  linkMachines: boolean;
+  machineLimit: number;
+}
+
+export interface Beat {
+  enabled: boolean;
+  entitlements?: Entitlements;
+  /** Changes whenever the list `peers` would give changes. */
+  peersEtag?: string;
+}
+
+const ETAG = /^[A-Za-z0-9_-]{1,128}$/;
+
+export function entitlementsFrom(raw: unknown): Entitlements | undefined {
+  const e = (raw ?? null) as Record<string, unknown> | null;
+  if (!e || typeof e !== "object" || typeof e.link_machines !== "boolean")
+    return undefined;
+  return {
+    plan: typeof e.plan === "string" ? clean(e.plan, 32) : "unknown",
+    linkMachines: e.link_machines,
+    machineLimit:
+      typeof e.machine_limit === "number" &&
+      Number.isSafeInteger(e.machine_limit)
+        ? e.machine_limit
+        : 0,
+  };
+}
+
+const etagFrom = (raw: unknown) =>
+  typeof raw === "string" && ETAG.test(raw) ? raw : undefined;
+
 export async function heartbeat(
   config: ConnectConfig,
   privateKey: string,
   version: string
-) {
+): Promise<Beat> {
   const res = await signed(config, privateKey)(
     "POST",
     `/api/v1/machines/${config.machineId}/heartbeat`,
     { version }
   );
-  return (await res.json()) as { enabled: boolean };
+  const r = ((await res.json().catch(() => null)) ?? {}) as Record<
+    string,
+    unknown
+  >;
+  return {
+    enabled: r.enabled !== false,
+    entitlements: entitlementsFrom(r.entitlements),
+    peersEtag: etagFrom(r.peers_etag),
+  };
+}
+
+/** Another machine on this account, as the Connect service introduces it. */
+export interface ListedMachine {
+  id: string;
+  name: string;
+  hostname: string;
+  /** Its Ed25519 key as the service holds it: SPKI PEM. */
+  publicKey: string;
+  lastSeenAt: string | null;
+  version: string | null;
+}
+
+export interface PeerList {
+  entitlements?: Entitlements;
+  peersEtag?: string;
+  peers: ListedMachine[];
+}
+
+const MAX_PEERS = 100;
+
+/**
+ * The account's other machines this one may link to. Only an introduction:
+ * anything malformed is dropped here, and the keys still have to be
+ * confirmed by a person on both machines before either trusts the other.
+ */
+export async function listPeers(
+  config: ConnectConfig,
+  privateKey: string
+): Promise<PeerList> {
+  const res = await signed(config, privateKey)(
+    "GET",
+    `/api/v1/machines/${config.machineId}/peers`
+  );
+  const r = (await res.json().catch(() => null)) as Record<
+    string,
+    unknown
+  > | null;
+  if (!r || !Array.isArray(r.peers))
+    throw new Error("the Connect service sent no machine list");
+  const str = (v: unknown, max: number) =>
+    typeof v === "string" && v.length <= max ? v : null;
+  const peers: ListedMachine[] = [];
+  for (const raw of r.peers.slice(0, MAX_PEERS)) {
+    const m = (raw ?? {}) as Record<string, unknown>;
+    const id = str(m.id, 8);
+    const hostname = str(m.hostname, 253);
+    const publicKey = str(m.public_key, 1024);
+    if (!id || !MACHINE_ID.test(id) || !hostname || !publicKey) continue;
+    if (id === config.machineId) continue;
+    peers.push({
+      id,
+      name: clean(str(m.name, 500) ?? id, 80),
+      hostname,
+      publicKey,
+      lastSeenAt: str(m.last_seen_at, 64),
+      version: str(m.agent_os_version, 64),
+    });
+  }
+  return {
+    entitlements: entitlementsFrom(r.entitlements),
+    peersEtag: etagFrom(r.peers_etag),
+    peers,
+  };
 }
 
 export interface PushRequestBody {

package/lib/connect/heartbeat.ts

@@ -7,12 +7,14 @@
 
 import fs from "fs";
 import path from "path";
-import { ApiError, clean } from "./control-api";
+import { ApiError, clean, type Beat } from "./control-api";
 
 export const HEARTBEAT_MS = 60_000;
 
 export interface HeartbeatOptions {
-  beat: () => Promise<{ enabled: boolean }>;
+  beat: () => Promise<Beat>;
+  /** Every answer, for what it says about the account (plan, peers). */
+  onBeat?: (beat: Beat) => void;
   /** The account disabled this machine: stop the tunnel. */
   onDisabled: () => void;
   /** The service no longer knows this machine. */
@@ -43,9 +45,15 @@ export function startHeartbeat(opts: HeartbeatOptions) {
   const tick = async () => {
     timer = null;
     try {
-      const { enabled } = await opts.beat();
+      const answer = await opts.beat();
+      const { enabled } = answer;
       if (stopped) return;
       failing = false;
+      try {
+        opts.onBeat?.(answer);
+      } catch (err) {
+        log(`connect: ${clean((err as Error)?.message ?? String(err))}`);
+      }
       if (enabled === false) {
         stop();
         log("connect: this machine was disabled in your account; tunnel off");

package/lib/connect/serve.ts

@@ -40,6 +40,7 @@ import {
   type Tunnel,
 } from "./control-api";
 import { appVersion, startHeartbeat } from "./heartbeat";
+import { noteHeartbeat } from "@/lib/peer/account";
 import { ensureCloudflared } from "./cloudflared";
 import {
   reapTunnel,
@@ -331,6 +332,7 @@ export function startConnect(
     const version = appVersion();
     const heartbeat = startHeartbeat({
       beat: () => beat(config, machineKey, version),
+      onBeat: noteHeartbeat,
       log,
       intervalMs: deps.heartbeatMs,
       onDisabled: () => {

package/lib/db/migrations/20261020T1258_peer_links.ts

@@ -0,0 +1,49 @@
+import type { Migration } from "../migration-types";
+
+export default {
+  key: "20261020T1258_peer_links",
+  up: (db) => {
+    // A machine linked through Connect (lib/peer): its pinned keys, never a
+    // token. Foreign keys aren't enforced here, so deleteHost removes the pin.
+    db.exec(`CREATE TABLE IF NOT EXISTS peer_links (
+      host_id TEXT PRIMARY KEY REFERENCES hosts(id) ON DELETE CASCADE,
+      machine_id TEXT NOT NULL UNIQUE,
+      hostname TEXT NOT NULL,
+      sign_pub TEXT NOT NULL,
+      box_pub TEXT NOT NULL,
+      state TEXT NOT NULL DEFAULT 'active',
+      suspended_reason TEXT,
+      paired_at TEXT NOT NULL DEFAULT (datetime('now')),
+      last_ok_at TEXT,
+      last_error TEXT
+    )`);
+    // Every peer call, both directions. Successful polling reads fold into
+    // one row per route per hour (`bucket`, `count`).
+    db.exec(`CREATE TABLE IF NOT EXISTS peer_audit (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      at INTEGER NOT NULL,
+      host_id TEXT,
+      machine_id TEXT,
+      direction TEXT NOT NULL,
+      method TEXT NOT NULL,
+      route TEXT NOT NULL,
+      status INTEGER NOT NULL,
+      ms INTEGER NOT NULL DEFAULT 0,
+      bytes INTEGER NOT NULL DEFAULT 0,
+      outcome TEXT NOT NULL,
+      bucket TEXT,
+      count INTEGER NOT NULL DEFAULT 1
+    )`);
+    db.exec(
+      `CREATE INDEX IF NOT EXISTS idx_peer_audit_host_at ON peer_audit(host_id, at)`
+    );
+    db.exec(
+      `CREATE INDEX IF NOT EXISTS idx_peer_audit_bucket ON peer_audit(host_id, direction, route, bucket)`
+    );
+    const cols = db.prepare(`PRAGMA table_info(hosts)`).all() as {
+      name: string;
+    }[];
+    if (!cols.some((c) => c.name === "kind"))
+      db.exec(`ALTER TABLE hosts ADD COLUMN kind TEXT NOT NULL DEFAULT 'ssh'`);
+  },
+} satisfies Migration;

package/lib/db/migrations/20261020T1345_peer_nonces.ts

@@ -0,0 +1,18 @@
+import type { Migration } from "../migration-types";
+
+export default {
+  key: "20261020T1345_peer_nonces",
+  up: (db) => {
+    // Nonces of linked machines' envelopes (lib/peer/replay.ts), kept until
+    // their timestamp would be stale anyway, so a restart can't replay one.
+    db.exec(`CREATE TABLE IF NOT EXISTS peer_nonces (
+      machine_id TEXT NOT NULL,
+      nonce TEXT NOT NULL,
+      until INTEGER NOT NULL,
+      PRIMARY KEY (machine_id, nonce)
+    )`);
+    db.exec(
+      `CREATE INDEX IF NOT EXISTS idx_peer_nonces_until ON peer_nonces(until)`
+    );
+  },
+} satisfies Migration;

package/lib/db/migrations/20261020T1600_bus_waits.ts

@@ -0,0 +1,35 @@
+import type { Migration } from "../migration-types";
+
+export default {
+  key: "20261020T1600_bus_waits",
+  up: (db) => {
+    // A session waiting on another (lib/bus/waits): for its reply, or for it
+    // to go idle or merge. Closed rows stay as history.
+    db.exec(`CREATE TABLE IF NOT EXISTS bus_waits (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      waiter_id TEXT NOT NULL,
+      target_id TEXT NOT NULL,
+      kind TEXT NOT NULL,
+      message_id INTEGER,
+      request TEXT,
+      deadline INTEGER NOT NULL,
+      created_at INTEGER NOT NULL,
+      seen_busy_at INTEGER,
+      nudged_at INTEGER,
+      closed_at INTEGER,
+      outcome TEXT,
+      reply_id INTEGER
+    )`);
+    db.exec(
+      `CREATE INDEX IF NOT EXISTS idx_bus_waits_open ON bus_waits(target_id) WHERE closed_at IS NULL`
+    );
+    db.exec(
+      `CREATE INDEX IF NOT EXISTS idx_bus_waits_waiter ON bus_waits(waiter_id) WHERE closed_at IS NULL`
+    );
+    const cols = db.prepare(`PRAGMA table_info(bus_messages)`).all() as {
+      name: string;
+    }[];
+    if (!cols.some((c) => c.name === "reply_to"))
+      db.exec(`ALTER TABLE bus_messages ADD COLUMN reply_to INTEGER`);
+  },
+} satisfies Migration;

package/lib/db/types.ts

@@ -90,6 +90,10 @@ export interface Session {
   pause_id?: number | null;
   // Why a requested resume hasn't happened yet.
   pause_note?: string | null;
+  // Set only by GET /api/sessions: the sessions it waits on, and who waits on
+  // it, as a short line (lib/bus/waits/store).
+  waiting_on?: string | null;
+  waited_by?: string | null;
   placed_note?: string | null;
   // Set by done: hidden from the sidebar and the orchestrator, never deleted.
   archived_at: string | null;
@@ -306,6 +310,8 @@ export interface Host {
   ssh_target: string;
   sort_order: number;
   created_at: string;
+  // 'connect': reached only through Connect (lib/peer), ssh_target is ''.
+  kind?: "ssh" | "connect";
   // Paired with that machine's own AgentOS (lib/hosts/link.ts); 0/1 from SQLite.
   linked?: boolean | number;
 }