@saadnvd1/agent-os 0.3.697 → 0.3.698

package/lib/hosts/remote-api.ts

@@ -4,38 +4,112 @@
  */
 
 import { getDb } from "../db";
+import { PeerCallError, peerCall } from "../peer/client";
+import type { PinState } from "../peer/links";
 import { getHost, isRemoteHost } from "./index";
+import { readCappedBytes } from "./read-capped-bytes";
 
+/** A machine linked through Connect: its pinned keys (lib/peer). */
+export interface ConnectPeer {
+  machineId: string;
+  hostname: string;
+  signPub: string;
+  boxPub: string;
+  state: PinState;
+  suspendedReason: string | null;
+}
+
+/**
+ * `token` (the default): a device token over Tailscale or ssh-reached URLs.
+ * `connect`: signed, encrypted envelopes to the pinned keys; url and token
+ * are unused (token is always empty).
+ */
 export interface HostLink {
   hostId: string;
   hostName: string;
   url: string;
   token: string;
+  transport?: "token" | "connect";
+  connect?: ConnectPeer;
+}
+
+interface LinkRow {
+  host_id: string;
+  name: string;
+  kind: string;
+  url: string | null;
+  token: string | null;
+  machine_id: string | null;
+  hostname: string | null;
+  sign_pub: string | null;
+  box_pub: string | null;
+  state: string | null;
+  suspended_reason: string | null;
 }
 
 export function hostLink(hostId: string | null | undefined): HostLink | null {
   if (!isRemoteHost(hostId)) return null;
   const row = getDb()
     .prepare(
-      `SELECT l.host_id, l.url, l.token, h.name FROM host_links l
-         JOIN hosts h ON h.id = l.host_id WHERE l.host_id = ?`
+      `SELECT h.id AS host_id, h.name, h.kind, l.url, l.token, p.machine_id,
+              p.hostname, p.sign_pub, p.box_pub, p.state, p.suspended_reason
+         FROM hosts h
+         LEFT JOIN host_links l ON l.host_id = h.id
+         LEFT JOIN peer_links p ON p.host_id = h.id
+        WHERE h.id = ?`
     )
-    .get(hostId) as
-    { host_id: string; url: string; token: string; name: string } | undefined;
-  return row
+    .get(hostId) as LinkRow | undefined;
+  if (!row) return null;
+  if (row.kind === "connect") {
+    if (!row.machine_id || !row.hostname || !row.sign_pub || !row.box_pub)
+      return null;
+    return {
+      hostId: row.host_id,
+      hostName: row.name,
+      url: `https://${row.hostname}`,
+      token: "",
+      transport: "connect",
+      connect: {
+        machineId: row.machine_id,
+        hostname: row.hostname,
+        signPub: row.sign_pub,
+        boxPub: row.box_pub,
+        state: row.state === "active" ? "active" : "suspended",
+        suspendedReason: row.suspended_reason,
+      },
+    };
+  }
+  return row.url && row.token
     ? {
         hostId: row.host_id,
         hostName: row.name,
         url: row.url,
         token: row.token,
+        transport: "token",
       }
     : null;
 }
 
+export const isConnectLink = (link: HostLink) => link.transport === "connect";
+
+/** For the calls that only work with a token link (sockets, raw fetches). */
+export function requireTokenLink(link: HostLink): HostLink {
+  if (isConnectLink(link))
+    throw new Error(
+      `${link.hostName} is linked through Connect: this needs a Tailscale/ssh link for now`
+    );
+  return link;
+}
+
 export function linkedHostIds(): Set<string> {
-  const rows = getDb().prepare(`SELECT host_id FROM host_links`).all() as {
-    host_id: string;
-  }[];
+  const rows = getDb()
+    .prepare(
+      `SELECT host_id FROM host_links
+         WHERE host_id NOT IN (SELECT id FROM hosts WHERE kind = 'connect')
+       UNION SELECT p.host_id FROM peer_links p JOIN hosts h ON h.id = p.host_id
+         WHERE h.kind = 'connect'`
+    )
+    .all() as { host_id: string }[];
   return new Set(rows.map((r) => r.host_id));
 }
 
@@ -103,23 +177,7 @@ export async function readCapped(
   res: Response,
   max = MAX_RESPONSE_BYTES
 ): Promise<string> {
-  if (Number(res.headers.get("content-length")) > max)
-    throw new Error("its answer is too large");
-  const reader = res.body?.getReader();
-  if (!reader) return "";
-  const chunks: Uint8Array[] = [];
-  let total = 0;
-  for (;;) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    total += value.byteLength;
-    if (total > max) {
-      await reader.cancel().catch(() => undefined);
-      throw new Error("its answer is too large");
-    }
-    chunks.push(value);
-  }
-  return Buffer.concat(chunks).toString("utf8");
+  return (await readCappedBytes(res, max)).toString("utf8");
 }
 
 /** A JSON call to the other machine's AgentOS; its error text on failure. */
@@ -133,21 +191,34 @@ export async function hostApi<T>(
       `Can't reach AgentOS on ${link.hostName}: ${cleanRemoteText(err instanceof Error ? err.message : err)}`,
       false
     );
-  let res: Response;
+  let status: number;
   let text: string;
   try {
-    res = await fetch(`${link.url}${path}`, {
-      method: opts.method ?? (opts.body === undefined ? "GET" : "POST"),
-      headers: {
-        Authorization: `Bearer ${link.token}`,
-        ...(opts.body !== undefined && { "Content-Type": "application/json" }),
-      },
-      body: opts.body === undefined ? undefined : JSON.stringify(opts.body),
-      signal: AbortSignal.timeout(opts.timeout ?? 15000),
-      redirect: "error",
-    });
-    text = await readCapped(res);
+    if (isConnectLink(link)) {
+      ({ status, text } = await peerCall(link, path, opts));
+    } else {
+      const res = await fetch(`${link.url}${path}`, {
+        method: opts.method ?? (opts.body === undefined ? "GET" : "POST"),
+        headers: {
+          Authorization: `Bearer ${link.token}`,
+          ...(opts.body !== undefined && {
+            "Content-Type": "application/json",
+          }),
+        },
+        body: opts.body === undefined ? undefined : JSON.stringify(opts.body),
+        signal: AbortSignal.timeout(opts.timeout ?? 15000),
+        redirect: "error",
+      });
+      status = res.status;
+      text = await readCapped(res);
+    }
   } catch (err) {
+    if (err instanceof PeerCallError && err.refused)
+      throw new HostApiError(
+        `${link.hostName}: ${cleanRemoteText(err.message)}`,
+        true,
+        err.status
+      );
     throw unknown(err);
   }
   let data: { error?: unknown } | null = null;
@@ -156,13 +227,13 @@ export async function hostApi<T>(
   } catch {
     data = null;
   }
-  if (!res.ok) {
+  if (status < 200 || status > 299) {
     // Only AgentOS's own refusal (a 4xx with its error) means nothing happened.
-    const refused = res.status >= 400 && res.status < 500 && !!data?.error;
+    const refused = status >= 400 && status < 500 && !!data?.error;
     throw new HostApiError(
-      `${link.hostName}: ${cleanRemoteText(data?.error) || `HTTP ${res.status}`}`,
+      `${link.hostName}: ${cleanRemoteText(data?.error) || `HTTP ${status}`}`,
       refused,
-      res.status
+      status
     );
   }
   if (!data) throw unknown(new Error("its answer wasn't JSON"));

package/lib/peer/allowlist.ts

@@ -0,0 +1,136 @@
+/**
+ * The only routes a linked machine may call here, by method and pattern.
+ * These are the calls the peer code makes today and nothing else. A path is
+ * taken only in its plain form: no encoding, no dot segments, no doubled or
+ * back slashes, so what is matched is exactly what the route sees. An id
+ * segment is [A-Za-z0-9_.-]{1,128}, and each route names its query keys.
+ */
+
+type Method = "GET" | "POST" | "DELETE";
+
+interface PeerRoute {
+  method: Method;
+  pattern: string;
+  query?: readonly string[];
+}
+
+const each = (
+  method: Method,
+  base: string,
+  tails: string[],
+  query?: string[]
+): PeerRoute[] =>
+  tails.map((t) => ({
+    method,
+    pattern: `${base}/${t}`,
+    ...(query && { query }),
+  }));
+
+export const PEER_ROUTES: readonly PeerRoute[] = [
+  { method: "GET", pattern: "/api/tasks" },
+  { method: "POST", pattern: "/api/tasks" },
+  { method: "POST", pattern: "/api/tasks/import" },
+  { method: "GET", pattern: "/api/tasks/arrived", query: ["from"] },
+  { method: "GET", pattern: "/api/tasks/held-replies" },
+  { method: "POST", pattern: "/api/tasks/held-replies" },
+  ...each("POST", "/api/tasks/:id", [
+    "export",
+    "approve-plan",
+    "hold",
+    "message",
+    "stop",
+    "done",
+    "drop",
+    "resume",
+    "moved",
+    "merge",
+  ]),
+  ...each("GET", "/api/tasks/:id", ["plan", "chat-facts"]),
+  { method: "GET", pattern: "/api/tasks/:id/chat-tail", query: ["lines"] },
+
+  { method: "GET", pattern: "/api/sessions" },
+  { method: "GET", pattern: "/api/sessions/status" },
+  { method: "POST", pattern: "/api/sessions" },
+  { method: "DELETE", pattern: "/api/sessions/:id" },
+  ...each("POST", "/api/sessions/:id", ["done", "unarchive", "resurrect"]),
+  ...each("GET", "/api/sessions/:id", ["preview", "artifacts", "setup"]),
+
+  { method: "GET", pattern: "/api/projects" },
+  { method: "POST", pattern: "/api/projects/availability" },
+  { method: "GET", pattern: "/api/tmux/discover" },
+  { method: "GET", pattern: "/api/load" },
+
+  { method: "GET", pattern: "/api/worktree-sweep" },
+  { method: "POST", pattern: "/api/worktree-sweep" },
+  { method: "POST", pattern: "/api/worktree-sweep/run" },
+
+  { method: "POST", pattern: "/api/peer/unlink" },
+  { method: "GET", pattern: "/api/peer/ping" },
+
+  // The work registry's exchange (not built yet: 404 until it is).
+  { method: "POST", pattern: "/api/claims/exchange" },
+];
+
+const PLAIN_PATH = /^\/api(\/[A-Za-z0-9_.-]{1,128}){1,6}$/;
+const ID = /^[A-Za-z0-9_.-]{1,128}$/;
+const MAX_QUERY = 4096;
+
+export type RouteMatch =
+  | { ok: true; route: string; path: string; query: string }
+  | { ok: false; reason: "bad_path" | "bad_query" | "not_allowed" };
+
+function segmentsMatch(pattern: string[], path: string[]): boolean {
+  return (
+    pattern.length === path.length &&
+    pattern.every((p, i) => (p === ":id" ? ID.test(path[i]) : p === path[i]))
+  );
+}
+
+function cleanQuery(
+  raw: string,
+  allowed: readonly string[] = []
+): string | null {
+  if (!raw) return "";
+  if (raw.length > MAX_QUERY || /[#\u0000-\u001f\u007f]/.test(raw)) return null;
+  const params = new URLSearchParams(raw);
+  const keys = [...params.keys()];
+  if (new Set(keys).size !== keys.length) return null;
+  for (const [k, v] of params) {
+    if (!allowed.includes(k) || v.length > 1024) return null;
+    if (/[\u0000-\u001f\u007f]/.test(v)) return null;
+  }
+  return params.toString();
+}
+
+/** The route a peer's call matches, with its path and query as dispatched. */
+export function matchPeerRoute(
+  method: string,
+  path: string,
+  query = ""
+): RouteMatch {
+  if (typeof path !== "string" || !PLAIN_PATH.test(path))
+    return { ok: false, reason: "bad_path" };
+  const parts = path.split("/").slice(1);
+  if (parts.some((p) => p === "." || p === ".."))
+    return { ok: false, reason: "bad_path" };
+  const route = PEER_ROUTES.find(
+    (r) =>
+      r.method === method && segmentsMatch(r.pattern.split("/").slice(1), parts)
+  );
+  if (!route) return { ok: false, reason: "not_allowed" };
+  const clean = cleanQuery(query, route.query);
+  if (clean === null) return { ok: false, reason: "bad_query" };
+  return { ok: true, route: route.pattern, path, query: clean };
+}
+
+/** For the gate: a request target (path and query) a peer may reach. */
+export function peerTargetAllowed(
+  method: string | undefined,
+  url: string | undefined
+) {
+  const target = url ?? "";
+  const q = target.indexOf("?");
+  const path = q < 0 ? target : target.slice(0, q);
+  const query = q < 0 ? "" : target.slice(q + 1);
+  return matchPeerRoute(method ?? "", path, query).ok;
+}

package/lib/peer/audit.ts

@@ -0,0 +1,144 @@
+/**
+ * Peer calls, in and out. Successful reads (the mirror polls every few
+ * seconds) fold into one row per route per hour. Failures are capped per
+ * minute so a flood can't fill the disk, in two budgets: envelopes refused
+ * here before their sender was proven (anyone can send those), and
+ * everything else (this machine's own failed calls, a linked machine's
+ * denied routes), so a forger's flood never hides the second kind. Rows older than 30 days go, checked at most hourly.
+ * Bookkeeping: a failure here never fails the call.
+ */
+
+import { getDb } from "../db";
+
+export type Outcome =
+  | "ok"
+  | "denied_route"
+  | "bad_envelope"
+  | "too_large"
+  | "too_slow"
+  | "busy"
+  | "bad_sig"
+  | "stale"
+  | "replay"
+  | "unknown_peer"
+  | "suspended";
+
+export interface AuditEntry {
+  hostId: string | null;
+  machineId?: string | null;
+  direction: "in" | "out";
+  method: string;
+  /** The route's template (/api/tasks/:id/merge), never the raw path. */
+  route: string;
+  status: number;
+  ms: number;
+  bytes: number;
+  outcome: Outcome;
+}
+
+const HOUR = 3_600_000;
+const RETAIN_MS = 30 * 24 * HOUR;
+const FAILURES_PER_MINUTE = 30;
+
+const state = { prunedAt: 0, minute: 0, forged: 0, other: 0 };
+
+/** For tests: forget the prune time and the failure budget. */
+export const resetAuditLimits = () => {
+  state.prunedAt = 0;
+  state.minute = 0;
+  state.forged = 0;
+  state.other = 0;
+};
+
+// Refusals of an inbound envelope that anyone could have sent: a stranger's,
+// a forgery, a malformed or oversized one, or a captured one sent again.
+const UNPROVEN: ReadonlySet<Outcome> = new Set([
+  "bad_envelope",
+  "too_large",
+  "too_slow",
+  "busy",
+  "bad_sig",
+  "unknown_peer",
+  "suspended",
+  "stale",
+  "replay",
+]);
+
+function underFailureCap(e: AuditEntry, now: number): boolean {
+  const minute = Math.floor(now / 60_000);
+  if (minute !== state.minute) {
+    state.minute = minute;
+    state.forged = 0;
+    state.other = 0;
+  }
+  const key =
+    e.direction === "in" && UNPROVEN.has(e.outcome) ? "forged" : "other";
+  return ++state[key] <= FAILURES_PER_MINUTE;
+}
+
+export function recordPeerCall(e: AuditEntry, now = Date.now()): void {
+  try {
+    if (e.outcome !== "ok" && !underFailureCap(e, now)) return;
+    const db = getDb();
+    if (now - state.prunedAt > HOUR) {
+      state.prunedAt = now;
+      db.prepare(`DELETE FROM peer_audit WHERE at < ?`).run(now - RETAIN_MS);
+    }
+    const machineId = e.machineId?.slice(0, 64) ?? null;
+    const fold = e.method === "GET" && e.outcome === "ok" && e.status < 400;
+    const bucket = fold ? String(Math.floor(now / HOUR)) : null;
+    if (bucket) {
+      const changed = db
+        .prepare(
+          `UPDATE peer_audit SET count = count + 1, at = ?, status = ?, ms = ?,
+             bytes = bytes + ?
+           WHERE host_id IS ? AND direction = ? AND route = ? AND bucket = ?`
+        )
+        .run(
+          now,
+          e.status,
+          e.ms,
+          e.bytes,
+          e.hostId,
+          e.direction,
+          e.route,
+          bucket
+        );
+      if (changed.changes > 0) return;
+    }
+    db.prepare(
+      `INSERT INTO peer_audit (at, host_id, machine_id, direction, method, route,
+         status, ms, bytes, outcome, bucket)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+    ).run(
+      now,
+      e.hostId,
+      machineId,
+      e.direction,
+      e.method.slice(0, 10),
+      e.route.slice(0, 200),
+      e.status,
+      Math.round(e.ms),
+      e.bytes,
+      e.outcome,
+      bucket
+    );
+  } catch (err) {
+    console.error("peer audit:", err);
+  }
+}
+
+export interface AuditRow extends AuditEntry {
+  at: number;
+  count: number;
+}
+
+export function peerAudit(hostId: string, limit = 200): AuditRow[] {
+  return getDb()
+    .prepare(
+      `SELECT at, host_id AS hostId, machine_id AS machineId, direction, method,
+           route, status, ms, bytes, outcome, count
+         FROM peer_audit WHERE host_id = ? ORDER BY at DESC, id DESC LIMIT ?`
+    )
+    .all(hostId, limit) as AuditRow[];
+}

package/lib/peer/client.ts

@@ -0,0 +1,188 @@
+/**
+ * hostApi's transport for a machine linked through Connect: each call is a
+ * fresh envelope (envelope.ts) to POST https://<its m- name>/api/peer/rpc,
+ * and the answer only counts if it is signed by the pinned key and bound to
+ * this request's nonce. No token is ever sent or stored.
+ */
+
+import { matchPeerRoute } from "./allowlist";
+import { recordPeerCall, type AuditEntry, type Outcome } from "./audit";
+import {
+  MAX_REQUEST_BODY,
+  MAX_RESPONSE_ENVELOPE,
+  RPC_PATH,
+  openResponse,
+  sealRequest,
+} from "./envelope";
+import { boxPublic, currentIdentity, signPublic, type Identity } from "./keys";
+import { notePinResult } from "./links";
+import type { HostLink } from "../hosts/remote-api";
+import { readCappedBytes } from "../hosts/read-capped-bytes";
+
+/**
+ * A failed peer call. `refused`: the other machine (or this one, before
+ * sending) said no, so nothing happened there.
+ */
+export class PeerCallError extends Error {
+  constructor(
+    message: string,
+    readonly refused: boolean,
+    readonly status?: number
+  ) {
+    super(message);
+  }
+}
+
+const DOMAIN = /^[a-z0-9-]+(\.[a-z0-9-]+)+$/;
+
+/**
+ * Only a Connect machine's own name, so a lying account list can't aim this
+ * machine at an internal address. AGENTOS_CONNECT_PEER_DOMAIN is for a
+ * staging service.
+ */
+export function peerHostAllowed(hostname: string, env = process.env): boolean {
+  const configured = env.AGENTOS_CONNECT_PEER_DOMAIN ?? "";
+  const domain = DOMAIN.test(configured) ? configured : "runagentos.com";
+  const escaped = domain.replace(/\./g, "\\.");
+  return new RegExp(`^m-[0-9a-f]{16}\\.${escaped}$`).test(hostname);
+}
+
+// The fixed refusals rpc-server.ts sends before anything is decrypted.
+const REFUSALS: Record<string, Outcome> = {
+  "bad signature": "bad_sig",
+  "bad envelope": "bad_sig",
+  stale: "stale",
+  replay: "replay",
+  busy: "replay",
+  "too large": "bad_sig",
+};
+
+const HINTS: Record<string, string> = {
+  "bad signature":
+    "it doesn't know this machine, the link is suspended there, or the keys don't match",
+  stale: "the clocks on the two machines differ by more than 2 minutes",
+};
+
+export interface ClientDeps {
+  identity: () => Identity | null;
+  audit: (e: AuditEntry) => void;
+  noteResult: (hostId: string, error: string | null) => void;
+}
+
+const realDeps: ClientDeps = {
+  identity: () => currentIdentity(),
+  audit: recordPeerCall,
+  noteResult: notePinResult,
+};
+
+export async function peerCall(
+  link: HostLink,
+  target: string,
+  opts: { method?: string; body?: unknown; timeout?: number },
+  deps: ClientDeps = realDeps
+): Promise<{ status: number; text: string }> {
+  const peer = link.connect;
+  const refuse = (msg: string): never => {
+    throw new PeerCallError(msg, true);
+  };
+  if (!peer) return refuse("isn't linked through Connect");
+  if (peer.state !== "active")
+    return refuse(
+      `the link is suspended (${peer.suspendedReason ?? "no reason given"})`
+    );
+  if (!peerHostAllowed(peer.hostname))
+    return refuse("its address isn't a Connect machine's");
+  const method = (
+    opts.method ?? (opts.body === undefined ? "GET" : "POST")
+  ).toUpperCase();
+  const q = target.indexOf("?");
+  const match = matchPeerRoute(
+    method,
+    q < 0 ? target : target.slice(0, q),
+    q < 0 ? "" : target.slice(q + 1)
+  );
+  if (!match.ok) return refuse("that call isn't open to linked machines");
+  const identity = deps.identity();
+  if (!identity)
+    return refuse("this machine isn't connected to AgentOS Connect");
+  const body =
+    opts.body === undefined
+      ? Buffer.alloc(0)
+      : Buffer.from(JSON.stringify(opts.body));
+  if (body.length > MAX_REQUEST_BODY) return refuse("the request is too large");
+
+  const started = Date.now();
+  const audit = (outcome: Outcome, status: number, bytes: number) =>
+    deps.audit({
+      hostId: link.hostId,
+      machineId: peer.machineId,
+      direction: "out",
+      method,
+      route: match.route,
+      status,
+      ms: Date.now() - started,
+      bytes,
+      outcome,
+    });
+  const sealed = sealRequest({
+    from: identity.machineId,
+    to: peer.machineId,
+    signKey: identity.signKey,
+    peerBox: boxPublic(peer.boxPub),
+    request: {
+      method,
+      path: match.path,
+      query: match.query,
+      contentType: body.length ? "application/json" : "",
+      body,
+    },
+  });
+  let res: Response;
+  let raw: Buffer;
+  try {
+    res = await fetch(`https://${peer.hostname}${RPC_PATH}`, {
+      method: "POST",
+      headers: { "Content-Type": "application/octet-stream" },
+      body: new Uint8Array(sealed.envelope),
+      signal: AbortSignal.timeout(opts.timeout ?? 15000),
+      redirect: "error",
+    });
+    raw = await readCappedBytes(res, MAX_RESPONSE_ENVELOPE);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    deps.noteResult(link.hostId, message);
+    throw new PeerCallError(message, false);
+  }
+  if (res.status !== 200) {
+    const text = raw.subarray(0, 64).toString("utf8").trim();
+    const outcome = REFUSALS[text];
+    if (outcome && res.status >= 400 && res.status < 500) {
+      audit(outcome, res.status, sealed.envelope.length);
+      const message = HINTS[text] ? `${text}: ${HINTS[text]}` : text;
+      deps.noteResult(link.hostId, message);
+      // A refusal is unsigned: anyone on the path could forge one. A read
+      // changes nothing either way, but a write may have happened.
+      throw new PeerCallError(message, method === "GET", res.status);
+    }
+    deps.noteResult(link.hostId, `HTTP ${res.status}`);
+    throw new PeerCallError(`HTTP ${res.status}`, false, res.status);
+  }
+  let answer;
+  try {
+    answer = openResponse({
+      envelope: raw,
+      nonce: sealed.nonce,
+      resKey: sealed.resKey,
+      peerSign: signPublic(peer.signPub),
+    });
+  } catch (err) {
+    // It may have run there; the answer just can't be trusted.
+    const message = err instanceof Error ? err.message : String(err);
+    audit("bad_sig", 200, sealed.envelope.length + raw.length);
+    deps.noteResult(link.hostId, message);
+    throw new PeerCallError(message, false);
+  }
+  audit("ok", answer.status, body.length + answer.body.length);
+  deps.noteResult(link.hostId, null);
+  return { status: answer.status, text: answer.body.toString("utf8") };
+}