agent-relay-orchestrator 0.10.22 → 0.10.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.10.22",
+  "version": "0.10.24",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {

package/src/api.ts

@@ -7,6 +7,7 @@ import type { OrchestratorConfig } from "./config";
 import type { ProviderProbeCache } from "./provider-probe";
 import { captureSession, captureTerminal, createTerminalGuest, listSessions, sendTerminalInput, resizeTerminal, stopTerminalGuest } from "./spawn";
 import type { TerminalSnapshot } from "./spawn";
+import { acquireTerminalStream, type TerminalStreamHandle, type TerminalStreamSubscriber } from "./terminal-stream";
 import { VERSION, runtimeMetadata } from "./version";
 import { previewWorkspaceMerge, probeWorkspace, workspaceDiff, workspaceGitState } from "./workspace-probe";
 
@@ -58,16 +59,19 @@ export interface FileStatResult {
 }
 
 const MAX_FILE_PREVIEW_BYTES = 1024 * 1024;
-const DEFAULT_TERMINAL_STREAM_HEARTBEAT_MS = 5_000;
 
 interface TerminalSocketData {
   kind: "terminal";
   config: OrchestratorConfig;
   session: string;
-  timer?: ReturnType<typeof setInterval>;
-  lastSnapshotSignature?: string;
-  lastHeartbeatAt?: number;
+  stream?: TerminalStreamHandle;
   paused?: boolean;
+  // Backfill is deferred until the client reports its size, so the captured pane
+  // wraps identically to the viewer's xterm. Until then, live bytes queue.
+  synced?: boolean;
+  ready?: boolean;
+  queue?: Uint8Array[];
+  syncTimer?: ReturnType<typeof setTimeout>;
 }
 
 type TerminalSocket = ServerWebSocket<TerminalSocketData>;
@@ -555,11 +559,91 @@ export function startApiServer(config: OrchestratorConfig, probeCache: ProviderP
 }
 
 function startTerminalSocket(ws: TerminalSocket): void {
-  sendTerminalSnapshot(ws, true);
-  const intervalMs = Number(process.env.AGENT_RELAY_TERMINAL_STREAM_INTERVAL_MS) || 125;
-  ws.data.timer = setInterval(() => {
-    if (!ws.data.paused) sendTerminalSnapshot(ws, false);
-  }, Math.max(50, intervalMs));
+  ws.data.queue = [];
+  const subscriber: TerminalStreamSubscriber = {
+    onData: (bytes) => {
+      if (ws.data.paused) return;
+      // Queue live bytes until the client has reported its size and the backfill is
+      // flushed, so the viewer never sees live output before (or mis-sized against) history.
+      if (!ws.data.ready) {
+        ws.data.queue?.push(bytes);
+        return;
+      }
+      try {
+        ws.send(bytes);
+      } catch {}
+    },
+    onClose: (reason) => {
+      try {
+        ws.send(JSON.stringify({ type: "closed", reason: reason ?? null }));
+      } catch {}
+    },
+    bufferedAmount: () => {
+      try {
+        return ws.getBufferedAmount();
+      } catch {
+        return 0;
+      }
+    },
+  };
+  try {
+    ws.data.stream = acquireTerminalStream(ws.data.session, ws.data.config, subscriber);
+    // Wait for the client's resize to size the pane before backfilling. Fall back to
+    // the pane's current size if no resize arrives (e.g. a non-fitting client).
+    ws.data.syncTimer = setTimeout(() => syncAndBackfill(ws), 700);
+  } catch (e) {
+    ws.send(JSON.stringify({ type: "error", error: e instanceof Error ? e.message : String(e) }));
+    ws.close();
+  }
+}
+
+// Resize the pane to the viewer's dimensions, capture the matching backfill, and
+// release the queued live bytes. Idempotent-safe: only the first call backfills.
+function syncAndBackfill(ws: TerminalSocket, cols?: number, rows?: number): void {
+  if (ws.data.syncTimer) {
+    clearTimeout(ws.data.syncTimer);
+    ws.data.syncTimer = undefined;
+  }
+  ws.data.synced = true;
+  sendBackfill(ws, cols, rows);
+  ws.data.ready = true;
+  const queued = ws.data.queue ?? [];
+  ws.data.queue = [];
+  for (const bytes of queued) {
+    try {
+      ws.send(bytes);
+    } catch {}
+  }
+}
+
+// Send a full reset: a control frame with current geometry/status, then the
+// captured scrollback as a raw byte frame. Used on connect, resume, and refresh.
+function sendBackfill(ws: TerminalSocket, cols?: number, rows?: number): void {
+  const stream = ws.data.stream;
+  if (!stream) return;
+  const snapshot = stream.backfill(cols, rows);
+  ws.send(JSON.stringify({
+    type: "reset",
+    session: snapshot.session,
+    running: snapshot.running,
+    agentAlive: snapshot.agentAlive,
+    cols: snapshot.cols,
+    rows: snapshot.rows,
+    cursorX: snapshot.cursorX,
+    cursorY: snapshot.cursorY,
+    capturedAt: snapshot.capturedAt,
+  }));
+  if (snapshot.content) {
+    // Park the cursor where tmux actually has it (screen-relative). Without this the
+    // cursor sits at the end of the captured text, so the TUI's first relative redraw
+    // (cursor-up + rewrite of its prompt/statusline) lands at the wrong row and stacks
+    // a ghost copy into scrollback.
+    let content = snapshot.content;
+    if (snapshot.cursorX != null && snapshot.cursorY != null) {
+      content += `\x1b[${snapshot.cursorY + 1};${snapshot.cursorX + 1}H`;
+    }
+    ws.send(new TextEncoder().encode(content));
+  }
 }
 
 function handleTerminalSocketMessage(ws: TerminalSocket, data: string | Buffer): void {
@@ -575,63 +659,37 @@ function handleTerminalSocketMessage(ws: TerminalSocket, data: string | Buffer):
   const frame = payload as Record<string, unknown>;
   try {
     if (frame.type === "input") {
-      sendTerminalInput(ws.data.session, ws.data.config, { data: typeof frame.data === "string" ? frame.data : "" });
-      sendTerminalSnapshot(ws, true);
+      const text = typeof frame.data === "string" ? frame.data : "";
+      if (text) ws.data.stream?.write(new TextEncoder().encode(text));
     } else if (frame.type === "resize") {
-      resizeTerminal(ws.data.session, ws.data.config, { cols: frame.cols, rows: frame.rows });
-      sendTerminalSnapshot(ws, true);
+      const cols = Number(frame.cols);
+      const rows = Number(frame.rows);
+      // First resize sizes the pane and triggers the (size-matched) backfill;
+      // later ones just reflow the live stream.
+      if (!ws.data.synced) {
+        syncAndBackfill(ws, cols, rows);
+      } else {
+        ws.data.stream?.resize(cols, rows);
+      }
     } else if (frame.type === "pause") {
       ws.data.paused = frame.paused === true;
-      if (!ws.data.paused) sendTerminalSnapshot(ws, true);
+      // We drop (not buffer) bytes while paused, so resync with a fresh backfill.
+      if (!ws.data.paused && ws.data.synced) sendBackfill(ws);
     } else if (frame.type === "refresh") {
-      sendTerminalSnapshot(ws, true);
-    }
-  } catch (e) {
-    ws.send(JSON.stringify({ type: "error", error: e instanceof Error ? e.message : String(e) }));
-  }
-}
-
-function sendTerminalSnapshot(ws: TerminalSocket, force: boolean): void {
-  try {
-    const snapshot = captureTerminal(ws.data.session, ws.data.config);
-    const signature = terminalSnapshotSignature(snapshot);
-    const changed = force || signature !== ws.data.lastSnapshotSignature;
-    if (!changed) {
-      sendTerminalHeartbeat(ws, snapshot, signature);
-      return;
+      if (ws.data.synced) sendBackfill(ws);
     }
-    ws.data.lastSnapshotSignature = signature;
-    ws.data.lastHeartbeatAt = snapshot.capturedAt;
-    ws.send(JSON.stringify({ type: "snapshot", ...snapshot }));
   } catch (e) {
     ws.send(JSON.stringify({ type: "error", error: e instanceof Error ? e.message : String(e) }));
-    ws.close();
   }
 }
 
-function sendTerminalHeartbeat(ws: TerminalSocket, snapshot: TerminalSnapshot, signature: string): void {
-  const heartbeatMs = Math.max(
-    1_000,
-    Number(process.env.AGENT_RELAY_TERMINAL_STREAM_HEARTBEAT_MS) || DEFAULT_TERMINAL_STREAM_HEARTBEAT_MS,
-  );
-  if (ws.data.lastHeartbeatAt && snapshot.capturedAt - ws.data.lastHeartbeatAt < heartbeatMs) return;
-  ws.data.lastHeartbeatAt = snapshot.capturedAt;
-  ws.send(JSON.stringify({
-    type: "heartbeat",
-    session: snapshot.session,
-    running: snapshot.running,
-    cols: snapshot.cols,
-    rows: snapshot.rows,
-    cursorX: snapshot.cursorX,
-    cursorY: snapshot.cursorY,
-    capturedAt: snapshot.capturedAt,
-    signature,
-  }));
-}
-
 function stopTerminalSocket(ws: TerminalSocket): void {
-  if (ws.data.timer) clearInterval(ws.data.timer);
-  ws.data.timer = undefined;
+  if (ws.data.syncTimer) {
+    clearTimeout(ws.data.syncTimer);
+    ws.data.syncTimer = undefined;
+  }
+  ws.data.stream?.release();
+  ws.data.stream = undefined;
 }
 
 function cleanTerminalGuestInput(value: unknown): { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string } {

package/src/spawn.ts

@@ -1096,12 +1096,12 @@ export function resizeTerminal(name: string, config: OrchestratorConfig, input:
   return { session: name, ...clamped };
 }
 
-function tmuxSocketForSession(name: string): string | undefined {
+export function tmuxSocketForSession(name: string): string | undefined {
   const record = loadState().find((item) => item.name === name);
   return record ? readRunnerInfo(record)?.tmuxSocket : undefined;
 }
 
-function tmuxCommand(socketName: string | undefined, ...args: string[]): string[] {
+export function tmuxCommand(socketName: string | undefined, ...args: string[]): string[] {
   return socketName ? ["tmux", "-L", socketName, ...args] : ["tmux", ...args];
 }
 

package/src/terminal-stream.ts

@@ -0,0 +1,335 @@
+// Live terminal streaming via tmux control mode (`tmux -C`).
+//
+// One control-mode client is attached per tmux *session* and shared (ref-counted)
+// across every connected viewer, so the cost is flat in viewer count: the control
+// protocol is parsed once per session, then raw bytes fan out to all subscribers.
+//
+// Output: tmux emits `%output %<pane> <octal-escaped-bytes>` lines; we decode them
+// to raw bytes, coalesce on a short window, and broadcast. No screen-scraping, no
+// full-buffer repaint — the client just writes the byte stream to xterm.
+//
+// Input/resize: written as plain command lines to the control client's stdin
+// (`send-keys -H <hex>`, `resize-window`), so a keystroke costs no process spawn.
+// The relay's own injection path (message delivery, /compact, initial prompts) uses
+// its separate `tmux send-keys` calls and is unaffected — control mode is just
+// another observing client.
+
+import { captureTerminal, tmuxCommand, tmuxSocketForSession, type TerminalSnapshot } from "./spawn";
+import type { OrchestratorConfig } from "./config";
+
+const FLUSH_MS = Math.max(0, Number(process.env.AGENT_RELAY_TERMINAL_FLUSH_MS) || 6);
+const FLUSH_MAX_BYTES = Math.max(4096, Number(process.env.AGENT_RELAY_TERMINAL_FLUSH_MAX_BYTES) || 65536);
+const BACKPRESSURE_MAX_BYTES = Math.max(
+  1 << 20,
+  Number(process.env.AGENT_RELAY_TERMINAL_BACKPRESSURE_MAX_BYTES) || 8 << 20,
+);
+
+export interface TerminalStreamSubscriber {
+  onData(bytes: Uint8Array): void;
+  onClose(reason?: string): void;
+  // Bytes currently buffered on the subscriber's transport (e.g. WS bufferedAmount).
+  // When it exceeds the backpressure cap the subscriber is dropped and expected to
+  // reconnect and re-backfill — append streams can't drop bytes, so we shed the slow
+  // client instead of ballooning orchestrator memory.
+  bufferedAmount?(): number;
+}
+
+export interface TerminalStreamHandle {
+  backfill(cols?: number, rows?: number): TerminalSnapshot;
+  write(bytes: Uint8Array): void;
+  resize(cols: number, rows: number): void;
+  release(): void;
+}
+
+// --- Pure protocol helpers (unit-tested) ---
+
+// Decode a tmux control-mode `%output` payload: printable ASCII is literal, every
+// other byte is `\ooo` octal, and backslash is `\\`.
+export function decodeControlOutput(data: string): Uint8Array {
+  const out: number[] = [];
+  for (let i = 0; i < data.length; ) {
+    const ch = data[i]!;
+    if (ch === "\\") {
+      const next = data[i + 1];
+      if (next === "\\") {
+        out.push(0x5c);
+        i += 2;
+        continue;
+      }
+      let oct = "";
+      let j = i + 1;
+      while (j < data.length && oct.length < 3 && data[j]! >= "0" && data[j]! <= "7") {
+        oct += data[j];
+        j += 1;
+      }
+      if (oct.length > 0) {
+        out.push(parseInt(oct, 8) & 0xff);
+        i = j;
+        continue;
+      }
+      out.push(0x5c);
+      i += 1;
+      continue;
+    }
+    out.push(ch.charCodeAt(0) & 0xff);
+    i += 1;
+  }
+  return Uint8Array.from(out);
+}
+
+export type ControlLine =
+  | { type: "output"; pane: string; bytes: Uint8Array }
+  | { type: "exit"; reason?: string }
+  | { type: "other" };
+
+export function parseControlLine(line: string): ControlLine {
+  if (line.startsWith("%output ")) {
+    const rest = line.slice(8);
+    const sp = rest.indexOf(" ");
+    const pane = sp === -1 ? rest : rest.slice(0, sp);
+    const data = sp === -1 ? "" : rest.slice(sp + 1);
+    return { type: "output", pane, bytes: decodeControlOutput(data) };
+  }
+  if (line === "%exit" || line.startsWith("%exit ")) {
+    const reason = line.slice(5).trim();
+    return { type: "exit", ...(reason ? { reason } : {}) };
+  }
+  return { type: "other" };
+}
+
+// Encode raw input bytes for `send-keys -H` (space-separated hex octets).
+export function encodeSendKeysHex(bytes: Uint8Array): string {
+  return Array.from(bytes)
+    .map((b) => b.toString(16).padStart(2, "0"))
+    .join(" ");
+}
+
+// --- Shared session stream ---
+
+class SessionStream {
+  private readonly subscribers = new Set<TerminalStreamSubscriber>();
+  private proc: ReturnType<typeof Bun.spawn> | null = null;
+  private pending: Uint8Array[] = [];
+  private pendingBytes = 0;
+  private flushTimer: ReturnType<typeof setTimeout> | null = null;
+  private lineBuf = "";
+  private closed = false;
+
+  constructor(
+    private readonly session: string,
+    private readonly config: OrchestratorConfig,
+    private readonly onEmpty: () => void,
+  ) {}
+
+  private socket: string | undefined;
+
+  start(): void {
+    const socket = tmuxSocketForSession(this.session);
+    this.socket = socket;
+    // Pin the window size before attaching so the control client can't reflow the
+    // pane (default window-size would shrink it to the new client's 80x24).
+    const snapshot = this.safeBackfill();
+    Bun.spawnSync(tmuxCommand(socket, "set-window-option", "-t", this.session, "window-size", "manual"), {
+      stdin: "ignore",
+      stdout: "ignore",
+      stderr: "ignore",
+    });
+    if (snapshot?.cols && snapshot?.rows) {
+      Bun.spawnSync(
+        tmuxCommand(socket, "resize-window", "-t", this.session, "-x", String(snapshot.cols), "-y", String(snapshot.rows)),
+        { stdin: "ignore", stdout: "ignore", stderr: "ignore" },
+      );
+    }
+    try {
+      this.proc = Bun.spawn(tmuxCommand(socket, "-C", "attach-session", "-t", this.session), {
+        stdin: "pipe",
+        stdout: "pipe",
+        stderr: "ignore",
+      });
+    } catch (e) {
+      this.fail(e instanceof Error ? e.message : String(e));
+      return;
+    }
+    void this.readLoop();
+    void this.proc.exited.then(() => this.fail("terminal session ended"));
+  }
+
+  private async readLoop(): Promise<void> {
+    const proc = this.proc;
+    if (!proc?.stdout || typeof proc.stdout === "number") return;
+    const reader = (proc.stdout as ReadableStream<Uint8Array>).getReader();
+    const decoder = new TextDecoder("latin1");
+    try {
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        this.lineBuf += decoder.decode(value, { stream: true });
+        let nl: number;
+        while ((nl = this.lineBuf.indexOf("\n")) !== -1) {
+          const line = this.lineBuf.slice(0, nl).replace(/\r$/, "");
+          this.lineBuf = this.lineBuf.slice(nl + 1);
+          this.handleLine(line);
+        }
+      }
+    } catch {
+      // reader aborted on teardown
+    }
+    this.fail("terminal stream closed");
+  }
+
+  private handleLine(line: string): void {
+    const parsed = parseControlLine(line);
+    if (parsed.type === "output") {
+      this.enqueue(parsed.bytes);
+    } else if (parsed.type === "exit") {
+      this.fail(parsed.reason ? `tmux exit: ${parsed.reason}` : "tmux exit");
+    }
+  }
+
+  private enqueue(bytes: Uint8Array): void {
+    if (bytes.length === 0) return;
+    this.pending.push(bytes);
+    this.pendingBytes += bytes.length;
+    if (this.pendingBytes >= FLUSH_MAX_BYTES) {
+      this.flush();
+      return;
+    }
+    if (this.flushTimer === null) {
+      this.flushTimer = setTimeout(() => this.flush(), FLUSH_MS);
+    }
+  }
+
+  private flush(): void {
+    if (this.flushTimer !== null) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+    if (this.pendingBytes === 0) return;
+    const merged = new Uint8Array(this.pendingBytes);
+    let offset = 0;
+    for (const chunk of this.pending) {
+      merged.set(chunk, offset);
+      offset += chunk.length;
+    }
+    this.pending = [];
+    this.pendingBytes = 0;
+    for (const sub of [...this.subscribers]) {
+      if (sub.bufferedAmount && sub.bufferedAmount() > BACKPRESSURE_MAX_BYTES) {
+        this.removeSubscriber(sub);
+        try {
+          sub.onClose("backpressure");
+        } catch {}
+        continue;
+      }
+      try {
+        sub.onData(merged);
+      } catch {}
+    }
+  }
+
+  private safeBackfill(): TerminalSnapshot | null {
+    try {
+      return captureTerminal(this.session, this.config);
+    } catch {
+      return null;
+    }
+  }
+
+  // Capture the priming snapshot. When the viewer's dimensions are given, resize the
+  // tmux window to exactly that size *first* (synchronously) so the captured content
+  // wraps identically to the client's xterm — a width mismatch desyncs every
+  // absolute-positioned redraw the TUI emits and stacks ghost frames into scrollback.
+  backfill(cols?: number, rows?: number): TerminalSnapshot {
+    if (Number.isFinite(cols) && Number.isFinite(rows) && (cols as number) >= 10 && (rows as number) >= 5) {
+      Bun.spawnSync(
+        tmuxCommand(this.socket, "resize-window", "-t", this.session, "-x", String(Math.trunc(cols as number)), "-y", String(Math.trunc(rows as number))),
+        { stdin: "ignore", stdout: "ignore", stderr: "ignore" },
+      );
+    }
+    return captureTerminal(this.session, this.config);
+  }
+
+  write(bytes: Uint8Array): void {
+    if (this.closed || !this.proc || bytes.length === 0) return;
+    this.command(`send-keys -t "${this.session}" -H ${encodeSendKeysHex(bytes)}`);
+  }
+
+  resize(cols: number, rows: number): void {
+    if (this.closed || !this.proc) return;
+    if (!Number.isFinite(cols) || !Number.isFinite(rows) || cols < 10 || rows < 5) return;
+    this.command(`resize-window -t "${this.session}" -x ${Math.trunc(cols)} -y ${Math.trunc(rows)}`);
+  }
+
+  private command(line: string): void {
+    const stdin = this.proc?.stdin;
+    if (!stdin || typeof stdin === "number") return;
+    try {
+      (stdin as { write(data: string): void; flush?(): void }).write(`${line}\n`);
+      (stdin as { flush?(): void }).flush?.();
+    } catch {}
+  }
+
+  addSubscriber(sub: TerminalStreamSubscriber): void {
+    this.subscribers.add(sub);
+  }
+
+  removeSubscriber(sub: TerminalStreamSubscriber): void {
+    if (!this.subscribers.delete(sub)) return;
+    if (this.subscribers.size === 0) this.destroy();
+  }
+
+  private fail(reason: string): void {
+    if (this.closed) return;
+    const subs = [...this.subscribers];
+    this.subscribers.clear();
+    this.destroy();
+    for (const sub of subs) {
+      try {
+        sub.onClose(reason);
+      } catch {}
+    }
+  }
+
+  private destroy(): void {
+    if (this.closed) return;
+    this.closed = true;
+    if (this.flushTimer !== null) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+    try {
+      this.proc?.kill();
+    } catch {}
+    this.proc = null;
+    this.onEmpty();
+  }
+}
+
+const streams = new Map<string, SessionStream>();
+
+export function acquireTerminalStream(
+  session: string,
+  config: OrchestratorConfig,
+  subscriber: TerminalStreamSubscriber,
+): TerminalStreamHandle {
+  let stream = streams.get(session);
+  if (!stream) {
+    stream = new SessionStream(session, config, () => {
+      if (streams.get(session) === stream) streams.delete(session);
+    });
+    streams.set(session, stream);
+    stream.start();
+  }
+  const active = stream;
+  active.addSubscriber(subscriber);
+  return {
+    backfill: (cols, rows) => active.backfill(cols, rows),
+    write: (bytes) => active.write(bytes),
+    resize: (cols, rows) => active.resize(cols, rows),
+    release: () => active.removeSubscriber(subscriber),
+  };
+}
+
+export function activeTerminalStreamCount(): number {
+  return streams.size;
+}