@moondeck/station 0.0.0-stage → 1.0.0

package/lib/agents/machine-durable.ts

@@ -0,0 +1,604 @@
+import { StringDecoder } from "node:string_decoder";
+import { agentOutputSource } from "./output-source";
+/**
+ * ADR-121 + ADR-061: durable agent sessions for MACHINE-backed tasks.
+ *
+ * The container path's DurableProcess polls the session files on a host
+ * bind mount — a machine shares no filesystem with the host, so this
+ * variant speaks the same runner protocol over the environment transport:
+ *
+ *   outbox   one long-lived `tail -c +<offset+1> -F` streamed over ssh;
+ *            byte offset = initial + bytes received (append-only file).
+ *            A dropped ssh stream is respawned from the current offset.
+ *   inbox    chained one-shot appends (`cat >> inbox.jsonl` with stdin).
+ *   liveness periodic heartbeat mtime probe via exec; the runner beats
+ *            every 5s, so >25s of silence without an exit meta = dead.
+ *
+ * The runner itself (runner.mjs) is dependency-free Node and runs on the
+ * machine unchanged; it is delivered per-session via writeWorkspaceFile so
+ * it can never skew against this host's version.
+ */
+
+import { readFileSync } from "node:fs";
+import { posix } from "node:path";
+
+import type {
+  TaskEnvironmentHandle,
+  TaskEnvironmentProcess,
+} from "../task-environment/types";
+import { runnerScriptPath } from "./durable-proc";
+import type { ExitHandler, LineHandler } from "./proc";
+import type { LineTransport } from "./transport";
+
+/** Runner beats every 5 s; 25 s of silence without an exit meta = dead. */
+const HEARTBEAT_STALE_MS = 25_000;
+const HEARTBEAT_PROBE_MS = 10_000;
+const CLOSE_GRACE_MS = 8_000;
+const STEP_TIMEOUT_MS = 30_000;
+const STEP_OUTPUT_BYTES = 64 * 1024;
+const TAIL_RESPAWN_DELAY_MS = 1_000;
+
+interface RunnerMeta {
+  __uai: string;
+  code?: number | null;
+  stderrTail?: string;
+}
+
+/** The handle ops this transport needs beyond the session surface. */
+export type MachineSessionEnvironment = Pick<
+  TaskEnvironmentHandle<unknown>,
+  | "descriptor"
+  | "exec"
+  | "spawnSession"
+  | "writeWorkspaceFile"
+  | "launchDetachedSession"
+>;
+
+async function execStep(
+  environment: MachineSessionEnvironment,
+  argv: readonly [string, ...string[]],
+  stdin?: Uint8Array,
+): Promise<{ exitCode: number | null; stdout: string; stderr: string }> {
+  const result = await environment.exec({
+    argv,
+    projectEnvProfile: "maintenance",
+    ...(stdin ? { stdin } : {}),
+    timeoutMs: STEP_TIMEOUT_MS,
+    maxOutputBytes: STEP_OUTPUT_BYTES,
+  });
+  return {
+    exitCode: result.exitCode,
+    stdout: Buffer.from(result.stdout).toString("utf8"),
+    stderr: Buffer.from(result.stderr).toString("utf8"),
+  };
+}
+
+/** Append bytes to an environment file (creating parents), via exec stdin. */
+export async function appendEnvironmentFile(
+  environment: MachineSessionEnvironment,
+  path: string,
+  bytes: string,
+): Promise<void> {
+  const result = await execStep(
+    environment,
+    [
+      "/bin/sh",
+      "-c",
+      'mkdir -p "$(dirname "$1")" && cat >> "$1"',
+      "append",
+      path,
+    ],
+    Buffer.from(bytes),
+  );
+  if (result.exitCode !== 0) {
+    throw new Error(`environment append to ${path} exited ${result.exitCode}`);
+  }
+}
+
+/** Heartbeat freshness of an environment-side session dir (attach check). */
+export async function machineHeartbeatFresh(
+  environment: MachineSessionEnvironment,
+  sessionDir: string,
+  freshMs: number,
+): Promise<boolean> {
+  try {
+    const result = await execStep(environment, [
+      "/bin/sh",
+      "-c",
+      'stat -c %Y "$1/heartbeat"',
+      "stat",
+      sessionDir,
+    ]);
+    if (result.exitCode !== 0) return false;
+    const mtimeSeconds = Number(result.stdout.trim());
+    if (!Number.isFinite(mtimeSeconds)) return false;
+    return Date.now() - mtimeSeconds * 1000 < freshMs;
+  } catch {
+    return false;
+  }
+}
+
+export interface MachineDurableOptions {
+  environment: MachineSessionEnvironment;
+  /** Environment-absolute session dir (inside the workspace). */
+  sessionDir: string;
+  /** Resume consumption from this outbox byte offset (attach path). */
+  initialOutboxOffset?: number;
+  /** Adapter history is restored before tail delivery or checkpoint advance. */
+  ready?: Promise<void>;
+  onOffsetAdvance?: (offset: number) => void;
+  onCloseRequested?: () => void;
+  /** Provider-owned detached launch. Omit to attach to a live runner. */
+  launch?: () => Promise<{ exitCode: number | null; stderr: Uint8Array }>;
+  debugLabel?: string;
+}
+
+export class MachineDurableProcess implements LineTransport {
+  readonly durableSessionDir: string;
+  readonly #environment: MachineSessionEnvironment;
+  readonly #sessionDir: string;
+  readonly #outboxPath: string;
+  readonly #inboxPath: string;
+
+  readonly #lineHandlers = new Set<LineHandler>();
+  readonly #exitHandlers = new Set<ExitHandler>();
+
+  readonly #decoder = new StringDecoder("utf8");
+  #lineOffset = 0;
+  #offset: number;
+  #lineBuf = "";
+  #stderrBuf = "";
+  #closed = false;
+  #detached = false;
+  #sawSpawnMeta = false;
+  /** Completed-and-empty direct outbox probes past the spawn grace. */
+  #noMetaStrikes = 0;
+  readonly #startedAt = Date.now();
+  #tail: TaskEnvironmentProcess | null = null;
+  #tailGeneration = 0;
+  #heartbeatTimer: ReturnType<typeof setInterval> | null = null;
+  #closeTimer: ReturnType<typeof setTimeout> | null = null;
+  #respawnTimer: ReturnType<typeof setTimeout> | null = null;
+  #probing = false;
+  #inboxChain: Promise<void> = Promise.resolve();
+  readonly #onOffsetAdvance: ((offset: number) => void) | null;
+  readonly #onCloseRequested: (() => void) | null;
+  readonly #debug: string | null;
+
+  constructor(opts: MachineDurableOptions) {
+    this.durableSessionDir = opts.sessionDir;
+    this.#environment = opts.environment;
+    this.#sessionDir = opts.sessionDir;
+    this.#outboxPath = posix.join(opts.sessionDir, "outbox.jsonl");
+    this.#inboxPath = posix.join(opts.sessionDir, "inbox.jsonl");
+    this.#offset = opts.initialOutboxOffset ?? 0;
+    this.#lineOffset = this.#offset;
+    this.#onOffsetAdvance = opts.onOffsetAdvance ?? null;
+    this.#onCloseRequested = opts.onCloseRequested ?? null;
+    this.#debug =
+      opts.debugLabel && process.env.UAI_DEBUG_AGENTS ? opts.debugLabel : null;
+
+    if (opts.launch) {
+      void opts.launch().then(
+        (result) => {
+          const stderr = Buffer.from(result.stderr).toString("utf8");
+          if (stderr) this.#stderrBuf = (this.#stderrBuf + stderr).slice(-8192);
+          if (result.exitCode !== 0) {
+            this.#finish(null, `launch exited ${result.exitCode}`);
+          }
+        },
+        (error: unknown) => {
+          this.#stderrBuf = (
+            this.#stderrBuf +
+            (error instanceof Error ? error.message : String(error))
+          ).slice(-8192);
+          this.#finish(null, "launch threw");
+        },
+      );
+    } else {
+      this.#sawSpawnMeta = true; // attach: the runner pre-exists
+      // The container path pre-checks heartbeat freshness with a sync stat;
+      // over ssh that check is async, so attach optimistically and probe
+      // NOW — a dead runner finishes within a round-trip instead of a
+      // full probe interval, and the orchestrator respawns fresh.
+      void this.#probeLiveness();
+    }
+
+    const start = () => {
+      if (this.#closed || this.#detached) return;
+      void this.#spawnTail();
+      this.#heartbeatTimer = setInterval(() => {
+        void this.#probeLiveness();
+      }, HEARTBEAT_PROBE_MS);
+    };
+    if (opts.ready) {
+      this.#inboxChain = opts.ready;
+      void opts.ready.then(start, () => this.#finish(null, "protocol history unavailable"));
+    } else start();
+  }
+
+  #log(message: string): void {
+    console.error(`[uai-agent ${this.#debug}] ${message}`);
+  }
+
+  // ---- outbox tail over ssh ------------------------------------------------
+
+  async #spawnTail(): Promise<void> {
+    if (this.#closed || this.#detached) return;
+    const generation = ++this.#tailGeneration;
+    let tail: TaskEnvironmentProcess;
+    try {
+      // spawnSession, deliberately: a session request carries NO timeoutMs,
+      // and the streaming runner only arms its kill timer when one exists —
+      // the tail must live as long as the runner does.
+      tail = await this.#environment.spawnSession({
+        // -c +N is 1-based (start AT byte N): resume just past the consumed
+        // prefix. -F survives the outbox not existing yet; its chatter goes
+        // to stderr, never the byte stream.
+        argv: [
+          "/bin/sh",
+          "-c",
+          'exec tail -c +"$1" -F "$2" 2>/dev/null',
+          "tail",
+          String(this.#offset + 1),
+          this.#outboxPath,
+        ],
+        projectEnvProfile: "maintenance",
+        maxOutputBytes: 8 * 1024 * 1024,
+      });
+    } catch {
+      this.#scheduleTailRespawn(generation);
+      return;
+    }
+    if (this.#closed || this.#detached || generation !== this.#tailGeneration) {
+      await tail.terminate().catch(() => {});
+      return;
+    }
+    this.#tail = tail;
+    void (async () => {
+      try {
+        for await (const chunk of tail.stdout) {
+          if (this.#closed || this.#detached) break;
+          if (generation !== this.#tailGeneration) break;
+          const text = Buffer.from(chunk);
+          this.#offset += text.byteLength;
+          this.#lineBuf += this.#decoder.write(text);
+          this.#drainLines();
+          if (this.#onOffsetAdvance && !this.#closed) {
+            try {
+              this.#onOffsetAdvance(this.#lineOffset);
+            } catch {
+              // Persistence hiccup — the next batch retries larger.
+            }
+          }
+        }
+      } catch {
+        // Stream error — the respawn below re-establishes the tail.
+      }
+      // The ssh stream ended (drop, machine restart, terminate). If the
+      // session is still live, come back from the current offset.
+      if (!this.#closed && !this.#detached && generation === this.#tailGeneration) {
+        this.#scheduleTailRespawn(generation);
+      }
+    })();
+  }
+
+  #scheduleTailRespawn(generation: number): void {
+    if (this.#closed || this.#detached) return;
+    if (generation !== this.#tailGeneration) return;
+    if (this.#respawnTimer) return;
+    this.#respawnTimer = setTimeout(() => {
+      this.#respawnTimer = null;
+      void this.#spawnTail();
+    }, TAIL_RESPAWN_DELAY_MS);
+  }
+
+  #drainLines(): void {
+    let nl: number;
+    while ((nl = this.#lineBuf.indexOf("\n")) >= 0) {
+      const rawLine = this.#lineBuf.slice(0, nl);
+      this.#lineOffset += Buffer.byteLength(rawLine) + 1;
+      const line = rawLine.trim();
+      this.#lineBuf = this.#lineBuf.slice(nl + 1);
+      if (line.length === 0) continue;
+      if (line.startsWith('{"__uai"')) {
+        this.#handleMeta(line);
+        continue;
+      }
+      if (this.#debug) this.#log(`<- ${line.slice(0, 1000)}`);
+      for (const handler of this.#lineHandlers) {
+        try {
+          agentOutputSource.run(this.durableSessionDir + ":" + this.#lineOffset, () => handler(line));
+        } catch {
+          // A broken handler must not wedge the tail loop.
+        }
+      }
+    }
+  }
+
+  #handleMeta(line: string): void {
+    let meta: RunnerMeta;
+    try {
+      meta = JSON.parse(line) as RunnerMeta;
+    } catch {
+      return;
+    }
+    if (meta.__uai === "spawn") {
+      this.#sawSpawnMeta = true;
+    } else if (meta.__uai === "exit") {
+      if (typeof meta.stderrTail === "string" && meta.stderrTail) {
+        this.#stderrBuf = meta.stderrTail.slice(-8192);
+      }
+      if (this.#debug) this.#log(`exit meta: code ${meta.code ?? null}`);
+      this.#finish(
+        typeof meta.code === "number" ? meta.code : null,
+        "runner exit meta",
+      );
+    }
+  }
+
+  // ---- liveness ------------------------------------------------------------
+
+  async #probeLiveness(): Promise<void> {
+    if (this.#closed || this.#detached || this.#probing) return;
+    this.#probing = true;
+    try {
+      if (!this.#sawSpawnMeta) {
+        if (Date.now() - this.#startedAt <= HEARTBEAT_STALE_MS) return;
+        // Grace elapsed on OUR clock — but the spawn meta arrives via the
+        // tail, an ssh child of THIS host, and both starve together when the
+        // host CPU is pegged (live 2026-08-29: a 12-task launch burst pushed
+        // load to 16, tails lagged, and healthy runners were declared dead —
+        // then respawned, ORPHANING the originals). Before any verdict, ask
+        // the machine directly: the runner writes its spawn meta as the
+        // outbox's first line.
+        let sawOnMachine = false;
+        try {
+          const probe = await execStep(this.#environment, [
+            "/bin/sh",
+            "-c",
+            'head -n1 "$1/outbox.jsonl" 2>/dev/null | grep -c __uai',
+            "probe",
+            this.#sessionDir,
+          ]);
+          sawOnMachine =
+            probe.exitCode === 0 && Number(probe.stdout.trim()) > 0;
+        } catch {
+          // Unreachable machine: counts as a strike below.
+        }
+        if (this.#closed || this.#detached) return;
+        if (sawOnMachine) {
+          this.#sawSpawnMeta = true;
+          return;
+        }
+        // Two COMPLETED empty probes past the grace — not elapsed time — are
+        // the death sentence; probes that never ran prove nothing.
+        this.#noMetaStrikes += 1;
+        if (this.#noMetaStrikes < 2) return;
+        await this.#killRemoteRunner();
+        if (this.#closed || this.#detached) return;
+        this.#finish(null, "no spawn meta within grace");
+        return;
+      }
+      const fresh = await machineHeartbeatFresh(
+        this.#environment,
+        this.#sessionDir,
+        HEARTBEAT_STALE_MS,
+      );
+      if (this.#closed || this.#detached) return;
+      if (!fresh) {
+        // A stale-but-alive runner would be doubled by the respawn; the
+        // single-runner-per-session invariant is enforced here, not hoped for.
+        await this.#killRemoteRunner();
+        if (this.#closed || this.#detached) return;
+        this.#finish(null, "heartbeat stale");
+      }
+    } finally {
+      this.#probing = false;
+    }
+  }
+
+  /** Best-effort remote kill of this session's runner before declaring it
+   * dead: a wrong verdict must cost a restart, never an orphaned agent.
+   * `runner[.]mjs` so the kill-shell's own argv (which carries this script
+   * text) can never self-match — the pgrep/pkill self-match class. */
+  async #killRemoteRunner(): Promise<void> {
+    try {
+      await execStep(this.#environment, [
+        "/bin/sh",
+        "-c",
+        'pkill -f "runner[.]mjs $1" >/dev/null 2>&1; true',
+        "kill",
+        this.#sessionDir,
+      ]);
+    } catch {
+      // The finish verdict stands either way; the orphan sweep in recovery
+      // remains the backstop for an unreachable machine.
+    }
+  }
+
+  #finish(code: number | null, reason = "unspecified"): void {
+    if (this.#closed) return;
+    // Session deaths are load-bearing and were undiagnosable without this
+    // (live 2026-08-27: a 65s recycle loop with no trace). Always log.
+    console.warn(
+      `[machine-durable] session ${this.#sessionDir} finished (code ${code ?? "null"}): ${reason}`,
+    );
+    this.#closed = true;
+    this.#stopTimers();
+    const tail = this.#tail;
+    this.#tail = null;
+    if (tail) void tail.terminate().catch(() => {});
+    for (const handler of this.#exitHandlers) handler(code);
+  }
+
+  #stopTimers(): void {
+    if (this.#heartbeatTimer) clearInterval(this.#heartbeatTimer);
+    if (this.#closeTimer) clearTimeout(this.#closeTimer);
+    if (this.#respawnTimer) clearTimeout(this.#respawnTimer);
+    this.#heartbeatTimer = null;
+    this.#closeTimer = null;
+    this.#respawnTimer = null;
+  }
+
+  // ---- LineProcess-compatible surface ---------------------------------------
+
+  onLine(handler: LineHandler): void {
+    this.#lineHandlers.add(handler);
+  }
+
+  onExit(handler: ExitHandler): void {
+    this.#exitHandlers.add(handler);
+  }
+
+  /** `__uai` MUST stay the first key: both runner and host filter meta lines
+   * with a byte-prefix check, not a parse (see DurableProcess.writeLine). */
+  writeLine(value: unknown): void {
+    if (this.#closed || this.#detached) return;
+    const json = JSON.stringify(value);
+    const inputMeta = JSON.stringify({ __uai: "input", ts: Date.now() });
+    if (this.#debug) this.#log(`-> ${json.slice(0, 1000)}`);
+    this.#inboxChain = this.#inboxChain
+      .then(() =>
+        this.#closed || this.#detached ? undefined : appendEnvironmentFile(
+          this.#environment,
+          this.#inboxPath,
+          `${inputMeta}\n${json}\n`,
+        ),
+      )
+      .catch(() => {
+        if (this.#closed || this.#detached) return;
+        // A fresh heartbeat proves the runner lives, not that it received the
+        // message. Fail immediately through the adapter's normal error path.
+        // Never include the frame or exec stderr: either may contain secrets.
+        this.#stderrBuf = "Could not deliver the message to the agent because its input could not be written. Please try again.";
+        this.#finish(null, "inbox append failed");
+        void this.#killRemoteRunner();
+      });
+  }
+
+  get stderrTail(): string {
+    return this.#stderrBuf;
+  }
+
+  get isClosed(): boolean {
+    return this.#closed;
+  }
+
+  get outboxOffset(): number {
+    return this.#lineOffset;
+  }
+
+  /** Stop consuming WITHOUT touching the runner (host restart in miniature). */
+  detach(): void {
+    if (this.#detached || this.#closed) return;
+    this.#detached = true;
+    this.#stopTimers();
+    const tail = this.#tail;
+    this.#tail = null;
+    if (tail) void tail.terminate().catch(() => {});
+  }
+
+  /** Real teardown: ask the runner to stop its CLI, then observe the exit. */
+  async close(): Promise<void> {
+    if (this.#closed || this.#detached) return;
+    try {
+      this.#onCloseRequested?.();
+    } catch {
+      // Bookkeeping only.
+    }
+    try {
+      await this.#inboxChain;
+      await appendEnvironmentFile(
+        this.#environment,
+        this.#inboxPath,
+        '{"__uai":"stop"}\n',
+      );
+    } catch {
+      // Inbox unwritable — fall through to the grace timer.
+    }
+    this.#closeTimer = setTimeout(
+      () => this.#finish(null, "close grace elapsed"),
+      CLOSE_GRACE_MS,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Session-dir plumbing shared with the transport layer.
+// ---------------------------------------------------------------------------
+
+export function machineSessionDir(
+  workspacePath: string,
+  agentId: string,
+): string {
+  return posix.join(
+    workspacePath,
+    ".moondeck",
+    "sessions",
+    agentId,
+    `${String(Date.now())}-${process.pid}`,
+  );
+}
+
+/** Deliver this host's runner into the environment session dir. */
+export async function deployRunner(
+  environment: MachineSessionEnvironment,
+  sessionDir: string,
+): Promise<string> {
+  const runnerPath = posix.join(sessionDir, "runner.mjs");
+  await environment.writeWorkspaceFile(
+    runnerPath,
+    readFileSync(runnerScriptPath()),
+  );
+  return runnerPath;
+}
+
+/** Environment-side `current` pointer (agents read the path from their
+ * preamble). Best-effort: failure must never kill a session. */
+export async function publishMachineCurrentSession(
+  environment: MachineSessionEnvironment,
+  sessionDir: string,
+): Promise<void> {
+  try {
+    await execStep(environment, [
+      "/bin/sh",
+      "-c",
+      'ln -sfn "$(basename "$1")" "$(dirname "$1")/current"',
+      "publish",
+      sessionDir,
+    ]);
+  } catch {
+    // Diagnostic pointer only.
+  }
+}
+
+/** Ask a (possibly dead) predecessor runner to stop — same gesture as the
+ * host-FS append, over the transport, and just as harmless when it's gone. */
+export async function stopMachinePredecessor(
+  environment: MachineSessionEnvironment,
+  sessionDir: string,
+): Promise<void> {
+  try {
+    await appendEnvironmentFile(
+      environment,
+      posix.join(sessionDir, "inbox.jsonl"),
+      '{"__uai":"stop"}\n',
+    );
+  } catch {
+    // Dir gone / runner dead — nothing to stop.
+  }
+}
+
+/** Removal/recovery variant: failure is not success because no later attach
+ * will provide the transport layer's usual predecessor-stop backstop. */
+export async function stopMachinePredecessorStrict(
+  environment: MachineSessionEnvironment,
+  sessionDir: string,
+): Promise<void> {
+  await appendEnvironmentFile(
+    environment,
+    posix.join(sessionDir, "inbox.jsonl"),
+    '{"__uai":"stop"}\n',
+  );
+}