@parall/codex-agent 1.18.0

package/src/dispatch.ts

@@ -0,0 +1,559 @@
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+import { execSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type {
+  CleanupForkOpts,
+  DispatchAdapter,
+  DispatchOpts,
+  ForkOpts,
+  ForkSessionHandle,
+  GatewayLogger,
+  RuntimeEvent,
+} from "@parall/agent-core";
+import type { CodexAgentConfig } from "./config.js";
+import { normalizeApprovalPolicy, normalizeSandbox } from "./config.js";
+import { EventMapper } from "./event-mapping.js";
+import { JsonRpcStdioClient } from "./jsonrpc-client.js";
+import type { CodexSessionManager } from "./session-manager.js";
+
+type CodexAppServerAdapterOptions = Pick<
+  CodexAgentConfig,
+  | "approvalPolicy"
+  | "codexBin"
+  | "codexHome"
+  | "model"
+  | "reasoningEffort"
+  | "sandbox"
+  | "workspaceDir"
+> & {
+  sessionManager: CodexSessionManager;
+  log?: GatewayLogger;
+};
+
+type TurnEventEnvelope =
+  | { kind: "runtime"; event: RuntimeEvent }
+  | { kind: "turn_end"; threadId?: string }
+  | { kind: "error"; message: string };
+
+/**
+ * Bridge driver backed by `codex app-server --listen stdio://`.
+ *
+ * Lifecycle:
+ *   - bridge startup: spawn one `codex app-server` subprocess, do the
+ *     `initialize` / `initialized` handshake once, keep the pipe open
+ *   - per Parall dispatch: `thread/start` (first time) or `thread/resume`,
+ *     then `turn/start`; forward server notifications until `turn/completed`
+ *   - fork-on-busy: `thread/fork` creates a disposable sibling thread
+ *
+ * Concurrency: agent-core routes one dispatch per sessionKey at a time, so
+ * main + fork can interleave turns on the same stdio pipe. We route
+ * notifications by threadId the server stamps on every item/turn event.
+ */
+export class CodexAppServerAdapter implements DispatchAdapter {
+  private client: JsonRpcStdioClient | null = null;
+  private proc: ChildProcessWithoutNullStreams | null = null;
+  private initialized = false;
+  private startPromise: Promise<void> | null = null;
+  private readonly activeTurns = new Map<string, TurnSink>();
+  private readonly resumedThreadIds = new Set<string>();
+  private stopping = false;
+
+  /**
+   * Store an active turn sink keyed by threadId. If a sink already exists for
+   * the same threadId, log a warning and fail the existing sink — this
+   * shouldn't happen in practice (agent-core serialises per sessionKey, and
+   * thread/fork returns a distinct id) but the failure mode of a silent
+   * replacement would be a perpetually hung dispatch generator, which is
+   * hard to debug.
+   */
+  private setActiveTurn(threadId: string, sink: TurnSink, log?: GatewayLogger): void {
+    const existing = this.activeTurns.get(threadId);
+    if (existing) {
+      (log ?? this.opts.log)?.warn?.(
+        `codex-agent: thread ${threadId} already had an active turn; failing the previous dispatch`,
+      );
+      existing.push({ kind: "error", message: `thread ${threadId} replaced by concurrent turn` });
+      existing.close();
+    }
+    this.activeTurns.set(threadId, sink);
+  }
+
+  constructor(private readonly opts: CodexAppServerAdapterOptions) {}
+
+  async *dispatch({ bodyForAgent, sessionKey, context }: DispatchOpts): AsyncIterable<RuntimeEvent> {
+    await this.ensureStarted(context.log);
+    // After ensureStarted resolves the subprocess could still die before we
+    // capture the client (handleSubprocessClose nulls this.client). Yield a
+    // clean error event instead of relying on a non-null assertion that would
+    // throw a TypeError on the next sendRequest call.
+    const client = this.client;
+    if (!client) {
+      yield { type: "error", message: "Codex app-server not available (subprocess died during dispatch start)" };
+      return;
+    }
+    const log = this.opts.log ?? context.log;
+    const isMainSession = this.opts.sessionManager.isMain(sessionKey);
+
+    let threadId = this.opts.sessionManager.getThreadId(sessionKey);
+    if (!threadId) {
+      try {
+        threadId = await this.openThread(client, { resumeId: undefined });
+        this.opts.sessionManager.recordThreadId(sessionKey, threadId);
+        // Mark this freshly-started thread as already live in the current
+        // app-server process. Without this, the second dispatch after a cold
+        // start would enter the thread/resume branch below for a thread this
+        // process just created — and if the server treats resume-of-just-
+        // created-thread as an attach-to-detached-thread operation, it could
+        // fail the resume path and replace the thread, losing the first turn.
+        this.resumedThreadIds.add(threadId);
+      } catch (err) {
+        yield { type: "error", message: `Codex thread/start failed: ${errToString(err)}` };
+        return;
+      }
+    } else if (isMainSession && !this.resumedThreadIds.has(threadId)) {
+      // We have a persisted threadId from a previous bridge run — resume it.
+      // Mirror the deferred-clear pattern from the turn/start retry below: only
+      // discard the persisted id once a fresh-thread start has actually
+      // succeeded, so a transient resume failure (network/timeout/upstream
+      // hiccup) doesn't permanently throw away the prior conversation context.
+      try {
+        threadId = await this.openThread(client, { resumeId: threadId });
+        this.opts.sessionManager.recordThreadId(sessionKey, threadId);
+        this.resumedThreadIds.add(threadId);
+      } catch (err) {
+        log?.warn?.(`codex-agent: thread/resume failed (${errToString(err)}); attempting one-shot fresh-thread start`);
+        let freshThreadId: string;
+        try {
+          freshThreadId = await this.openThread(client, { resumeId: undefined });
+        } catch (innerErr) {
+          yield {
+            type: "error",
+            message: `Codex thread/start failed after resume error (persisted thread retained): ${errToString(innerErr)}`,
+          };
+          return;
+        }
+        this.opts.sessionManager.clearMainThread();
+        this.opts.sessionManager.recordThreadId(sessionKey, freshThreadId);
+        this.resumedThreadIds.add(freshThreadId);
+        threadId = freshThreadId;
+      }
+    }
+
+    const sink = new TurnSink();
+    this.setActiveTurn(threadId, sink, log);
+    const groupKey = randomUUID();
+    let sawTurnEnd = false;
+
+    try {
+      // Start the turn. If the stored threadId is dead, we retry once with a fresh thread.
+      let turnStartResult: unknown;
+      try {
+        turnStartResult = await client.sendRequest("turn/start", {
+          threadId,
+          input: buildTurnInput(bodyForAgent),
+        });
+      } catch (err) {
+        const message = errToString(err);
+        // For the main session we attempt one fresh-thread retry. Codex CLI's
+        // wording for stale threads shifts across versions (not found / unknown
+        // / invalid / expired / gone / ...), so we don't gate the retry on a
+        // regex. But we only commit to discarding the persisted threadId after
+        // the retry succeeds — if the failure was transient (network, timeout,
+        // upstream model hiccup), keeping the old id lets the next dispatch
+        // resume normally instead of permanently losing conversation context.
+        // Fork sessions don't retry: agent-core spawns a fresh fork next trigger.
+        if (!isMainSession) {
+          yield { type: "error", message: `Codex turn/start failed: ${message}` };
+          return;
+        }
+        log?.warn?.(`codex-agent: turn/start on thread ${threadId} failed (${message}); attempting one-shot fresh-thread retry`);
+        this.activeTurns.delete(threadId);
+        let freshThreadId: string;
+        try {
+          freshThreadId = await this.openThread(client, { resumeId: undefined });
+        } catch (createErr) {
+          yield { type: "error", message: `Codex turn/start failed; could not create replacement thread: ${errToString(createErr)}` };
+          return;
+        }
+        this.setActiveTurn(freshThreadId, sink, log);
+        try {
+          turnStartResult = await client.sendRequest("turn/start", {
+            threadId: freshThreadId,
+            input: buildTurnInput(bodyForAgent),
+          });
+        } catch (retryErr) {
+          // Retry also failed — likely transient or systemic, not a stale-thread
+          // issue. Don't clobber the persisted threadId; next dispatch will try
+          // resume again.
+          this.activeTurns.delete(freshThreadId);
+          yield { type: "error", message: `Codex turn/start failed after retry: ${errToString(retryErr)}` };
+          return;
+        }
+        // Retry accepted — the original thread really was unusable. Now safe
+        // to discard the old persisted id and persist the fresh one.
+        this.opts.sessionManager.clearMainThread();
+        this.opts.sessionManager.recordThreadId(sessionKey, freshThreadId);
+        this.resumedThreadIds.add(freshThreadId);
+        threadId = freshThreadId;
+      }
+
+      void turnStartResult; // turnId comes back here but isn't needed — we key by threadId
+
+      while (true) {
+        const envelope = await sink.next();
+        if (envelope.kind === "turn_end") {
+          sawTurnEnd = true;
+          break;
+        }
+        if (envelope.kind === "error") {
+          yield { type: "error", message: envelope.message };
+          continue;
+        }
+        const event = envelope.event;
+        if (event.type === "error") {
+          yield event;
+          continue;
+        }
+        if (event.type === "text") {
+          // Layer 0 symmetric output contract: Codex's plain text is never
+          // projected as a chat message. Outbound messages must come from the
+          // agent explicitly invoking `@parall/cli messages send` / `dm` via
+          // the shell/exec tool. Text events are still yielded so agent-core
+          // records them as suppressed session steps for audit.
+          yield { ...event, project: false, groupKey };
+          continue;
+        }
+        yield { ...event, groupKey };
+      }
+    } finally {
+      this.activeTurns.delete(threadId!);
+      if (!sawTurnEnd) {
+        // Defensive: if we bailed early, make sure we leave no dangling sink.
+        sink.close();
+      }
+    }
+  }
+
+  async forkSession({ sessionKey: parentSessionKey }: ForkOpts): Promise<ForkSessionHandle | null> {
+    const client = this.client;
+    if (!client) return null;
+    const parentThreadId = this.opts.sessionManager.getThreadId(parentSessionKey);
+    if (!parentThreadId) return null;
+    const handle = this.opts.sessionManager.createForkSessionKey();
+    try {
+      const result = await client.sendRequest("thread/fork", {
+        threadId: parentThreadId,
+        ephemeral: true,
+      });
+      const forkedThreadId = extractThreadId(result);
+      if (!forkedThreadId) {
+        this.opts.log?.warn?.("codex-agent: thread/fork returned no thread id");
+        return null;
+      }
+      this.opts.sessionManager.recordThreadId(handle.sessionKey, forkedThreadId);
+      return handle;
+    } catch (err) {
+      return this.logForkFailure(err);
+    }
+  }
+
+  cleanupFork({ fork }: CleanupForkOpts) {
+    this.opts.sessionManager.cleanupFork(fork.sessionKey);
+  }
+
+  private logForkFailure(err: unknown): null {
+    this.opts.log?.warn?.(`codex-agent: thread/fork failed: ${errToString(err)}`);
+    return null;
+  }
+
+  async stop() {
+    this.stopping = true;
+    const proc = this.proc;
+    const client = this.client;
+    this.proc = null;
+    this.client = null;
+    this.initialized = false;
+    if (client) client.dispose(new Error("adapter stopped"));
+    if (proc && proc.exitCode === null && proc.signalCode === null) {
+      proc.kill("SIGTERM");
+    }
+  }
+
+  private async ensureStarted(log?: GatewayLogger): Promise<void> {
+    if (this.initialized && this.client) return;
+    if (this.startPromise) return this.startPromise;
+    this.startPromise = this.doStart(log);
+    try {
+      await this.startPromise;
+    } finally {
+      // Keep startPromise set only while actively starting; reset once resolved
+      // (or rejected) so the next ensureStarted() after a subprocess death can
+      // restart cleanly.
+      this.startPromise = null;
+    }
+  }
+
+  private async doStart(log?: GatewayLogger): Promise<void> {
+    // Reset graceful-stop flag in case the adapter is being restarted after a
+    // previous stop() — otherwise handleSubprocessClose would mislabel the next
+    // unexpected exit as "during graceful stop".
+    this.stopping = false;
+    ensureGitRepo(this.opts.workspaceDir);
+
+    // Only steer Codex's own state via CODEX_HOME. Leaving HOME untouched
+    // preserves the user's real dotfiles for any subprocess Codex spawns
+    // (git/ssh/npm/etc.). Earlier versions also rewrote HOME, which broke
+    // tooling for users who set a custom CODEX_HOME pointing somewhere
+    // other than their shell home.
+    const env = {
+      ...process.env,
+      CODEX_HOME: this.opts.codexHome,
+      FORCE_COLOR: "0",
+      NO_COLOR: "1",
+    };
+    const args = ["app-server", "--listen", "stdio://"];
+    (log ?? this.opts.log)?.info?.(`codex-agent: spawning ${this.opts.codexBin} ${args.join(" ")}`);
+    const proc = spawn(this.opts.codexBin, args, {
+      cwd: this.opts.workspaceDir,
+      env,
+      stdio: ["pipe", "pipe", "pipe"],
+    }) as ChildProcessWithoutNullStreams;
+
+    proc.stderr.setEncoding("utf8");
+    proc.stderr.on("data", (chunk: string) => {
+      (log ?? this.opts.log)?.warn?.(`codex-agent[stderr]: ${chunk.trim()}`);
+    });
+
+    // Don't publish proc/client until the initialize handshake succeeds. If
+    // initialize rejects (timeout, malformed handshake, app-server crash on
+    // start), we kill the half-spawned subprocess and clear local refs here
+    // — leaving them on `this` would leak the orphan process and cause the
+    // next ensureStarted() to spawn a second app-server on top of it.
+    const client = new JsonRpcStdioClient(proc);
+    client.setNotificationHandler((method, params) => this.routeNotification(method, params));
+
+    proc.once("close", (code, signal) => this.handleSubprocessClose(proc, code, signal, log));
+    proc.once("error", (err) => this.handleSubprocessClose(proc, null, null, log, err));
+
+    try {
+      await client.sendRequest("initialize", {
+        clientInfo: { name: "parall-codex-agent", version: "1" },
+        capabilities: { experimentalApi: false },
+      });
+      client.sendNotification("initialized", {});
+    } catch (err) {
+      client.dispose(err instanceof Error ? err : new Error(String(err)));
+      if (proc.exitCode === null && proc.signalCode === null) proc.kill("SIGTERM");
+      throw err;
+    }
+
+    this.proc = proc;
+    this.client = client;
+    this.initialized = true;
+  }
+
+  private handleSubprocessClose(
+    proc: ChildProcessWithoutNullStreams,
+    code: number | null,
+    signal: NodeJS.Signals | null,
+    log?: GatewayLogger,
+    err?: Error,
+  ): void {
+    // Ignore close/error callbacks from a stale subprocess instance: the
+    // adapter may already have spawned a fresh app-server (e.g. after a
+    // recovered handshake failure), and we don't want a dying old child to
+    // tear down the new healthy one.
+    if (this.proc !== null && this.proc !== proc) return;
+    const reason = err
+      ? `spawn error: ${err.message}`
+      : `exited (code=${code ?? "null"}${signal ? `, signal=${signal}` : ""})`;
+    const logger = log ?? this.opts.log;
+    if (this.stopping) {
+      logger?.info?.(`codex-agent: app-server subprocess ${reason} during graceful stop`);
+    } else {
+      logger?.warn?.(`codex-agent: app-server subprocess ${reason}; resetting adapter`);
+    }
+    this.client?.dispose(err ?? new Error(`app-server ${reason}`));
+    for (const activeSink of this.activeTurns.values()) {
+      activeSink.push({ kind: "error", message: `Codex app-server ${reason}` });
+      activeSink.close();
+    }
+    this.activeTurns.clear();
+    this.resumedThreadIds.clear();
+    this.client = null;
+    this.proc = null;
+    this.initialized = false;
+  }
+
+  private async openThread(
+    client: JsonRpcStdioClient,
+    opts: { resumeId: string | undefined },
+  ): Promise<string> {
+    // Per the app-server protocol, `thread/resume` accepts the same
+    // configuration overrides as `thread/start` (sandbox, approvalPolicy,
+    // model, reasoningEffort). Passing them on resume lets the user change
+    // PRLL_CODEX_* env vars and have the bridge pick them up on the next
+    // restart instead of being stuck on the values baked into the persisted
+    // thread. Sandbox / approval values are normalised to the camelCase enum
+    // the protocol expects (`workspaceWrite`, `onRequest`, etc.), so users
+    // can supply either CLI-style or protocol-style env values.
+    const commonParams: Record<string, unknown> = {
+      approvalPolicy: normalizeApprovalPolicy(this.opts.approvalPolicy),
+      sandbox: normalizeSandbox(this.opts.sandbox),
+    };
+    if (this.opts.model) commonParams.model = this.opts.model;
+    if (this.opts.reasoningEffort) {
+      // The app-server JSON-RPC surface uses camelCase for overrides —
+      // `modelReasoningEffort` parallels `approvalPolicy` / `sandbox` on the
+      // top-level params. The corresponding config.toml key is
+      // `model_reasoning_effort` (snake_case), but the nested `config` on
+      // `thread/start` / `thread/resume` takes the camelCase form.
+      commonParams.config = { modelReasoningEffort: this.opts.reasoningEffort };
+    }
+
+    const method = opts.resumeId ? "thread/resume" : "thread/start";
+    const params: Record<string, unknown> = opts.resumeId
+      ? { threadId: opts.resumeId, ...commonParams }
+      : { cwd: this.opts.workspaceDir, ...commonParams };
+
+    const result = await client.sendRequest(method, params);
+    const threadId = extractThreadId(result) ?? opts.resumeId;
+    if (!threadId) {
+      throw new Error(`${method} returned no thread id`);
+    }
+    return threadId;
+  }
+
+  private routeNotification(method: string, params: unknown) {
+    const threadId = extractThreadIdFromNotification(params);
+    if (!threadId) {
+      // Surface server-initiated generic errors to every active turn.
+      if (method === "error") {
+        const msg = (params as { message?: unknown })?.message ?? "Codex app-server error";
+        for (const sink of this.activeTurns.values()) {
+          sink.push({ kind: "error", message: String(msg) });
+        }
+      }
+      return;
+    }
+
+    const sink = this.activeTurns.get(threadId);
+    if (!sink) return;
+
+    // For turn/completed we still run the mapper first — it emits a
+    // RuntimeEvent error if `turn.status === "failed"`. Enqueue those events
+    // before the turn_end sentinel so the dispatch loop can yield them.
+    for (const event of sink.mapper.map(method, params)) {
+      sink.push({ kind: "runtime", event });
+    }
+    if (method === "turn/completed") {
+      sink.push({ kind: "turn_end", threadId });
+    }
+  }
+
+}
+
+/** Per-turn buffered sink backed by an unbounded promise queue. */
+class TurnSink {
+  readonly mapper = new EventMapper();
+  private readonly queue: TurnEventEnvelope[] = [];
+  private resolver: ((value: TurnEventEnvelope) => void) | null = null;
+  private closed = false;
+
+  push(envelope: TurnEventEnvelope) {
+    if (this.closed) return;
+    if (this.resolver) {
+      const r = this.resolver;
+      this.resolver = null;
+      r(envelope);
+      return;
+    }
+    this.queue.push(envelope);
+  }
+
+  next(): Promise<TurnEventEnvelope> {
+    // Drain any queued envelopes first, even after close(). Otherwise a final
+    // error envelope enqueued right before close() (e.g. by
+    // handleSubprocessClose) is silently dropped because the consumer would
+    // see turn_end before it.
+    const pending = this.queue.shift();
+    if (pending) return Promise.resolve(pending);
+    if (this.closed) {
+      return Promise.resolve({ kind: "turn_end" });
+    }
+    return new Promise((resolve) => {
+      this.resolver = resolve;
+    });
+  }
+
+  close() {
+    this.closed = true;
+    const r = this.resolver;
+    this.resolver = null;
+    r?.({ kind: "turn_end" });
+  }
+}
+
+/**
+ * Codex app-server's `turn/start` expects `input` as an array of content
+ * items (each `{ type: "text", text: "..." }` or similar), not a raw string.
+ * Wrap the event body so the protocol contract is honoured — sending a bare
+ * string has worked historically via undocumented coercion but isn't stable.
+ */
+function buildTurnInput(body: string): Array<{ type: "text"; text: string }> {
+  return [{ type: "text", text: body }];
+}
+
+function ensureGitRepo(workingDirectory: string): void {
+  fs.mkdirSync(workingDirectory, { recursive: true });
+  // Only `git init` if the workspace isn't already inside any git repo. A
+  // bare existsSync(.git) check would miss the common case of a user pointing
+  // PRLL_CODEX_WORKSPACE_DIR at a subdirectory of their existing project,
+  // and silently creating a nested repo there would mangle their layout.
+  try {
+    execSync("git rev-parse --is-inside-work-tree", { cwd: workingDirectory, stdio: "pipe" });
+    return;
+  } catch {
+    // Not inside a repo — fall through to init.
+  }
+  const env = {
+    ...process.env,
+    GIT_AUTHOR_NAME: "parall-codex-agent",
+    GIT_AUTHOR_EMAIL: "agent@parall.local",
+    GIT_COMMITTER_NAME: "parall-codex-agent",
+    GIT_COMMITTER_EMAIL: "agent@parall.local",
+  };
+  try {
+    execSync("git init", { cwd: workingDirectory, stdio: "pipe", env });
+    execSync("git commit --allow-empty -m init", { cwd: workingDirectory, stdio: "pipe", env });
+  } catch {
+    // Non-fatal: codex app-server may still accept a bare directory. Let it raise at turn time.
+  }
+}
+
+function extractThreadId(result: unknown): string | undefined {
+  if (!result || typeof result !== "object") return undefined;
+  const r = result as Record<string, unknown>;
+  if (typeof r.threadId === "string") return r.threadId;
+  const thread = r.thread as Record<string, unknown> | undefined;
+  if (thread && typeof thread.id === "string") return thread.id;
+  return undefined;
+}
+
+function extractThreadIdFromNotification(params: unknown): string | undefined {
+  if (!params || typeof params !== "object") return undefined;
+  const p = params as Record<string, unknown>;
+  if (typeof p.threadId === "string") return p.threadId;
+  const thread = p.thread as Record<string, unknown> | undefined;
+  if (thread && typeof thread.id === "string") return thread.id;
+  const meta = (p._meta ?? p.meta) as Record<string, unknown> | undefined;
+  if (meta && typeof meta.threadId === "string") return meta.threadId;
+  return undefined;
+}
+
+function errToString(err: unknown): string {
+  if (err instanceof Error) return err.message;
+  return String(err);
+}