@iislee/opencodex 2.11.0

package/src/server/proxy-liveness.ts

@@ -0,0 +1,201 @@
+/**
+ * Runtime-state-first proxy liveness with identity checking.
+ *
+ * Historically `ensure`/`start` probed only `config.port` and accepted ANY 2xx /healthz:
+ * a proxy that started on a fallback port was invisible (duplicate starts, Codex synced
+ * back to a dead port), and a foreign app answering 200 on the configured port counted
+ * as "our proxy". Liveness now (1) prefers the pid + runtime-port record and (2) requires
+ * the /healthz body to identify as opencodex.
+ *
+ * Lives outside cli.ts (which dispatches argv at module top level) so tests can import it.
+ */
+import { loadConfig, readAlivePid, readRuntimePort, verifyPidIdentity } from "../config";
+
+export interface HealthzIdentity {
+  service?: unknown;
+  status?: unknown;
+  version?: unknown;
+  uptime?: unknown;
+  pid?: unknown;
+}
+
+export interface LivenessIo {
+  fetchFn?: typeof fetch;
+  readPidFn?: () => number | null;
+  /**
+   * Full identity check of the passed candidate pid; must return the SAME pid or null.
+   * Destructive callers only ever receive pids that passed this gate.
+   */
+  verifyPidFn?: (candidatePid: number) => number | null;
+  readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string } | null;
+  configFn?: () => { port?: number; hostname?: string };
+  timeoutMs?: number;
+  /**
+   * How many times to retry a probe that failed with a transport error (timeout /
+   * connection refused). Definitive answers (non-OK HTTP, foreign /healthz body, pid
+   * mismatch) do not retry. Default 1 = no retry. Stop paths should pass 2–3 (#764).
+   */
+  attempts?: number;
+  sleepFn?: (ms: number) => Promise<void>;
+  /**
+   * Absolute wall-clock deadline for discovery. When set, each probe attempt aborts
+   * once the remaining budget cannot cover another fetch — so multi-candidate
+   * `findLiveProxy` under `SERVICE_STOP_LIVENESS` cannot overrun the stop-path
+   * verification window (#764 / CodeRabbit).
+   */
+  deadlineAt?: number;
+  nowFn?: () => number;
+}
+
+/** Default probe options for service stop / orphan cleanup — a just-bound proxy can miss a single 750ms probe. */
+export const SERVICE_STOP_LIVENESS: Pick<LivenessIo, "timeoutMs" | "attempts"> = {
+  timeoutMs: 1500,
+  attempts: 3,
+};
+
+export interface LiveProxy {
+  pid: number | null;
+  port: number;
+  /** Raw bind hostname the probe succeeded against; compose URLs via `probeHostname`. */
+  hostname?: string;
+  /** Whether the successful probe used runtime-port metadata or the configured listen port. */
+  source: "runtime" | "config";
+}
+
+/**
+ * Host to probe for a given bind hostname: wildcards answer on IPv4 loopback, and raw
+ * IPv6 addresses must be bracketed or the composed URL is invalid.
+ */
+export function probeHostname(hostname: string | undefined): string {
+  const trimmed = (hostname ?? "").trim();
+  if (!trimmed || trimmed === "0.0.0.0" || trimmed === "::" || trimmed === "[::]") return "127.0.0.1";
+  if (trimmed.startsWith("[") && trimmed.endsWith("]")) return trimmed;
+  return trimmed.includes(":") ? `[${trimmed}]` : trimmed;
+}
+
+/**
+ * True when a /healthz body identifies an opencodex proxy. Accepts the explicit
+ * `service: "opencodex"` marker, plus the legacy `{status, version, uptime}` trio so a
+ * still-running pre-identity proxy (e.g. right after `ocx update`) is not mistaken for a
+ * foreign server and shadow-started over.
+ */
+export function isOpencodexHealthz(body: HealthzIdentity | null): boolean {
+  if (!body) return false;
+  if (body.service === "opencodex") return true;
+  if (body.service !== undefined) return false;
+  return body.status === "ok" && typeof body.version === "string" && typeof body.uptime === "number";
+}
+
+/** Identity-checked /healthz probe; null when unreachable, non-OK, or not our proxy. */
+export async function proxyIdentityAt(
+  port: number,
+  opts: { hostname?: string; expectedPid?: number } = {},
+  io: LivenessIo = {},
+): Promise<{ pid: number | null } | null> {
+  const fetchFn = io.fetchFn ?? fetch;
+  const sleepFn = io.sleepFn ?? ((ms: number) => new Promise<void>(r => setTimeout(r, ms)));
+  const nowFn = io.nowFn ?? Date.now;
+  const baseTimeoutMs = io.timeoutMs ?? 750;
+  const requestedAttempts = Math.trunc(io.attempts ?? 1);
+  const attempts = Number.isNaN(requestedAttempts)
+    ? 1
+    : Math.max(1, Math.min(requestedAttempts, 5));
+
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    const remainingMs = io.deadlineAt === undefined ? baseTimeoutMs : io.deadlineAt - nowFn();
+    if (remainingMs <= 0) return null;
+    const timeoutMs = Math.min(baseTimeoutMs, remainingMs);
+    try {
+      const res = await fetchFn(`http://${probeHostname(opts.hostname)}:${port}/healthz`, {
+        signal: AbortSignal.timeout(timeoutMs),
+      });
+      if (!res.ok) return null;
+      const body = (await res.json().catch(() => null)) as HealthzIdentity | null;
+      if (!isOpencodexHealthz(body)) return null;
+      const pid = typeof body?.pid === "number" ? body.pid : null;
+      if (opts.expectedPid !== undefined && pid !== null && pid !== opts.expectedPid) return null;
+      return { pid };
+    } catch {
+      // Transport failure (timeout / refused) — retry while budget remains; a proxy that
+      // has only just begun listening can miss a single short probe (#764).
+      if (attempt >= attempts) return null;
+      if (io.deadlineAt !== undefined && io.deadlineAt - nowFn() <= 0) return null;
+      await sleepFn(100);
+    }
+  }
+  return null;
+}
+
+/**
+ * Locate the live proxy: pid file → runtime-port record → identity probe. Falls back to
+ * the configured port ONLY when no runtime record answers, so a fallback-port proxy is
+ * found and a foreign listener on the configured port is rejected.
+ */
+export async function findLiveProxy(io: LivenessIo = {}): Promise<LiveProxy | null> {
+  // Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too
+  // expensive for waitForProxy's 150ms poll loop, and /healthz identity is the real trust gate.
+  const readPidFn = io.readPidFn ?? readAlivePid;
+  const verifyPidFn = io.verifyPidFn ?? verifyPidIdentity;
+  const readRuntimeFn = io.readRuntimeFn ?? readRuntimePort;
+  const configFn = io.configFn ?? loadConfig;
+
+  // The cheap pid is discovery-only. Before it can appear in a returned (killable) result
+  // it must pass the full identity check AND the verifier must echo the exact candidate —
+  // a pidfile rewrite between discovery and verification can never swap in another process.
+  const killablePid = (candidate: number | null): number | null => {
+    if (candidate === null) return null;
+    const verified = verifyPidFn(candidate);
+    return verified === candidate ? verified : null;
+  };
+
+  const verifiedReportedPid = (reported: number | null): number | null => {
+    if (reported === null) return null;
+    if (!Number.isSafeInteger(reported) || reported <= 0) return null;
+    const verified = verifyPidFn(reported);
+    return verified === reported ? verified : null;
+  };
+
+  const pid = readPidFn();
+  let probedPort: number | null = null;
+  if (pid) {
+    const runtime = readRuntimeFn(pid);
+    if (runtime?.port) {
+      probedPort = runtime.port;
+      const identity = await proxyIdentityAt(runtime.port, { hostname: runtime.hostname, expectedPid: pid }, io);
+      if (identity) {
+        // healthz confirmed the pid itself → trusted; a pidless legacy body did not,
+        // so the cheap pid must pass full identity verification before it is returned.
+        const trusted = identity.pid === pid ? pid : killablePid(pid);
+        return { pid: trusted, port: runtime.port, hostname: runtime.hostname, source: "runtime" };
+      }
+    }
+  }
+
+  // Orphan recovery: the pid file can be lost/corrupt while the proxy is alive (crash of a
+  // sibling command, manual deletion). The runtime record still says where it listens —
+  // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it.
+  const record = readRuntimeFn();
+  if (record?.port && record.port !== probedPort) {
+    const expectedPid = typeof record.pid === "number" ? record.pid : undefined;
+    const identity = await proxyIdentityAt(record.port, { hostname: record.hostname, expectedPid }, io);
+    // Only the healthz-reported pid is authoritative here. The record's pid may be stale
+    // (its process dead, the port reused by a pidless legacy proxy) — synthesizing it
+    // would hand destructive callers (stopProxy → kill fallback) a reusable pid.
+    if (identity) {
+      return { pid: verifiedReportedPid(identity.pid), port: record.port, hostname: record.hostname, source: "runtime" };
+    }
+  }
+
+  const config = configFn();
+  const port = config.port ?? 10100;
+  const identity = await proxyIdentityAt(port, { hostname: config.hostname }, io);
+  if (identity) {
+    return {
+      pid: verifiedReportedPid(identity.pid) ?? killablePid(pid),
+      port,
+      hostname: config.hostname,
+      source: "config",
+    };
+  }
+  return null;
+}

package/src/server/relay-eager.ts

@@ -0,0 +1,313 @@
+/**
+ * Eager bounded single-reader SSE relay (#314 mitigation, WP2).
+ *
+ * Replaces the tee()+background-inspection passthrough shape on runtimes where
+ * the Bun#32111 async-pull cancel fix is present (src/lib/bun-stream-caps.ts):
+ * ONE eager producer loop reads upstream, feeds every chunk through the shared
+ * SSE inspector (terminal outcome, quota, request log, context cache), and
+ * enqueues it into a byte-bounded client queue. When the queue is full the
+ * producer pauses — no unbounded tee branch queue can build up behind a slow
+ * client.
+ *
+ * Honesty caveats (audit M5): full leak relief additionally assumes the
+ * runtime carries the Bun#29831 fetch receive-backpressure fix and that Bun's
+ * native Response sink pull-paces a JS ReadableStream. Neither is provable in
+ * bun:test (a JS reader always paces); both remain "awaiting Windows user
+ * verification".
+ *
+ * #44 cancel semantics: after client cancel the relay keeps reading upstream in
+ * DISCARD-DRAIN mode (inspection only) until a terminal is seen or the bounded
+ * drain window (ms/bytes) expires — a genuinely reached terminal records as
+ * completed/failed, never downgraded to cancel. Only when no terminal arrives
+ * within bounds does onClientCancel fire. This bounds today's unbounded tee
+ * drain; the tradeoff is that client-cancel log finalization may be delayed by
+ * up to the drain window.
+ */
+
+import { buildFailedTailPayload } from "./relay";
+import {
+  nextSseBlock,
+  replaceSseDataPayload,
+  sseDataPayload,
+  type SsePayloadRewrite,
+} from "./sse-payload-rewrite";
+import type { TranslatorBudget } from "../lib/translator-budget";
+
+export type EagerRelayHooks = {
+  /** Feed one upstream chunk through SSE inspection (createSseInspector.feed). */
+  inspectChunk: (chunk: Uint8Array) => void;
+  /**
+   * Optional inline client-facing payload rewrite, framed to complete SSE
+   * blocks inside the single reader. This is what lets win32 rewrite traffic
+   * (image_gen restore, item-id repair) use this relay instead of the
+   * Bun#32111-unsafe tee()+JS-pull chain (#864).
+   */
+  rewritePayload?: SsePayloadRewrite;
+  /** Flush inspection at upstream end (createSseInspector.finish). */
+  finishInspection: () => void;
+  /** Drop inspector-owned frame/item state during producer teardown. */
+  disposeInspection?: () => void;
+  /** True once inspection has reported a protocol terminal (inspector.reported). */
+  sawTerminal: () => boolean;
+  /** Record a synthetic terminal (caller decides incomplete vs failed-502). */
+  onSynthetic: (kind: "incomplete" | "failed") => void;
+  /** Client cancelled and NO terminal arrived within the drain bounds. */
+  onClientCancel: () => void;
+  /** Exactly once, after the producer fully stops (unregisterTurn parity). */
+  onDone: () => void;
+};
+
+export type EagerRelayOptions = {
+  /** Bounded client queue in bytes; producer pauses above it. Default 8 MiB. */
+  maxQueueBytes?: number;
+  /** Transient-budget owner for the inline-rewrite frame buffer. */
+  rewriteBudget?: TranslatorBudget;
+  /** Post-cancel discard-drain wall-clock bound. Default 15 000 ms. */
+  postCancelDrainMs?: number;
+  /** Post-cancel discard-drain byte bound. Default 32 MiB. */
+  postCancelDrainBytes?: number;
+  /** Injectable clock for tests. */
+  now?: () => number;
+};
+
+const DEFAULT_MAX_QUEUE_BYTES = 8 * 1024 * 1024;
+const DEFAULT_DRAIN_MS = 15_000;
+const DEFAULT_DRAIN_BYTES = 32 * 1024 * 1024;
+
+/**
+ * Relay `body` to the returned stream with eager bounded reading and inline
+ * inspection. `upstream` is aborted on cancel-drain expiry and observed for
+ * shutdown teardown (its abort wakes a paused producer and suppresses
+ * synthetic terminals — audit M3).
+ */
+export function relaySseEagerBounded(
+  body: ReadableStream<Uint8Array>,
+  upstream: AbortController,
+  hooks: EagerRelayHooks,
+  opts?: EagerRelayOptions,
+): ReadableStream<Uint8Array> {
+  const maxQueueBytes = opts?.maxQueueBytes ?? DEFAULT_MAX_QUEUE_BYTES;
+  const drainMs = opts?.postCancelDrainMs ?? DEFAULT_DRAIN_MS;
+  const drainBytes = opts?.postCancelDrainBytes ?? DEFAULT_DRAIN_BYTES;
+  const now = opts?.now ?? Date.now;
+
+  const reader = body.getReader();
+  const rewrite = hooks.rewritePayload;
+  const rewriteDecoder = rewrite ? new TextDecoder() : null;
+  const rewriteEncoder = rewrite ? new TextEncoder() : null;
+  const rewriteBudget = opts?.rewriteBudget;
+  let frameBuffer = "";
+  let frameBufferBytes = 0;
+  /** Frame complete SSE blocks and rewrite each block's data payload in place. */
+  const rewriteOutbound = (value: Uint8Array): Uint8Array => {
+    let out = "";
+    const fragment = rewriteDecoder!.decode(value, { stream: true });
+    if (rewriteBudget) {
+      const nextBytes = frameBufferBytes + rewriteEncoder!.encode(fragment).byteLength;
+      const reservation = rewriteBudget.reserveTransient(nextBytes, { kind: "live_transient" });
+      try {
+        frameBuffer += fragment;
+        reservation.commitRetained();
+        rewriteBudget.releaseRetained(frameBufferBytes, { kind: "live_transient" });
+        frameBufferBytes = nextBytes;
+      } catch (error) {
+        reservation.release();
+        throw error;
+      }
+    } else {
+      frameBuffer += fragment;
+      frameBufferBytes += value.byteLength;
+    }
+    for (;;) {
+      const next = nextSseBlock(frameBuffer);
+      if (!next) break;
+      const payload = sseDataPayload(next.block);
+      const rewrittenPayload = payload === null ? null : rewrite!(payload);
+      // Replace only on an actual change: replaceSseDataPayload collapses
+      // multi-data-line events and normalizes newline style even when the
+      // payload is identical, which corrupts valid streams.
+      const block = payload !== null && rewrittenPayload !== payload
+        ? replaceSseDataPayload(next.block, rewrittenPayload!)
+        : next.block;
+      out += block + next.delimiter;
+      frameBuffer = next.rest;
+    }
+    if (rewriteBudget) {
+      const remaining = rewriteEncoder!.encode(frameBuffer).byteLength;
+      rewriteBudget.releaseRetained(frameBufferBytes - remaining, { kind: "live_transient" });
+      frameBufferBytes = remaining;
+    } else {
+      frameBufferBytes = rewriteEncoder!.encode(frameBuffer).byteLength;
+    }
+    return rewriteEncoder!.encode(out);
+  };
+  /** Flush any trailing partial block at upstream end (rewrite applied, matching the pull relay). */
+  const flushRewriteTail = (): Uint8Array => {
+    if (!rewrite) return new Uint8Array(0);
+    // Decoder-flushed bytes logically follow everything already decoded.
+    let tail = frameBuffer + rewriteDecoder!.decode();
+    const payload = sseDataPayload(tail);
+    if (payload !== null) {
+      const rewrittenPayload = rewrite(payload);
+      if (rewrittenPayload !== payload) tail = replaceSseDataPayload(tail, rewrittenPayload);
+    }
+    frameBuffer = "";
+    if (rewriteBudget && frameBufferBytes > 0) {
+      rewriteBudget.releaseRetained(frameBufferBytes, { kind: "live_transient" });
+    }
+    frameBufferBytes = 0;
+    return rewriteEncoder!.encode(tail);
+  };
+  let queuedBytes = 0;
+  let cancelled = false;
+  let done = false;
+  // Pause gate: resolved by client pull, client cancel, or upstream abort so a
+  // paused producer ALWAYS resumes (audit blocker 2 — no deadlock; onDone and
+  // turn unregistration stay reachable, drainAndShutdown never hangs).
+  let wake: (() => void) | null = null;
+  const wakeUp = () => { const w = wake; wake = null; w?.(); };
+  const paused = () => new Promise<void>(resolve => { wake = resolve; });
+  upstream.signal.addEventListener("abort", wakeUp, { once: true });
+
+  let controllerRef: ReadableStreamDefaultController<Uint8Array> | null = null;
+  let doneFired = false;
+  let drainTimer: ReturnType<typeof setTimeout> | null = null;
+  const fireDone = () => {
+    if (doneFired) return;
+    doneFired = true;
+    if (drainTimer) { clearTimeout(drainTimer); drainTimer = null; }
+    try { hooks.onDone(); } catch { /* lifecycle callbacks must not break teardown */ }
+  };
+  // A silent upstream after cancel would park the drain loop in reader.read();
+  // the wall-clock bound must fire regardless, so cancel arms a hard timer that
+  // aborts upstream at the deadline (the abort wakes the read).
+  const armDrainTimer = () => {
+    if (drainTimer) return;
+    drainTimer = setTimeout(() => {
+      drainTimer = null;
+      upstream.abort(new Error("post-cancel drain window expired"));
+    }, drainMs);
+    (drainTimer as { unref?: () => void }).unref?.();
+  };
+
+  const producer = async () => {
+    let syntheticKind: "incomplete" | "failed" | null = null;
+    // reader.read() is not intrinsically tied to the upstream AbortController
+    // (a fetch body usually rejects on abort, but that coupling is the fetch
+    // implementation's, not the stream's). Race every read against the abort
+    // signal so cancel-drain expiry and shutdown teardown ALWAYS break the
+    // loop even on a silent upstream.
+    const aborted: Promise<"aborted"> = new Promise(resolve => {
+      if (upstream.signal.aborted) resolve("aborted");
+      else upstream.signal.addEventListener("abort", () => resolve("aborted"), { once: true });
+    });
+    try {
+      for (;;) {
+        const result = await Promise.race([reader.read(), aborted]);
+        if (result === "aborted") break;
+        const { done: upstreamDone, value } = result;
+        if (upstreamDone) {
+          hooks.finishInspection();
+          if (rewrite) {
+            const tail = flushRewriteTail();
+            if (tail.byteLength > 0 && !cancelled) {
+              queuedBytes += tail.byteLength;
+              try { controllerRef?.enqueue(tail); } catch { /* client already gone */ }
+            }
+          }
+          if (!hooks.sawTerminal() && !cancelled && !upstream.signal.aborted) {
+            syntheticKind = "incomplete";
+          }
+          break;
+        }
+        hooks.inspectChunk(value);
+        if (cancelled) {
+          // Discard-drain: inspection only, nothing queued. Stop at terminal
+          // or when the bounded window expires.
+          drainedBytes += value.byteLength;
+          if (hooks.sawTerminal() || drainedBytes >= drainBytes || now() >= drainDeadline) {
+            break;
+          }
+          continue;
+        }
+        const outbound = rewrite ? rewriteOutbound(value) : value;
+        if (outbound.byteLength === 0) continue;
+        queuedBytes += outbound.byteLength;
+        try {
+          controllerRef?.enqueue(outbound);
+        } catch {
+          // Controller already torn down (client went away without cancel()).
+          cancelled = true;
+          drainDeadline = now() + drainMs;
+          armDrainTimer();
+          continue;
+        }
+        while (queuedBytes > maxQueueBytes && !cancelled && !upstream.signal.aborted) {
+          await paused();
+        }
+      }
+    } catch (err) {
+      // Upstream read failure. Distinguish genuine mid-stream reset from
+      // abort-driven teardown (shutdown/cancel-expiry) — audit M3.
+      if (!hooks.sawTerminal() && !cancelled && !upstream.signal.aborted) {
+        // Serializing `err` can run user-defined accessors (Error.message
+        // getters, toString) that re-entrantly cancel the client or abort the
+        // upstream. Build the tail FIRST, then re-check eligibility before
+        // committing to the synthetic terminal (adversarial review blocker).
+        const tail = new TextEncoder().encode(
+          `\n\nevent: response.failed\ndata: ${buildFailedTailPayload(err)}\n\ndata: [DONE]\n\n`,
+        );
+        if (!hooks.sawTerminal() && !cancelled && !upstream.signal.aborted) {
+          syntheticKind = "failed";
+          queuedBytes += tail.byteLength;
+          try { controllerRef?.enqueue(tail); } catch { /* client already torn down */ }
+          try { controllerRef?.close(); } catch { /* client already torn down */ }
+        }
+      }
+    } finally {
+      // Release any retained rewrite-buffer bytes on every teardown path
+      // (error, cancel, upstream abort) — consumption/EOF release alone
+      // leaves them charged.
+      if (rewriteBudget && frameBufferBytes > 0) {
+        try { rewriteBudget.releaseRetained(frameBufferBytes, { kind: "live_transient" }); } catch { /* teardown must not throw */ }
+        frameBufferBytes = 0;
+      }
+      if (syntheticKind) hooks.onSynthetic(syntheticKind);
+      if (cancelled && !hooks.sawTerminal()) {
+        hooks.onClientCancel();
+      }
+      if (cancelled || upstream.signal.aborted || syntheticKind === "failed") {
+        upstream.abort();
+        reader.cancel().catch(() => {});
+      }
+      if (!cancelled) {
+        try { controllerRef?.close(); } catch { /* already closed/errored */ }
+      }
+      try { hooks.disposeInspection?.(); } catch { /* inspection teardown must not block lifecycle cleanup */ }
+      fireDone();
+    }
+  };
+
+  let drainedBytes = 0;
+  let drainDeadline = Number.POSITIVE_INFINITY;
+
+  return new ReadableStream<Uint8Array>({
+    start(controller) {
+      controllerRef = controller;
+      void producer();
+    },
+    pull() {
+      // The client consumed from the queue; approximate accounting: reset on
+      // pull below cap. desiredSize reflects internal queue in chunks, not
+      // bytes, so we track bytes ourselves and drain optimistically.
+      queuedBytes = 0;
+      wakeUp();
+    },
+    cancel() {
+      cancelled = true;
+      drainDeadline = now() + drainMs;
+      armDrainTimer();
+      wakeUp();
+    },
+  });
+}