@seawork/server 2.0.2 → 2.0.3

package/dist/server/server/agent/providers/codex-app-server-agent.js

@@ -24,7 +24,20 @@ const TURN_START_TIMEOUT_MS = 90 * 1000;
 // stalled and force-complete it. Codex normally streams item/reasoning/output
 // events continuously, so a healthy long-running turn re-arms the watchdog far
 // more often than this; only a turn whose `turn/completed` is lost goes silent.
-const TURN_WATCHDOG_IDLE_MS = 5 * 60 * 1000;
+//
+// issue #1886: must stay STRICTLY GREATER than codex's own `stream_idle_timeout`
+// (DEFAULT_STREAM_IDLE_TIMEOUT_MS = 5min). codex logs `Request completed` when the
+// /responses *response headers* arrive — which is when #1824's upstream-liveness
+// re-arm fires — but then reads the SSE *body* stream, and an upstream that sends
+// 200 headers and then stalls the body emits no further stderr and no protocol
+// notification. codex's stream_idle_timeout is the correct authority for that
+// stall (it retries / surfaces an error, both of which re-arm us), so the daemon
+// watchdog must give it a full idle window plus a buffer before tripping. At
+// 5min == 5min the two timers raced from the same "headers arrived" instant and
+// the daemon won, force-failing a turn codex was still legitimately waiting on.
+// 6min lets codex's idle_timeout fire first; the daemon only steps in if codex
+// itself goes silent (process wedged / lost notification).
+const TURN_WATCHDOG_IDLE_MS = 6 * 60 * 1000;
 // issue #1427: when the watchdog decides a turn is stalled, do NOT force-fail
 // immediately. A successful turn's `turn/completed` can arrive tens of ms after
 // the watchdog fires — codex emits it only after a serial flush_rollout +
@@ -125,7 +138,8 @@ const SEAWORK_AUTO_APPROVE_TOOLS = [
     "generation_models",
     "generation_task",
 ];
-const CODEX_IMAGE_ATTACHMENT_DIR = "seawork-attachments";
+const CODEX_EPHEMERAL_IMAGE_ATTACHMENT_DIR = "seawork-attachments";
+const CODEX_IMAGE_ATTACHMENT_DIR = "codex-image-attachments";
 const CODEX_PLAN_IMPLEMENTATION_PROMPT_PREFIX = "The user approved the plan. Implement it now. Do not restate or revise the plan unless blocked.";
 // codex's experimental `goals` feature ships in 0.128.0+. Older binaries reject
 // `--enable goals`, so we version-gate both the launch flag and the /goal
@@ -593,18 +607,6 @@ function toCodexMcpConfig(config) {
             };
     }
 }
-function summarizeJsonRpcParamsForLog(params) {
-    if (params && typeof params === "object" && !Array.isArray(params)) {
-        return { paramKeys: Object.keys(params).sort() };
-    }
-    if (params === null) {
-        return { paramType: "null" };
-    }
-    if (Array.isArray(params)) {
-        return { paramType: "array" };
-    }
-    return { paramType: typeof params };
-}
 // Pull the gateway `x-request-id` out of codex's `codex_client::default_client:
 // Request completed ... url=.../responses ... headers={... "x-request-id": "..." ...}`
 // debug line. Only `/responses` (model turn) requests are matched so a `/models`
@@ -620,6 +622,37 @@ export function extractResponsesRequestId(logLine) {
     }
     return line.match(/"x-request-id":\s*"([0-9a-fA-F-]+)"/)?.[1] ?? null;
 }
+// DIAGNOSTIC (watchdog mis-kill): pull the identifying fields out of a codex
+// notification's params for logging, without dumping the (potentially huge)
+// full payload. Surfaces the turn/item identity so a gap in a specific turn's
+// lifecycle notifications is attributable in daemon.log.
+function summarizeJsonRpcParamsForLog(params) {
+    if (!params || typeof params !== "object")
+        return {};
+    if (Array.isArray(params))
+        return { paramType: "array" };
+    const p = params;
+    const out = {};
+    for (const key of ["turnId", "turn_id", "itemId", "item_id", "id", "type", "status"]) {
+        if (p[key] !== undefined && (typeof p[key] === "string" || typeof p[key] === "number")) {
+            out[key] = p[key];
+        }
+    }
+    // codex nests the turn id under turn:{id} / item:{id} in some notifications.
+    const turn = p.turn;
+    if (turn && typeof turn.id === "string")
+        out.turnId = turn.id;
+    const item = p.item;
+    if (item && typeof item.id === "string")
+        out.itemId = item.id;
+    if (item && typeof item.type === "string")
+        out.itemType = item.type;
+    // Fallback for unknown object shapes: surface key names only (no values) so
+    // the log entry is attributable without leaking payload content.
+    if (Object.keys(out).length === 0)
+        return { paramKeys: Object.keys(p).sort() };
+    return out;
+}
 class CodexAppServerClient {
     constructor(child, logger) {
         this.child = child;
@@ -631,9 +664,24 @@ class CodexAppServerClient {
         this.upstreamLivenessHandler = null;
         this.nextId = 1;
         this.disposed = false;
+        // DIAGNOSTIC (watchdog mis-kill): per-method lifetime counts of notifications
+        // seen off codex stdout, and per-delta-method last-logged time for rollups.
+        this.notificationLineCounts = new Map();
+        this.notificationDeltaLastLoggedAt = new Map();
         this.stderrBuffer = "";
         this.stderrLineBuf = "";
+        // DIAGNOSTIC (watchdog mis-kill): separate line-assembly buffer for mirroring
+        // codex stderr (RUST_LOG output) into daemon.log; kept apart from stderrLineBuf
+        // which the responses-request-id scraper consumes.
+        this.stderrLogLineBuf = "";
         this.lastResponsesRequestId = null;
+        // issue #1886 DIAGNOSTIC: wall-clock of the last stderr byte from codex. Even at
+        // the default `error` RUST_LOG level codex emits *something* while alive (reqwest
+        // debug, retries), so when the watchdog trips, a fresh value here means codex is
+        // alive-but-not-notifying (gap is codex→daemon / SSE body stall) while a stale
+        // one means codex itself went silent (process wedged / upstream death) — the
+        // distinction the force-fail diagnostic could not previously make.
+        this.stderrLastActivityAt = null;
         this.resolveExitPromise = null;
         this.rl = readline.createInterface({ input: child.stdout });
         this.exitPromise = new Promise((resolve) => {
@@ -642,11 +690,31 @@ class CodexAppServerClient {
         this.rl.on("line", (line) => this.handleLine(line));
         child.stderr.on("data", (chunk) => {
             const text = chunk.toString();
+            this.stderrLastActivityAt = Date.now();
             this.stderrBuffer += text;
             if (this.stderrBuffer.length > 8192) {
                 this.stderrBuffer = this.stderrBuffer.slice(-8192);
             }
             this.captureResponsesRequestId(text);
+            // DIAGNOSTIC (watchdog mis-kill): the stderr buffer is a rolling 8KB tail
+            // kept only for crash reports, so codex's RUST_LOG output (the only window
+            // into codex-internal turn/event lifecycle) was otherwise discarded. Mirror
+            // it line-by-line into daemon.log at debug level so that, when [notif-rx]
+            // shows zero notifications during a stall, we can still see whether codex
+            // core emitted the event and whether the app-server tried to write it —
+            // i.e. attribute a notification gap to codex vs the daemon. Lines are
+            // already RUST_LOG-gated upstream, so volume tracks the configured level.
+            this.stderrLogLineBuf += text;
+            let nlIdx;
+            while ((nlIdx = this.stderrLogLineBuf.indexOf("\n")) >= 0) {
+                const logLine = this.stderrLogLineBuf.slice(0, nlIdx).trimEnd();
+                this.stderrLogLineBuf = this.stderrLogLineBuf.slice(nlIdx + 1);
+                if (logLine)
+                    this.logger.debug({ codexStderr: logLine }, "[codex-stderr]");
+            }
+            if (this.stderrLogLineBuf.length > 16384) {
+                this.stderrLogLineBuf = this.stderrLogLineBuf.slice(-16384);
+            }
         });
         child.on("error", (err) => {
             this.logger.error({ err }, "Codex app-server child process error");
@@ -701,6 +769,23 @@ class CodexAppServerClient {
     getLastResponsesRequestId() {
         return this.lastResponsesRequestId;
     }
+    // issue #1886 DIAGNOSTIC: liveness signals the session folds into the watchdog
+    // force-fail snapshot. `sinceLastStderrMs` is the key one — see the field comment.
+    getLivenessSnapshot() {
+        // Session-cumulative (the line-count map is never reset per turn); a near-zero
+        // value confirms codex sent almost nothing the daemon ever saw.
+        let notificationsSeenTotal = 0;
+        for (const count of this.notificationLineCounts.values()) {
+            notificationsSeenTotal += count;
+        }
+        return {
+            sinceLastStderrMs: this.stderrLastActivityAt !== null ? Date.now() - this.stderrLastActivityAt : null,
+            childAlive: !this.child.killed && !this.disposed,
+            childPid: this.child.pid,
+            lastResponsesRequestId: this.lastResponsesRequestId,
+            notificationsSeenTotal,
+        };
+    }
     resetLastResponsesRequestId() {
         this.lastResponsesRequestId = null;
     }
@@ -826,6 +911,33 @@ class CodexAppServerClient {
         }
         if (typeof msg.method === "string") {
             const notification = msg;
+            // DIAGNOSTIC (watchdog mis-kill investigation): log every notification as it
+            // arrives off codex's stdout — this is the first daemon-side touch point, so
+            // it proves whether codex actually emitted a notification (vs the daemon
+            // dropping it later). High-frequency delta methods are counted, not logged
+            // per-event, to avoid flooding; lifecycle methods (turn/*, item/started,
+            // item/completed, etc.) are logged individually with their turnId so a
+            // "5min with zero turn notifications" gap is directly visible in daemon.log.
+            const method = notification.method;
+            const isHighFreqDelta = method.includes("/delta") || method.endsWith("Delta");
+            this.notificationLineCounts.set(method, (this.notificationLineCounts.get(method) ?? 0) + 1);
+            if (isHighFreqDelta) {
+                // Log a rollup at most once per second per method so deltas stay visible
+                // (proving the stream is alive) without one line per token.
+                const now = Date.now();
+                const lastLogged = this.notificationDeltaLastLoggedAt.get(method) ?? 0;
+                if (now - lastLogged >= 1000) {
+                    this.notificationDeltaLastLoggedAt.set(method, now);
+                    this.logger.debug({ method, totalSoFar: this.notificationLineCounts.get(method) }, "[notif-rx] codex delta notification (rolled up)");
+                }
+            }
+            else {
+                this.logger.debug({
+                    method,
+                    ...summarizeJsonRpcParamsForLog(notification.params),
+                    totalSoFar: this.notificationLineCounts.get(method),
+                }, "[notif-rx] codex notification received off stdout");
+            }
             this.notificationHandler?.(notification.method, notification.params);
         }
     }
@@ -1457,15 +1569,23 @@ function normalizeCodexGeneratedImageSource(item) {
         nonEmptyString(item.saved_path) ??
         nonEmptyString(item.path) ??
         null;
-    if (savedPath) {
+    const result = nonEmptyString(item.result);
+    if (savedPath && !isEphemeralCodexGeneratedImagePath(savedPath)) {
         return savedPath;
     }
-    const result = nonEmptyString(item.result);
     if (!result) {
+        if (savedPath) {
+            return savedPath;
+        }
         return null;
     }
     return result.startsWith("data:") ? result : `data:image/png;base64,${result}`;
 }
+function isEphemeralCodexGeneratedImagePath(savedPath) {
+    const normalizedPath = path.resolve(savedPath);
+    const attachmentsDir = path.resolve(os.tmpdir(), CODEX_EPHEMERAL_IMAGE_ATTACHMENT_DIR);
+    return (normalizedPath === attachmentsDir || normalizedPath.startsWith(`${attachmentsDir}${path.sep}`));
+}
 function markdownImageSource(source) {
     if (/^(https?:|data:|blob:)/i.test(source)) {
         return source;
@@ -1585,6 +1705,9 @@ function getImageExtension(mimeType) {
             return "bin";
     }
 }
+// MIME types we re-encode to WebP. Excludes gif (animation is lost) and webp
+// (avoids a needless second lossy pass); anything not listed is written as-is.
+const WEBP_CONVERTIBLE = new Set(["image/png", "image/jpeg", "image/bmp", "image/tiff"]);
 function normalizeImageData(mimeType, data) {
     if (data.startsWith("data:")) {
         const match = data.match(/^data:([^;]+);base64,(.*)$/);
@@ -2274,16 +2397,44 @@ function parseReconnectAttemptCounts(message) {
     }
     return { attempt: Number(match[1]), maxAttempts: Number(match[2]) };
 }
-async function writeImageAttachment(mimeType, data) {
-    const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR);
+async function writeImageAttachment(mimeType, data, logger) {
+    const attachmentsDir = path.join(resolveSeaworkHome(), CODEX_IMAGE_ATTACHMENT_DIR);
     await fs.mkdir(attachmentsDir, { recursive: true });
     const normalized = normalizeImageData(mimeType, data);
-    const extension = getImageExtension(normalized.mimeType);
-    const filename = `${randomUUID()}.${extension}`;
+    const rawBuffer = Buffer.from(normalized.data, "base64");
+    let outBuffer = rawBuffer;
+    let outMime = normalized.mimeType;
+    if (WEBP_CONVERTIBLE.has(normalized.mimeType)) {
+        try {
+            const sharp = (await import("sharp")).default;
+            // Downscale oversized images (longest edge > 2048) and re-encode to WebP.
+            // #1824: large image-2 outputs accumulate in every turn's request body
+            // (170MB → 502); downscaling is where the bulk of the savings comes from.
+            const webpBuffer = await sharp(rawBuffer)
+                .resize(2048, 2048, { fit: "inside", withoutEnlargement: true })
+                .webp({ quality: 80 })
+                .toBuffer();
+            // Pathological inputs (noise, already-compressed) can grow under WebP;
+            // only adopt the conversion when it actually shrinks the payload.
+            if (webpBuffer.length < rawBuffer.length) {
+                outBuffer = webpBuffer;
+                outMime = "image/webp";
+            }
+        }
+        catch (error) {
+            // Conversion failed (sharp unavailable / decode error): keep the original
+            // buffer + mime so the turn does not fail and the attachment is preserved.
+            logger.warn({ error }, "sharp webp conversion failed, keeping original");
+        }
+    }
+    const filename = `${randomUUID()}.${getImageExtension(outMime)}`;
     const filePath = path.join(attachmentsDir, filename);
-    await fs.writeFile(filePath, Buffer.from(normalized.data, "base64"));
+    await fs.writeFile(filePath, outBuffer);
     return filePath;
 }
+function resolveSeaworkHome() {
+    return process.env.SEAWORK_HOME ?? path.join(os.homedir(), ".seawork");
+}
 async function readCodexConfiguredDefaults(client, logger) {
     let savedConfigDefaults = {};
     try {
@@ -2328,7 +2479,7 @@ export async function codexAppServerTurnInputFromPrompt(prompt, logger) {
             typeof record.mimeType === "string" &&
             typeof record.data === "string") {
             try {
-                const filePath = await writeImageAttachment(record.mimeType, record.data);
+                const filePath = await writeImageAttachment(record.mimeType, record.data, logger);
                 output.push({ type: "localImage", path: filePath });
             }
             catch (error) {
@@ -2387,7 +2538,34 @@ function buildCodexAppServerEnv(runtimeSettings, launchEnv) {
     // the id from the error message. Scoped to a single module plus a global
     // `error` level so stderr stays lean. A user-set RUST_LOG always wins.
     if (!readStringMetadata(merged.RUST_LOG)) {
-        merged.RUST_LOG = "error,codex_client::default_client=debug";
+        // DIAGNOSTIC (watchdog mis-kill): SEAWORK_CODEX_VERBOSE_LOG escalates codex's
+        // RUST_LOG so the [codex-stderr] mirror captures codex-internal turn/event
+        // lifecycle. Use this when [notif-rx] shows zero notifications during a stall
+        // and the gap must be attributed to codex (core never emitted / app-server
+        // never wrote stdout) vs the daemon. Off by default to keep stderr lean —
+        // these levels are high-volume on a busy turn. Set in the daemon's
+        // environment, then reload the agent so codex respawns with the new RUST_LOG
+        // (no daemon restart needed once the env is present at daemon start).
+        //   1 / "on"  -> turn+event lifecycle (codex_core::tasks + app_server)
+        //   "trace"   -> also SSE frames (codex_api=trace) — very high volume;
+        //                SECURITY: codex_api=trace logs the raw SSE payload for every
+        //                frame (including model output tokens, tool call contents, and
+        //                assistant text). Use only on isolated dev/debug environments —
+        //                never in production — as daemon.log will contain user prompt
+        //                and model response data. Proves upstream byte arrival when
+        //                core appears stalled.
+        const verbose = readStringMetadata(merged.SEAWORK_CODEX_VERBOSE_LOG)?.toLowerCase();
+        if (verbose === "trace") {
+            merged.RUST_LOG =
+                "error,codex_client::default_client=debug,codex_core::tasks=debug,codex_app_server=debug,codex_api=trace,codex_client=debug";
+        }
+        else if (verbose === "1" || verbose === "on" || verbose === "true") {
+            merged.RUST_LOG =
+                "error,codex_client::default_client=debug,codex_core::tasks=debug,codex_app_server=debug";
+        }
+        else {
+            merged.RUST_LOG = "error,codex_client::default_client=debug";
+        }
     }
     return merged;
 }
@@ -4372,6 +4550,29 @@ class CodexAppServerAgentSession {
         }
         return stderrTail;
     }
+    // issue #1886 DIAGNOSTIC: structured liveness dump emitted when the watchdog
+    // judges the active turn stalled. Unconditional (not RUST_LOG-gated) so a
+    // production stall is always attributable: pairs daemon-side silence with
+    // codex-side liveness to pin the gap on the codex→daemon channel / SSE body
+    // stall vs codex itself going silent.
+    logWatchdogLivenessSnapshot() {
+        const liveness = this.client?.getLivenessSnapshot();
+        this.logger.warn({
+            turnId: this.activeForegroundTurnId,
+            codexTurnId: this.expectedCodexTurnId,
+            sinceLastNotificationMs: this.lastTurnNotificationTime !== null
+                ? Date.now() - this.lastTurnNotificationTime
+                : null,
+            sinceLastStderrMs: liveness?.sinceLastStderrMs ?? null,
+            childAlive: liveness?.childAlive ?? null,
+            childPid: liveness?.childPid ?? null,
+            lastResponsesRequestId: liveness?.lastResponsesRequestId ?? null,
+            inFlightToolCalls: this.inFlightToolCalls.size,
+            compactionInFlight: this.compactionInFlight.size,
+            pendingPermissions: this.pendingPermissionHandlers.size,
+            notificationsSeenTotal: liveness?.notificationsSeenTotal ?? null,
+        }, "[watchdog-snapshot] turn judged stalled; liveness at decision time");
+    }
     emitEvent(event) {
         // Note: assistant-message buffers (`pendingAgentMessages`) are NOT cleared
         // here. Streaming now emits multiple `assistant_message` timeline items
@@ -4484,6 +4685,15 @@ class CodexAppServerAgentSession {
                 }
             }
             const stallError = "Codex turn stalled (no events received); recovered by watchdog";
+            // issue #1886 DIAGNOSTIC: the moment the watchdog decides the turn is stalled,
+            // dump a structured liveness snapshot unconditionally (no RUST_LOG gating).
+            // This is the line that closes the gap: it pairs daemon-side silence
+            // (sinceLastNotification) with codex-side liveness (sinceLastStderr, childAlive,
+            // lastResponsesRequestId). Fresh stderr + alive child ⇒ the gap is the
+            // codex→daemon channel / a stalled SSE body (the #1886 case the 6min bump
+            // targets); equally-silent stderr ⇒ codex itself wedged. Previously the
+            // force-fail log carried only sinceLastNotification and could tell neither apart.
+            this.logWatchdogLivenessSnapshot();
             const context = [
                 `pendingPermissions=${pendingPermissionsForActiveTurn}`,
                 `watchdogCycles inflight=${this.turnWatchdogInflightCycles} compaction=${this.turnWatchdogCompactionCycles}`,
@@ -4815,6 +5025,14 @@ class CodexAppServerAgentSession {
             this.lastTurnNotificationTime = Date.now();
             this.armTurnWatchdog();
         }
+        else {
+            // DIAGNOSTIC (watchdog mis-kill): a notification arrived but there is no
+            // active foreground turn to re-arm the watchdog against. If this fires
+            // during a window where the watchdog later force-fails, it means codex's
+            // liveness signal landed but could not renew the timer — pinpointing the
+            // gap as a turn-identity mismatch rather than codex silence.
+            this.logger.debug({ method, kind: parsed.kind }, "[notif-rx] notification with no activeForegroundTurnId — watchdog NOT re-armed");
+        }
         // issue #1427: a real (non-reconnect) notification after a reconnect marker
         // means the stream is flowing again — close the loading marker so the UI
         // stops showing "Reconnecting…".