@pikiloom/kernel 0.3.15 → 0.3.17

package/dist/drivers/codex.d.ts

@@ -19,8 +19,15 @@ export declare function captureCodexAgentMessage(item: any, s: CodexContentState
 export declare function captureCodexReasoning(text: string, s: CodexContentState, emit: (e: DriverEvent) => void): void;
 export declare function codexFinalText(s: CodexContentState): string;
 export declare function codexFinalReasoning(s: CodexContentState): string;
+export interface CodexLivenessOptions {
+    /** Silence after an accepted steer before the turn is healed (interrupt + replay). */
+    steerStallMs?: number;
+    /** Idle silence (no running tool, no pending HITL request) before the turn is force-closed. */
+    turnStallMs?: number;
+}
 export declare class CodexDriver implements AgentDriver {
     private readonly bin;
+    private readonly liveness;
     readonly id = "codex";
     readonly capabilities: {
         steer: boolean;
@@ -29,7 +36,7 @@ export declare class CodexDriver implements AgentDriver {
         tui: boolean;
         fork: boolean;
     };
-    constructor(bin?: string);
+    constructor(bin?: string, liveness?: CodexLivenessOptions);
     run(input: AgentTurnInput, ctx: DriverContext): Promise<DriverResult>;
     tui(input: TuiInput): TuiSpec;
     listNativeSessions(opts: {

package/dist/drivers/codex.js

@@ -234,12 +234,53 @@ export function codexFinalText(s) {
 export function codexFinalReasoning(s) {
     return s.reasoning.trim() ? s.reasoning : s.thinkParts.join('\n\n');
 }
+// ── Turn liveness (steer race + silent-stall recovery) ──────────────────────────
+// codex app-server has a turn-boundary race: a `turn/steer` that lands in the instant an
+// item completes is ACCEPTED (recorded into the rollout) but never dispatched — the agent
+// loop goes idle, no further notifications arrive, and `turn/completed` never fires, so the
+// turn hangs forever while the process sits at 0% CPU (observed on codex-cli 0.144.x;
+// same family as openai/codex#15714 / #23807). Two defenses below:
+//
+// 1. A successful steer is treated as ACCEPTED, NOT CONSUMED: it stays pending until a
+//    progress notification proves the loop picked it up. If the turn instead goes silent
+//    (or completes without consuming it), the driver heals in place — interrupt the wedged
+//    turn and restart it with the same input. The steered text is already in the thread
+//    history, so the worst false-positive cost is one duplicated user message; the turn
+//    keeps running under the same run()/task, invisible to upper layers.
+// 2. A generic silence backstop: a turn with no notifications for a long stretch while
+//    nothing is visibly in flight (no running tool call, no server->client request awaiting
+//    a human) is declared stalled and force-closed, so the session ends as a visible error
+//    instead of spinning forever.
+// Thresholds are calibrated against measured rollouts: with reasoning summaries off, a
+// HEALTHY turn shows fully silent thinking stretches of up to ~320s (clustered just above
+// codex core's own 300s stream watchdog, whose retry produces the next event). 600s has
+// never been reached intra-turn in days of observed sessions, so a heal at that point is
+// near-certainly a real wedge — and even a misfire only costs one duplicated user message
+// plus the in-flight thinking (completed items live in the thread history).
+const CODEX_STEER_STALL_MS = 600_000;
+const CODEX_TURN_STALL_MS = 900_000;
+// Notifications that prove the agent loop is making forward progress AFTER a steer.
+// item/completed and tokenUsage are deliberately excluded — they can be the trailing edge
+// of work that finished before the steer landed (the exact race signature). Progress that
+// arrives within CODEX_STEER_CONSUMED_MS of the steer only refreshes the pending marker's
+// clock rather than clearing it: the app-server can flush pre-acceptance stream output in
+// the same write as the steer response, so near-simultaneous events are not proof the
+// loop survived the injection. Progress beyond that window is.
+const CODEX_STEER_CONSUMED_MS = 2_000;
+const CODEX_PROGRESS_METHODS = new Set([
+    'turn/started', 'item/started', 'item/agentMessage/delta',
+    'item/reasoning/textDelta', 'item/reasoning/summaryTextDelta',
+    'item/commandExecution/outputDelta', 'item/fileChange/outputDelta',
+    'item/fileChange/patchUpdated', 'turn/plan/updated', 'rawResponseItem/completed',
+]);
 export class CodexDriver {
     bin;
+    liveness;
     id = 'codex';
     capabilities = { steer: true, interact: false, resume: true, tui: true, fork: true };
-    constructor(bin = 'codex') {
+    constructor(bin = 'codex', liveness = {}) {
         this.bin = bin;
+        this.liveness = liveness;
     }
     async run(input, ctx) {
         // BYOK provider routing arrives as `-c key=value` overrides; pass them through so the
@@ -261,6 +302,15 @@ export class CodexDriver {
         const deltaItems = new Set();
         let lastTextItemId = null;
         let steerRegistered = false;
+        // Liveness state (see the CODEX_STEER_STALL_MS block comment above).
+        const steerStallMs = this.liveness.steerStallMs ?? CODEX_STEER_STALL_MS;
+        const turnStallMs = this.liveness.turnStallMs ?? CODEX_TURN_STALL_MS;
+        let lastEventAt = Date.now();
+        let pendingSteer = null;
+        let pendingServerRequests = 0;
+        let healing = false;
+        let stalled = false;
+        let livenessTimer = null;
         if (!srv.start())
             return { ok: false, text: '', error: 'failed to start codex app-server', stopReason: 'error' };
         let settled = false;
@@ -320,23 +370,78 @@ export class CodexDriver {
                 state.sessionId = threadId;
                 ctx.emit({ type: 'session', sessionId: threadId });
             }
+            // Heal a wedged/lost steer in place: (optionally) interrupt the dead turn, then restart
+            // it with the same input. Runs under the SAME run()/turnDone, so upper layers just see
+            // the turn continue. `healing` suppresses the interrupt's own turn/completed echo.
+            const heal = async (opts) => {
+                if (settled || healing || !pendingSteer)
+                    return;
+                healing = true;
+                const replayInput = pendingSteer.input;
+                pendingSteer = null;
+                if (opts.interrupt && state.sessionId && state.turnId) {
+                    await srv.request('turn/interrupt', { threadId: state.sessionId, turnId: state.turnId }, 5_000);
+                }
+                if (settled) {
+                    healing = false;
+                    return;
+                } // raced with abort / process death
+                const resp = await srv.request('turn/start', {
+                    threadId: state.sessionId,
+                    input: replayInput,
+                    model: input.model || undefined,
+                    effort: input.effort || undefined,
+                });
+                if (settled) {
+                    healing = false;
+                    return;
+                }
+                if (resp.error) {
+                    state.status = 'error';
+                    state.error = `steer recovery failed: ${resp.error.message || 'turn/start failed'}`;
+                    healing = false;
+                    settle();
+                    return;
+                }
+                state.turnId = resp.result?.turn?.id ?? state.turnId;
+                lastEventAt = Date.now();
+                healing = false;
+            };
             // Codex emits no explicit compaction event, so track peak occupancy: a sharp
             // mid-turn drop is an auto-compaction, surfaced as a live `compaction` signal.
             let compactPeakTokens = 0;
             srv.onNotification((method, params) => {
                 if (params?.threadId && params.threadId !== state.sessionId && method !== 'turn/started')
                     return;
+                lastEventAt = Date.now();
+                if (pendingSteer && CODEX_PROGRESS_METHODS.has(method)) {
+                    const now = Date.now();
+                    if (now - pendingSteer.at >= CODEX_STEER_CONSUMED_MS)
+                        pendingSteer = null; // loop provably alive post-steer
+                    else
+                        pendingSteer.progressAt = now; // maybe pre-acceptance flush — keep watching
+                }
                 switch (method) {
                     case 'turn/started':
                         state.turnId = params?.turn?.id ?? null;
                         if (!steerRegistered && state.turnId) {
                             steerRegistered = true;
                             ctx.registerSteer(async (prompt, attachments = []) => {
-                                if (!state.sessionId || !state.turnId)
+                                if (settled || !state.sessionId || !state.turnId)
                                     return false;
-                                const r = await srv.request('turn/steer', { threadId: state.sessionId, expectedTurnId: state.turnId, input: buildTurnInput(prompt, attachments) }, 30_000);
-                                if (r.error)
+                                const steerInput = buildTurnInput(prompt, attachments);
+                                // Arm BEFORE sending — accepted, not yet consumed. Arming after the response
+                                // would race the response's own microtask against progress notifications the
+                                // server flushed in the same write, mis-arming against already-consumed steers.
+                                // Back-to-back steers accumulate so a heal replays everything swallowed.
+                                const armed = { input: [...(pendingSteer?.input ?? []), ...steerInput], at: Date.now(), progressAt: null };
+                                pendingSteer = armed;
+                                const r = await srv.request('turn/steer', { threadId: state.sessionId, expectedTurnId: state.turnId, input: steerInput }, 30_000);
+                                if (r.error) {
+                                    if (pendingSteer === armed)
+                                        pendingSteer = null; // rejected — nothing to watch
                                     return false;
+                                }
                                 state.turnId = r.result?.turnId ?? state.turnId;
                                 return true;
                             });
@@ -466,11 +571,21 @@ export class CodexDriver {
                     }
                     case 'turn/completed': {
                         const turn = params?.turn || {};
+                        applyCodexTokenUsage(state, params?.tokenUsage || turn.tokenUsage || turn.usage);
+                        ctx.emit({ type: 'usage', usage: codexUsageOf(state) });
+                        if (healing)
+                            break; // completion echo of the turn heal() just interrupted
+                        if (pendingSteer && !pendingSteer.progressAt && !ctx.signal.aborted && (turn.status ?? 'completed') === 'completed') {
+                            // The other face of the steer race: the turn finished without ever consuming
+                            // the injected input. Replay it as a fresh turn instead of settling — the
+                            // upper layers already dequeued the message on steer-ok, so settling here
+                            // would silently drop it.
+                            void heal({ interrupt: false });
+                            break;
+                        }
                         state.status = turn.status ?? 'completed';
                         if (turn.error)
                             state.error = turn.error.message || turn.error.code || 'turn error';
-                        applyCodexTokenUsage(state, params?.tokenUsage || turn.tokenUsage || turn.usage);
-                        ctx.emit({ type: 'usage', usage: codexUsageOf(state) });
                         settle();
                         break;
                     }
@@ -480,6 +595,7 @@ export class CodexDriver {
             // accept approvals by default (parity with the legacy codex driver). Never throw —
             // an unanswerable request degrades to an empty response, not a JSON-RPC error.
             srv.onRequest(async (method, params, id) => {
+                pendingServerRequests++; // a request awaiting a human legitimately silences the turn
                 try {
                     if (method === 'item/tool/requestUserInput') {
                         const interaction = codexUserInputToInteraction(params, `codex-input-${id}`);
@@ -497,6 +613,9 @@ export class CodexDriver {
                 catch {
                     return {};
                 }
+                finally {
+                    pendingServerRequests--;
+                }
             });
             const turnResp = await srv.request('turn/start', {
                 threadId: state.sessionId,
@@ -506,6 +625,35 @@ export class CodexDriver {
             });
             if (turnResp.error)
                 return { ok: false, text: state.text, error: turnResp.error.message || 'turn/start failed', stopReason: 'error', sessionId: state.sessionId };
+            // Liveness checker: heals a stalled steer, force-closes a silently dead turn. The
+            // silence clock only accumulates while nothing is visibly in flight — a running tool
+            // call or a server->client request parked on a human keeps the turn alive forever.
+            const checkEveryMs = Math.max(25, Math.min(5_000, Math.floor(Math.min(steerStallMs, turnStallMs) / 4)));
+            livenessTimer = setInterval(() => {
+                if (settled || healing)
+                    return;
+                const now = Date.now();
+                if (pendingSteer) {
+                    if (now - (pendingSteer.progressAt ?? pendingSteer.at) >= steerStallMs)
+                        void heal({ interrupt: true });
+                    return;
+                }
+                const busy = pendingServerRequests > 0 || [...toolCalls.values()].some(c => c.status === 'running');
+                if (busy) {
+                    lastEventAt = now;
+                    return;
+                }
+                if (now - lastEventAt < turnStallMs)
+                    return;
+                stalled = true;
+                state.error = state.error || `codex app-server went silent mid-turn (no events for ${Math.round(turnStallMs / 1000)}s); closing the turn`;
+                if (state.sessionId && state.turnId) {
+                    srv.request('turn/interrupt', { threadId: state.sessionId, turnId: state.turnId }, 5_000).finally(() => settle());
+                }
+                else
+                    settle();
+            }, checkEveryMs);
+            livenessTimer.unref?.();
             await turnDone;
             const usage = codexUsageOf(state);
             const ok2 = (state.status === 'completed' || state.status == null) && !state.error && !ctx.signal.aborted;
@@ -515,7 +663,7 @@ export class CodexDriver {
                 text: codexFinalText(state),
                 reasoning: finalReasoning || undefined,
                 error: state.error || (ctx.signal.aborted ? 'Interrupted by user.' : null),
-                stopReason: ctx.signal.aborted ? 'interrupted' : (state.status || 'end_turn'),
+                stopReason: ctx.signal.aborted ? 'interrupted' : stalled ? 'stalled' : (state.status || 'end_turn'),
                 sessionId: state.sessionId,
                 // Fork anchor: the turn id is the inclusive keep-boundary thread/fork's lastTurnId takes.
                 anchor: state.turnId,
@@ -523,6 +671,8 @@ export class CodexDriver {
             };
         }
         finally {
+            if (livenessTimer)
+                clearInterval(livenessTimer);
             srv.kill();
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikiloom/kernel",
-  "version": "0.3.15",
+  "version": "0.3.17",
   "description": "Heterogeneous coding agents (Claude / Codex / Gemini / ACP) -> an interaction-friendly, accumulating session snapshot + control handle, over pluggable surfaces (IM, Web, tunnel, TUI). The reusable core pikiloom itself is built on.",
   "type": "module",
   "license": "MIT",