@mjasnikovs/pi-task 0.18.36 → 0.18.38

package/dist/task/stream-watchdog.js

@@ -0,0 +1,90 @@
+import { getConfig } from '../config/config.js';
+import { realStreamTimerDeps, StreamWatchdog, streamStallReminder } from '../shared/stream-watchdog.js';
+import { noteWatchdogAbort, WATCHDOG_CANCEL_MARKER } from './command-watchdog.js';
+/**
+ * MAIN-SESSION adapter for the model-stream watchdog.
+ *
+ * WHY (mx5 run 14): three implementation turns died mid-turn — the session jsonl's
+ * last record is an ordinary assistant message, then silence forever, while the
+ * model container stayed Up(healthy). No error is ever thrown for this shape, so
+ * the connection-error retry (which needs a reported ModelError) cannot fire and
+ * the command watchdog, which only covers tool executions, never arms. The run sat
+ * dead for ~2.9h across the three until a human restarted it.
+ *
+ * HOW: pi's extension events ARE the stream. Any of them — a token delta, a
+ * thinking delta, a tool-call delta, the provider's response headers — resets the
+ * idle clock; only total silence for the configured window fires. On fire the turn
+ * is aborted and a follow-up user turn tells the model to CONTINUE from the
+ * transcript (its completed tool calls and results are already recorded, so a
+ * blind re-send would re-run them).
+ *
+ * ONE ABORT CHANNEL: the fire path goes through the command watchdog's existing
+ * {@link noteWatchdogAbort} flag and its WATCHDOG_CANCEL_MARKER, so
+ * steerUntilDone's already-fixed abort/steer race (b543d15) covers this watchdog
+ * too instead of racing a second, parallel abort mechanism.
+ *
+ * SUSPENDED DURING TOOLS: while a tool executes the model stream is legitimately
+ * idle — a 12-minute build emits nothing. That window belongs to the command
+ * watchdog (requestTimeoutMs); this one pauses between tool_execution_start and
+ * tool_execution_end so the two can never double-fire on the same silence.
+ *
+ * SCOPE: main session only. Children run `--no-extensions`, so their equivalent
+ * guard lives in runChild (shared/child-process.ts) and shares the same machine.
+ */
+export function registerStreamWatchdog(pi) {
+    // The ctx whose abort() ends the in-flight turn, refreshed on every event so
+    // the fire (which happens outside any handler) aborts the CURRENT operation.
+    let liveCtx;
+    const watchdog = new StreamWatchdog({
+        getTimeoutMs: () => getConfig().streamInactivityMs,
+        ...realStreamTimerDeps,
+        onFire: idleMs => {
+            const ctx = liveCtx;
+            liveCtx = undefined;
+            // Flag BEFORE the abort, exactly as the command watchdog does: the
+            // steer loop can otherwise observe the 'aborted' turn first and show
+            // a steering prompt to an empty room, wedging an unattended run.
+            if (ctx) {
+                noteWatchdogAbort();
+                ctx.abort();
+            }
+            pi.sendUserMessage(streamStallReminder(idleMs, WATCHDOG_CANCEL_MARKER), {
+                deliverAs: 'followUp'
+            });
+        }
+    });
+    // Any event proves the stream is alive. `arm` also (re)starts the machine, so
+    // a request that begins after a previous turn ended is watched again without
+    // needing a single canonical "request started" event.
+    const arm = (ctx) => {
+        if (ctx)
+            liveCtx = ctx;
+        watchdog.start();
+        watchdog.note();
+    };
+    pi.on('before_provider_request', (_e, ctx) => arm(ctx));
+    pi.on('after_provider_response', (_e, ctx) => arm(ctx));
+    pi.on('turn_start', (_e, ctx) => arm(ctx));
+    pi.on('message_start', (_e, ctx) => arm(ctx));
+    pi.on('message_update', (_e, ctx) => arm(ctx));
+    pi.on('message_end', (_e, ctx) => arm(ctx));
+    pi.on('tool_execution_start', (_e, ctx) => {
+        liveCtx = ctx;
+        watchdog.suspend();
+    });
+    pi.on('tool_execution_update', (_e, ctx) => {
+        liveCtx = ctx;
+    });
+    pi.on('tool_execution_end', (_e, ctx) => {
+        liveCtx = ctx;
+        watchdog.resume();
+    });
+    // Nothing is streaming between agent loops; stop so no timer can fire into an
+    // idle session (which would abort nothing and post a reminder to no one).
+    const stop = () => {
+        watchdog.stop();
+        liveCtx = undefined;
+    };
+    pi.on('agent_end', stop);
+    pi.on('session_shutdown', stop);
+}

package/dist/workers/pi-worker-core.d.ts

@@ -74,6 +74,16 @@ export interface RunWorkerInput {
         afterMs?: number;
         probe?: () => Promise<boolean>;
     } | false;
+    /**
+     * Stream-inactivity ceiling in ms (shared/stream-watchdog.ts). The stall guard
+     * above cannot catch a HUNG stream on a HEALTHY backend — it reads a reachable
+     * endpoint as proof of life, which is exactly what run 14's three hangs looked
+     * like. This one asks nothing of the backend: no output for this long (with
+     * tool executions excluded) ⇒ kill and restart the attempt with
+     * {@link streamStallHint}, inside the same shared restart budget.
+     * 0 / omitted = off.
+     */
+    streamInactivityMs?: number;
 }
 export interface RunWorkerResult {
     text: string;
@@ -129,6 +139,16 @@ export interface RunWorkerResult {
         toolName: string;
         timeoutMs: number;
     };
+    /**
+     * Set when the stream watchdog killed the worker's FINAL attempt: the model
+     * stream produced nothing for the configured window while no tool was running.
+     * Like loopHit/timedOut the text is partial — treat as a failure, and check it
+     * BEFORE `aborted` (the kill aborts too), or a hung backend is mislabeled a
+     * user cancel.
+     */
+    streamStalled?: {
+        idleMs: number;
+    };
 }
 /**
  * The per-command ceiling for attempt N, halving each time a hang recurs.

package/dist/workers/pi-worker-core.js

@@ -6,6 +6,7 @@ import { LoopDetector } from '../task/loop-detector.js';
 import { LOOP_WINDOW, LOOP_THRESHOLD, MAX_LOOP_RESTARTS, formatLoopHint } from '../task/child-runner.js';
 import { detectLeakedToolCall, leakedToolCallHint, MAX_LEAK_RETRIES } from '../shared/leaked-tool-call.js';
 import { discoverModelEndpoints, probeModelEndpoints } from '../shared/model-endpoint.js';
+import { streamStallHint } from '../shared/stream-watchdog.js';
 // `--mode json` makes pi emit structured events as they happen instead of
 // buffering the assistant text and flushing on exit. That matters for the
 // wait/work timing split: in text mode the first stdout chunk only arrives at
@@ -207,6 +208,9 @@ export async function runWorker(input) {
                                 ?? (() => probeModelEndpoints(discoverModelEndpoints()))
                         }
                     }),
+                ...(input.streamInactivityMs ?
+                    { streamInactivityMs: input.streamInactivityMs }
+                    : {}),
                 onFirstByte: () => (tFirstByte = Date.now()),
                 onToolCall: call => {
                     cmdWatch?.onStart(call);
@@ -240,6 +244,7 @@ export async function runWorker(input) {
         const text = result.text ?? '';
         const timedOut = timeout.timedOut();
         const commandKill = cmdWatch?.killed();
+        const streamStalled = result.streamStalled;
         // A loop-kill gets the same restart-with-hint treatment every other phase
         // already gets (runPhaseWithLoopGuard) — name the offending call so the
         // re-spawn avoids it. Bounded by the shared restart budget.
@@ -265,6 +270,14 @@ export async function runWorker(input) {
             hangKills++;
             continue;
         }
+        // A hung model stream is restartable on the same budget. Checked before
+        // the wall-clock timeout because it is the more specific diagnosis (and
+        // its hint does not blame the model: nothing it did caused the hang).
+        if (streamStalled && !loopHit && restarts < MAX_LOOP_RESTARTS) {
+            hint = streamStallHint(streamStalled.idleMs);
+            restarts++;
+            continue;
+        }
         // A wall-clock timeout (the backstop for varied thrash the exact-match
         // detector misses) is also restartable, sharing the same budget. Skip when
         // a loop also tripped — the loop hint above is more specific.
@@ -293,6 +306,7 @@ export async function runWorker(input) {
             ...(loopHit ? { loopHit } : {}),
             ...(timedOut ? { timedOut: true } : {}),
             ...(result.stalled ? { stalled: true } : {}),
+            ...(streamStalled ? { streamStalled } : {}),
             ...(commandKill ?
                 {
                     commandTimedOut: {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@mjasnikovs/pi-task",
-    "version": "0.18.36",
+    "version": "0.18.38",
     "description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
     "type": "module",
     "main": "./dist/index.js",