@mjasnikovs/pi-task 0.18.26 → 0.18.28

package/dist/workers/pi-worker-core.js

@@ -1,5 +1,6 @@
 import { getPiInvocation } from '../shared/pi-invocation.js';
 import { runChildDefault } from '../shared/child-process.js';
+import { CommandWatchdog, commandTimeoutHint, realTimerDeps } from '../shared/command-watchdog.js';
 import { childBaseArgs } from '../shared/child-extensions.js';
 import { LoopDetector } from '../task/loop-detector.js';
 import { LOOP_WINDOW, LOOP_THRESHOLD, MAX_LOOP_RESTARTS, formatLoopHint } from '../task/child-runner.js';
@@ -32,7 +33,12 @@ const RESEARCH_WORKER_TIMEOUT_MS = 240_000;
  * alive) just gets probed and waits on.
  */
 const STALL_AFTER_MS = 180_000;
-/** Restart hint after a wall-clock timeout — distinct from the loop hint. */
+/**
+ * Restart hint after a WHOLE-WORKER wall-clock timeout — distinct from both the
+ * loop hint and the per-command hint. This one diagnoses over-exploration, which
+ * is what the whole-worker cap actually catches. A single hung COMMAND is a
+ * different fault with a different fix, and gets commandTimeoutHint instead.
+ */
 const WORKER_TIMEOUT_HINT = '[SYSTEM NOTE: Your previous attempt ran out of time before answering — you '
     + 'were exploring too long. Be decisive: do the minimum reads/greps needed, '
     + 'then write your answer now. Do not re-explore ground you have already covered.]';
@@ -69,6 +75,85 @@ function workerTimeout(external, ms) {
         }
     };
 }
+/**
+ * The per-command ceiling for attempt N, halving each time a hang recurs.
+ *
+ * The first attempt gets the full configured ceiling — a genuinely slow build or
+ * test suite deserves it. But every hang-caused restart carries
+ * commandTimeoutHint, which tells the model in as many words to bound its
+ * command; a SECOND hang means it ignored an explicit instruction, and a third
+ * means it ignored it twice. Giving a non-complying child the full ceiling again
+ * would put the worst case at 3 × 15 min = 45 minutes of dead time, resting
+ * entirely on the model obeying prose. Halving bounds it at ~26 min while
+ * costing a complying child nothing.
+ *
+ * `priorHangs` counts watchdog kills specifically, NOT total restarts — the
+ * restart budget is shared with loop kills, and a child restarted for LOOPING
+ * never received the bound-your-command hint, so its first hang still deserves
+ * the full ceiling. Only a hang after a hang is defiance.
+ *
+ * Floored at 30s so repeated halving cannot shrink the ceiling to something no
+ * real command could finish inside — but never ABOVE the configured ceiling
+ * itself, or a caller asking for 10s would silently get 30.
+ */
+export function commandCeilingForAttempt(baseMs, priorHangs) {
+    if (!(baseMs > 0))
+        return 0;
+    const floor = Math.min(baseMs, 30_000);
+    return Math.max(floor, Math.round(baseMs / 2 ** priorHangs));
+}
+/**
+ * Build the child-side command watchdog for ONE attempt: a per-tool-call timer
+ * machine (shared with the main session) whose `onFire` aborts `signal`, which
+ * runChild turns into a process-GROUP kill — reaping the hung command itself,
+ * not just the pi child holding it.
+ *
+ * LIMIT: the group kill only reaches processes still IN the group. A hung
+ * command that detached a daemon (setsid/nohup dev server) leaves it running —
+ * the fresh attempt can then hit a port the dead attempt's escapee still holds
+ * (the run-9 orphan-dev-server → false-EADDRINUSE shape). No cheap fix from
+ * here; the restart hint's "check current state" line is the mitigation.
+ *
+ * Returns null when the watchdog is off, so the caller keeps the plain timeout
+ * signal and no per-call bookkeeping happens at all.
+ */
+function commandWatch(timeoutMs) {
+    if (!(timeoutMs > 0))
+        return null;
+    const ctrl = new AbortController();
+    // pi's toolCallId pairs start↔end. When it is absent (a fake stream in a
+    // test, an older pi), fall back to one shared slot: tool executions in a
+    // child are sequential, so a single slot is still correctly paired.
+    const key = (id) => id ?? 'anon';
+    const details = new Map();
+    let killed;
+    const watchdog = new CommandWatchdog({
+        getTimeoutMs: () => timeoutMs,
+        ...realTimerDeps,
+        onFire: (toolCallId, toolName, ms) => {
+            killed = {
+                toolName,
+                timeoutMs: ms,
+                ...(details.has(toolCallId) ? { detail: details.get(toolCallId) } : {})
+            };
+            ctrl.abort();
+        }
+    });
+    return {
+        onStart: call => {
+            const id = key(call.toolCallId);
+            const args = call.args;
+            if (typeof args?.command === 'string') {
+                details.set(id, args.command.slice(0, 120));
+            }
+            watchdog.onStart(id, call.name);
+        },
+        onEnd: id => watchdog.onEnd(key(id)),
+        killed: () => killed,
+        signal: ctrl.signal,
+        clear: () => watchdog.clearAll()
+    };
+}
 export async function runWorker(input) {
     const tools = input.tools ?? DEFAULT_TOOLS;
     const baseArgs = [...childBaseArgs(input.extensions ?? []), '--mode', 'json', '--tools', tools];
@@ -79,6 +164,10 @@ export async function runWorker(input) {
     // hint up to MAX_LOOP_RESTARTS times before we give up. Leaked tool calls
     // keep their own MAX_LEAK_RETRIES budget below — a different failure mode.
     let restarts = 0;
+    // Watchdog kills specifically — drives the ceiling halving. Kept apart from
+    // `restarts` (the shared budget) so a loop-caused restart doesn't shorten
+    // the rope of a child that has never hung (see commandCeilingForAttempt).
+    let hangKills = 0;
     let leakRetries = 0;
     for (;;) {
         const prompt = hint === null ? input.prompt : `${hint}\n\n${input.prompt}`;
@@ -101,9 +190,13 @@ export async function runWorker(input) {
         // caller couldn't distinguish from a crash.
         let loopHit;
         const timeout = workerTimeout(input.signal, timeoutMs);
+        // Per-tool-call watchdog for this attempt (null when off). Its abort is
+        // OR'd with the worker timeout / external cancel into the child's signal.
+        const cmdWatch = commandWatch(commandCeilingForAttempt(input.commandTimeoutMs ?? 0, hangKills));
+        const childSignal = cmdWatch ? AbortSignal.any([timeout.signal, cmdWatch.signal]) : timeout.signal;
         let result;
         try {
-            result = await runChildDefault(invocation, input.cwd, timeout.signal, {
+            result = await runChildDefault(invocation, input.cwd, childSignal, {
                 mode: 'json-events',
                 ...(input.stall === false ?
                     {}
@@ -116,6 +209,7 @@ export async function runWorker(input) {
                     }),
                 onFirstByte: () => (tFirstByte = Date.now()),
                 onToolCall: call => {
+                    cmdWatch?.onStart(call);
                     if (!loopDetector)
                         return null;
                     const hit = loopDetector.record(call);
@@ -124,18 +218,28 @@ export async function runWorker(input) {
                     return hit;
                 },
                 onLine: input.onLine,
-                onToolResult: input.onToolResult,
+                // Always wired when the watchdog is on — the sink only emits
+                // tool_execution_end if a handler exists, and without it every
+                // timer would stay armed and fire on a finished command.
+                onToolResult: cmdWatch ?
+                    r => {
+                        cmdWatch.onEnd(r.toolCallId);
+                        input.onToolResult?.(r);
+                    }
+                    : input.onToolResult,
                 onContextUsage: input.onContextUsage
             }, input.spawn);
         }
         finally {
             timeout.cleanup();
+            cmdWatch?.clear();
         }
         const tEnd = Date.now();
         const waitMs = tFirstByte === null ? tEnd - tStart : tFirstByte - tStart;
         const workMs = tFirstByte === null ? 0 : tEnd - tFirstByte;
         const text = result.text ?? '';
         const timedOut = timeout.timedOut();
+        const commandKill = cmdWatch?.killed();
         // A loop-kill gets the same restart-with-hint treatment every other phase
         // already gets (runPhaseWithLoopGuard) — name the offending call so the
         // re-spawn avoids it. Bounded by the shared restart budget.
@@ -144,6 +248,23 @@ export async function runWorker(input) {
             restarts++;
             continue;
         }
+        // A hung COMMAND is restartable too, on the same budget, but checked
+        // before the whole-worker timeout because its hint is the specific one:
+        // bound the command. (The two can't be confused — a watchdog kill leaves
+        // timeout.timedOut() false, since that flag tracks only its own timer.)
+        if (commandKill && !loopHit && restarts < MAX_LOOP_RESTARTS) {
+            hint = commandTimeoutHint(commandKill.toolName, commandKill.timeoutMs, {
+                commandDetail: commandKill.detail,
+                // Nothing reverts the tree between attempts, so a child that can
+                // mutate it (edit/write, or bash side effects) must not be told
+                // its previous attempt left no trace. Same capability test the
+                // gate logger uses — decided by tools, not by phase.
+                editsMayPersist: /\b(?:edit|bash|write)\b/.test(tools)
+            });
+            restarts++;
+            hangKills++;
+            continue;
+        }
         // A wall-clock timeout (the backstop for varied thrash the exact-match
         // detector misses) is also restartable, sharing the same budget. Skip when
         // a loop also tripped — the loop hint above is more specific.
@@ -171,7 +292,15 @@ export async function runWorker(input) {
             ...(leaked ? { leakedToolCall: leaked } : {}),
             ...(loopHit ? { loopHit } : {}),
             ...(timedOut ? { timedOut: true } : {}),
-            ...(result.stalled ? { stalled: true } : {})
+            ...(result.stalled ? { stalled: true } : {}),
+            ...(commandKill ?
+                {
+                    commandTimedOut: {
+                        toolName: commandKill.toolName,
+                        timeoutMs: commandKill.timeoutMs
+                    }
+                }
+                : {})
         };
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@mjasnikovs/pi-task",
-    "version": "0.18.26",
+    "version": "0.18.28",
     "description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
     "type": "module",
     "main": "./dist/index.js",