@north-light/crouter 0.3.168 → 0.3.170

package/dist/daemon/fleet.js

@@ -352,6 +352,10 @@ export class DaemonFleet {
             reviveNode(nodeId, {
                 resume: action === 'respawn-resume',
                 wakeReason: cleanAbort ? 'runtime-restart-abort' : undefined,
+                // Replacing a dead broker instance is recovery, not a wake: nothing
+                // external arrived for this node, so an armed deadline survives the
+                // relaunch instead of being consumed by it.
+                recovery: true,
             });
         }
         catch (err) {

package/dist/pi-extensions/canvas-bash-valve.js

@@ -27,7 +27,7 @@
 import { spawn } from 'node:child_process';
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmSync, statSync, writeFileSync, } from 'node:fs';
 import { homedir } from 'node:os';
-import { bashJobPaths, formatBashElapsed, newBashJobId } from '../core/bash-jobs.js';
+import { backgroundBashJob, bashJobPaths, formatBashElapsed, newBashJobId } from '../core/bash-jobs.js';
 import { createBashToolDefinition } from '@earendil-works/pi-coding-agent';
 // ---------------------------------------------------------------------------
 // Deadline — 5 minutes by default, overridable via CRTR_BASH_VALVE_MS
@@ -49,7 +49,8 @@ function resolveValveMs() {
 //
 // Positional argv (never interpolated into the script — the command text
 // lives only in cmd.sh):
-//   $1 cmd.sh   $2 job.log   $3 job.exit   $4 job.bg   $5 jobId   $6 nodeId
+//   $1 cmd.sh     $2 job.log   $3 job.exit   $4 job.run
+//   $5 job.done   $6 job.bg    $7 jobId      $8 nodeId
 //
 // `--` before the paths is a dummy $0 (a standard `bash -c script -- args`
 // idiom) so the real payload lands at $1 onward.
@@ -58,13 +59,38 @@ const SUPERVISOR_SCRIPT = [
     'bash "$1" > "$2" 2>&1',
     'ec=$?',
     'echo "$ec" > "$3"',
-    'if [ -f "$4" ]; then',
-    '  crtr node message send "Background bash job $5 finished (exit $ec). Log: $2  Command: $1" --to "$6" --tier urgent || true',
+    'if mv "$4" "$5" 2>/dev/null; then',
+    '  :',
+    'elif [ -f "$6" ]; then',
+    '  crtr node message send "Background bash job $7 finished (exit $ec). Log: $2  Command: $1" --to "$8" --tier urgent || true',
     'fi',
 ].join('\n');
 function makeJobPaths(contextDir) {
     return bashJobPaths(contextDir, newBashJobId());
 }
+const liveForegroundJobs = new Set();
+let terminationHandoffInstalled = false;
+/** The broker is about to exit. Persist every foreground handoff synchronously
+ * before pi's own termination listener aborts its active tool calls. */
+function backgroundLiveJobsForTermination() {
+    for (const paths of [...liveForegroundJobs]) {
+        try {
+            backgroundBashJob(paths);
+            liveForegroundJobs.delete(paths);
+        }
+        catch (err) {
+            process.stderr.write(`crtr: failed to background bash job ${paths.jobId} during broker termination: ${err.message}\n`);
+        }
+    }
+}
+function installTerminationHandoff() {
+    if (terminationHandoffInstalled)
+        return;
+    terminationHandoffInstalled = true;
+    process.prependListener('SIGTERM', backgroundLiveJobsForTermination);
+    if (process.platform !== 'win32')
+        process.prependListener('SIGHUP', backgroundLiveJobsForTermination);
+}
 function killGroup(pgid) {
     try {
         process.kill(-pgid, 'SIGTERM');
@@ -194,6 +220,7 @@ export function createValveOperations(nodeId, contextDir) {
                 mkdirSync(paths.dir, { recursive: true });
                 writeFileSync(paths.cmdSh, command);
                 writeFileSync(paths.jobLog, '');
+                writeFileSync(paths.jobRun, '');
                 let offset = 0;
                 let settled = false;
                 let pgid;
@@ -205,6 +232,7 @@ export function createValveOperations(nodeId, contextDir) {
                         clearInterval(interval);
                     if (signal)
                         signal.removeEventListener('abort', onAbort);
+                    liveForegroundJobs.delete(paths);
                 };
                 const finish = (act) => {
                     if (settled)
@@ -213,7 +241,7 @@ export function createValveOperations(nodeId, contextDir) {
                     teardown();
                     act();
                 };
-                function background(auto = false) {
+                function finishBackground(auto = false) {
                     const elapsed = formatBashElapsed(Date.now() - startedAt);
                     const notice = `\n[backgrounded as job ${paths.jobId} after ${elapsed} in the foreground${auto ? ' (auto)' : ''}. ` +
                         `It will interrupt you with a message when it exits. ` +
@@ -222,8 +250,35 @@ export function createValveOperations(nodeId, contextDir) {
                     finish(() => resolve({ exitCode: 0 }));
                     // Backgrounded — job dir persists (inspectable); no cleanup here.
                 }
+                function finishForeground() {
+                    offset = tailOnce(paths.jobLog, offset, onData);
+                    const exitCode = readExitCode(paths.jobExit);
+                    finish(() => resolve({ exitCode }));
+                    cleanupJobDir(paths.dir);
+                }
+                function requestBackground(auto = false) {
+                    const state = backgroundBashJob(paths);
+                    if (state === 'backgrounded' || state === 'already-backgrounded') {
+                        finishBackground(auto);
+                        return;
+                    }
+                    if (state === 'finished') {
+                        finishForeground();
+                        return;
+                    }
+                    finish(() => reject(new Error(`bash job ${paths.jobId} disappeared before it could be backgrounded`)));
+                    cleanupJobDir(paths.dir);
+                }
                 function onAbort() {
-                    tailOnce(paths.jobLog, offset, onData);
+                    offset = tailOnce(paths.jobLog, offset, onData);
+                    if (existsSync(paths.jobBg)) {
+                        finishBackground();
+                        return;
+                    }
+                    // Once the command has written its exit code, let the supervisor
+                    // finish the job.run → job.done transition before cleanup.
+                    if (existsSync(paths.jobExit))
+                        return;
                     if (pgid)
                         killGroup(pgid);
                     finish(() => reject(new Error('aborted')));
@@ -234,7 +289,7 @@ export function createValveOperations(nodeId, contextDir) {
                 const execution = resolveExecutionCwd(cwd);
                 if (execution.warning)
                     onData(Buffer.from(execution.warning));
-                const child = spawn('bash', ['-c', SUPERVISOR_SCRIPT, '--', paths.cmdSh, paths.jobLog, paths.jobExit, paths.jobBg, paths.jobId, nodeId], {
+                const child = spawn('bash', ['-c', SUPERVISOR_SCRIPT, '--', paths.cmdSh, paths.jobLog, paths.jobExit, paths.jobRun, paths.jobDone, paths.jobBg, paths.jobId, nodeId], {
                     cwd: execution.cwd,
                     detached: true,
                     stdio: 'ignore',
@@ -242,27 +297,31 @@ export function createValveOperations(nodeId, contextDir) {
                 });
                 child.on('error', (err) => {
                     finish(() => reject(err));
+                    cleanupJobDir(paths.dir);
                 });
                 pgid = child.pid;
                 // The detached supervisor IS the group leader, so its pid is the pgid.
                 // Persisting it is what lets a human stop the job from the Inspector
                 // later; the agent's own handoff notice quotes the same number.
-                if (pgid !== undefined)
+                if (pgid !== undefined) {
                     writeFileSync(paths.jobPgid, String(pgid));
+                    liveForegroundJobs.add(paths);
+                }
                 child.unref();
                 interval = setInterval(() => {
                     offset = tailOnce(paths.jobLog, offset, onData);
-                    if (existsSync(paths.jobExit)) {
-                        offset = tailOnce(paths.jobLog, offset, onData);
-                        const exitCode = readExitCode(paths.jobExit);
-                        finish(() => resolve({ exitCode }));
-                        cleanupJobDir(paths.dir);
+                    if (existsSync(paths.jobDone)) {
+                        finishForeground();
                         return;
                     }
                     if (existsSync(paths.jobBg)) {
-                        background();
+                        finishBackground();
                         return;
                     }
+                    // job.exit is written before the supervisor claims job.done. Do not
+                    // time out or delete the directory during that in-progress exit.
+                    if (existsSync(paths.jobExit))
+                        return;
                     const elapsedMs = Date.now() - startedAt;
                     if (timeout !== undefined && timeout > 0) {
                         if (elapsedMs >= timeout * 1000) {
@@ -273,10 +332,8 @@ export function createValveOperations(nodeId, contextDir) {
                         }
                         return;
                     }
-                    if (elapsedMs >= valveMs) {
-                        writeFileSync(paths.jobBg, '');
-                        background(true);
-                    }
+                    if (elapsedMs >= valveMs)
+                        requestBackground(true);
                 }, POLL_MS);
             });
         },
@@ -292,6 +349,7 @@ export default function (pi) {
     const contextDir = process.env['CRTR_CONTEXT_DIR'];
     if (contextDir === undefined || contextDir.trim() === '')
         return; // defensive; always set alongside CRTR_NODE_ID
+    installTerminationHandoff();
     // Registered on session_start (fires for startup, new, resume, fork, and
     // /reload) so we get ctx.cwd — the same session cwd the builtin bash tool
     // is built against. registerTool replaces the builtin by name; re-firing on

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.168",
+  "version": "0.3.170",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.168",
+  "version": "0.3.170",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/crouter",
-      "version": "0.3.168",
+      "version": "0.3.170",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {

package/scripts/install-runtime.mjs

@@ -25,6 +25,17 @@ function run(command, args, cwd, capture = false, env = process.env) {
   });
 }
 
+/** Capture a probe's stdout even when a candidate process exits during the probe. */
+function probe(command, args, cwd) {
+  return new Promise((resolveRun) => {
+    const child = spawn(command, args, { cwd, stdio: ['ignore', 'pipe', 'ignore'] });
+    let output = '';
+    child.stdout.on('data', (chunk) => { output += chunk; });
+    child.on('error', () => resolveRun(output));
+    child.on('exit', () => resolveRun(output));
+  });
+}
+
 function digest(value) {
   return createHash('sha256').update(value).digest('hex');
 }
@@ -146,32 +157,44 @@ async function pack(archiveDir) {
   return join(archiveDir, packed[0].filename);
 }
 
+/** Generation ids named anywhere in a `ps` or `lsof` dump. */
 function generationIdsIn(text) {
-  return [...text.matchAll(/generations\/([a-f0-9]{64})(?:\/|$)/g)].map((match) => match[1]);
+  return [...text.matchAll(/generations\/([a-f0-9]{64})(?:[/\s]|$)/g)].map((match) => match[1]);
 }
 
+/** Return every generation used by a running process and whether every stable
+ *  selector process exposed its generation lease. */
 async function liveGenerationIds() {
   const output = await run('ps', ['-axww', '-o', 'pid=,command='], RUNTIME_HOME, true);
   const live = new Set(generationIdsIn(output));
   const nodePids = [];
-  let uncertain = false;
+  const opaqueStablePids = new Set();
   for (const line of output.split('\n')) {
     const match = /^\s*(\d+)\s+(.*)$/.exec(line);
     if (!match) continue;
     const [, pid, command] = match;
-    if (/(?:^|\s)\S*\/bin\/(?:crtr|crtrd|crouter)(?:\s|$)/.test(command)) uncertain = true;
+    const stableSelector = /^(?:\S*node\s+)?\S*\/bin\/(?:crtr|crtrd|crouter)(?:\s|$)/.test(command);
+    if (stableSelector && generationIdsIn(command).length === 0) opaqueStablePids.add(pid);
     if (/(?:^|[\s/])(?:node|crtrd?|crouter)(?:[\s]|$)/.test(command)) nodePids.push(pid);
   }
-  // An opaque stable shim means GC will skip regardless, so do not spend
-  // minutes resolving process cwd values that cannot change that decision.
-  if (uncertain || nodePids.length === 0) return { live, uncertain };
+  if (nodePids.length === 0) return { live, uncertain: opaqueStablePids.size > 0 };
 
-  // argv identifies entry modules; cwd covers a node process launched from a
-  // generation directory. Ask lsof for every candidate in ONE invocation:
-  // starting lsof once per process costs ~0.5s on macOS, which turned hundreds
-  // of live agent/tool processes into a multi-minute post-install stall.
-  const cwd = await run('lsof', ['-n', '-P', '-a', '-p', nodePids.join(','), '-d', 'cwd', '-Fn'], RUNTIME_HOME, true);
-  for (const id of generationIdsIn(cwd)) live.add(id);
+  // One lsof invocation resolves cwd references and the runtime entry-file
+  // lease held by each stable CLI process without a per-process startup penalty.
+  const open = await probe('lsof', ['-n', '-P', '-a', '-p', nodePids.join(','), '-Fn'], RUNTIME_HOME);
+  let pid;
+  const exposedStablePids = new Set();
+  for (const line of open.split('\n')) {
+    const processMatch = /^p(\d+)$/.exec(line);
+    if (processMatch) {
+      pid = processMatch[1];
+      continue;
+    }
+    const ids = generationIdsIn(line);
+    for (const id of ids) live.add(id);
+    if (pid !== undefined && ids.length > 0 && opaqueStablePids.has(pid)) exposedStablePids.add(pid);
+  }
+  const uncertain = [...opaqueStablePids].some((stablePid) => !exposedStablePids.has(stablePid));
   return { live, uncertain };
 }
 
@@ -186,7 +209,7 @@ async function pruneGenerations() {
   const selected = await selectedGenerationId();
   const { live, uncertain } = await liveGenerationIds();
   if (uncertain) {
-    process.stderr.write('runtime generation GC skipped: a live stable crouter shim does not expose its generation path\n');
+    process.stderr.write('runtime generation GC skipped: a live stable crouter shim does not expose its generation lease\n');
     return;
   }
   const entries = await readdir(GENERATIONS, { withFileTypes: true });