mtech-ai-local-runner 0.1.39 → 0.1.41

package/README.md

@@ -1,3 +1,18 @@
+## 0.1.41: confirm a cleanup the guardian finished before a late terminate
+
+When Codex exits by itself, the owned guardian confirms the cleanup and exits. A busy daemon could then write its
+terminate request to the closed control socket before reading `cleaned`. That write error turned a confirmed cleanup
+into an unconfirmed one and suspended the job. A control socket error alone no longer does that: cleanup is
+confirmed by `cleaned` or by the guardian's exit code 0, which it uses only after sending `cleaned`. Timeouts,
+liveness errors, malformed messages and `unconfirmed` reports still fail closed.
+
+## 0.1.40: say why the owned guardian could not confirm cleanup
+
+When the owned guardian reports an unconfirmed cleanup, the `executor_cleanup` diagnostic now carries fixed
+`cleanup_guardian_*` observations: what started the cleanup, how TERM and KILL went, whether the executor group was
+still alive and its pipes still open, whether the root had exited, and whether the host's deadline passed. Cleanup
+decisions, deadlines and recovery are unchanged.
+
 ## 0.1.39: keep the first failure when executor cleanup is unconfirmed
 
 A job whose Codex executor could not confirm its cleanup was suspended with only `EXECUTION_UNACKNOWLEDGED` in the

package/dist/src/adapters/codex/exec-launch-host.js

@@ -2,7 +2,7 @@ import { performance } from 'node:perf_hooks';
 import { PreparationClock, CODEX_PREPARATION_BUDGET_MS } from '../../safety/preparation-timing.js';
 import { localEndpointObservations, localEndpointReadObservations } from '../../safety/local-tool-server.js';
 import { processStartupObservations, freezeProcessLaunchObservation } from '../../process/node-process-group-host.js';
-import { executionError, unconfirmedCleanupError } from '../../safety/execution-failure.js';
+import { executionError, unconfirmedCleanupError, cleanupObservations } from '../../safety/execution-failure.js';
 import { UNMANAGED_EXEC_MODE } from './unmanaged-scope.js';
 import { fingerprint as taskFingerprint } from '../../daemon/execution-coordinator.js';
 import { securityRequirementsFingerprint as hash } from '../../safety/security-requirements.js';
@@ -308,8 +308,10 @@ export class CodexExecLaunchHost {
                 else
                     this.handle?.kill('SIGKILL');
             }
-            catch {
+            catch (error) {
                 failed.add('cleanup_process_terminate_failed');
+                for (const observed of cleanupObservations(error))
+                    failed.add(observed);
             }
             try {
                 await this.deps.closeTools();

package/dist/src/process/node-process-group-host.js

@@ -1,5 +1,6 @@
 import path from 'node:path';
 import { OwnedLaunchObserver } from './owned-launch-observer.js';
+import { unconfirmedCleanupError } from '../safety/execution-failure.js';
 const startup = new WeakMap();
 const launchObservers = new WeakMap();
 export function freezeProcessLaunchObservation(handle) { return launchObservers.get(handle)?.freeze(); }
@@ -142,7 +143,7 @@ export class NodeProcessGroupHost {
                 return;
             realExit = true;
             observations.add('host_process_exited');
-            void guardian.terminate().then(() => { owned = false; const result = guardian.outcome(); terminated(result?.code ?? null, result?.signal ?? null); }, () => { failure = new Error('Provider process group cleanup could not be confirmed'); terminated(null, null); notify(); });
+            void guardian.terminate().then(() => { owned = false; const result = guardian.outcome(); terminated(result?.code ?? null, result?.signal ?? null); }, () => { failure = unconfirmedCleanupError('Provider process group cleanup could not be confirmed', guardian.cleanupObservations?.() ?? []); terminated(null, null); notify(); });
         });
         // Drain from spawn, not from the first adapter event read: official exec can
         // emit startup events before it initializes MCP, while the adapter awaits
@@ -266,7 +267,18 @@ export class NodeProcessGroupHost {
                         clearTimeout(timer);
                 }
             },
-            terminateOwned: () => cleanupPromise ??= (async () => { launch?.stop(); stopInput(); await guardian.terminate(); owned = false; })(),
+            terminateOwned: () => cleanupPromise ??= (async () => {
+                launch?.stop();
+                stopInput();
+                // Keep the fixed message; carry only the guardian's fixed cleanup observations.
+                try {
+                    await guardian.terminate();
+                }
+                catch {
+                    throw unconfirmedCleanupError('Provider process group cleanup could not be confirmed', guardian.cleanupObservations?.() ?? []);
+                }
+                owned = false;
+            })(),
             once: (_event, callback) => { if (termination) {
                 const ended = termination;
                 queueMicrotask(() => callback(ended.code, ended.signal));

package/dist/src/process/owned-process-guardian.js

@@ -12,30 +12,35 @@ let child, ownedPid, ownedGone=false, cleaning, ready=Promise.resolve(), rootExi
 const send=value=>{if(!control.destroyed)control.write(JSON.stringify(value)+'\n');};
 const delay=ms=>new Promise(resolve=>setTimeout(resolve,ms));
 const alive=()=>{if(!ownedPid||ownedGone)return false;try{process.kill(-ownedPid,0);return true;}catch(e){if(e.code==='ESRCH'){ownedGone=true;return false;}if(e.code==='EPERM')return true;throw Error('cleanup');}};
-const signal=s=>{if(!ownedPid)throw Error('ownership');if(ownedGone)return;try{process.kill(-ownedPid,s);}catch(e){if(e.code==='ESRCH'){ownedGone=true;return;}if(e.code==='EPERM'){if(!rootExited)child.kill(s);return;}throw Error('cleanup');}};
-function cleanup(immediate=false){
+const signal=s=>{if(!ownedPid)throw Error('ownership');if(ownedGone)return 'esrch';try{process.kill(-ownedPid,s);return 'sent';}catch(e){if(e.code==='ESRCH'){ownedGone=true;return 'esrch';}if(e.code==='EPERM'){if(!rootExited)child.kill(s);return 'eperm';}throw Error('cleanup');}};
+// Fixed words only: what started cleanup, each signal outcome and what was still open.
+function cleanup(immediate=false,trigger='terminate'){
  if(cleaning)return cleaning;
+ const detail={trigger,term:'not_needed',kill:'not_needed'};
  cleaning=(async()=>{
   try{
    if(!child)throw Error('ownership');
    await ready;
    child.stdin.end();
    if(ownedPid){
-    if(alive())signal(immediate?'SIGKILL':'SIGTERM');
+    if(alive())detail.term=signal(immediate?'SIGKILL':'SIGTERM');
     const deadline=Date.now()+(immediate?0:2000);
     while(alive()&&Date.now()<deadline)await delay(20);
-    if(alive())signal('SIGKILL');
+    if(alive())detail.kill=signal('SIGKILL');
     const killed=Date.now()+2000;
     while((alive()||!closed)&&Date.now()<killed)await delay(20);
     if(alive()||!closed)throw Error('cleanup');
    }
    send({type:'cleaned'});finish(0);
-  }catch{send({type:'unconfirmed'});finish(1);}
+  }catch{
+   let groupAlive=true;try{groupAlive=alive();}catch{/* an unknown liveness error stays alive */}
+   send({type:'unconfirmed',detail:{...detail,alive:groupAlive,closed,root:rootExited}});finish(1);
+  }
  })();
  return cleaning;
 }
 function finish(code){process.exitCode=code;process.stdin.destroy();liveness.destroy();control.end(()=>control.destroy());process.stdout.end();process.stderr.end();}
-function lost(){orphaned=true;if(configured)void cleanup();else finish(1);}
+function lost(){orphaned=true;if(configured)void cleanup(false,'lost');else finish(1);}
 liveness.on('data',()=>{});liveness.once('end',lost);liveness.once('error',lost);
 control.on('error',lost);control.on('end',lost);
 control.on('data',chunk=>{
@@ -47,19 +52,19 @@ control.on('data',chunk=>{
    configured=true;
    try{child=spawn(message.command,message.args,{cwd:message.cwd,env:message.env,shell:false,detached:true,stdio:['pipe','pipe','pipe']});}catch{send({type:'spawn_failed'});finish(0);return;}
    ready=new Promise(resolve=>{
-    child.once('spawn',()=>{ownedPid=child.pid;send({type:'spawned',pid:ownedPid,parentPid:process.pid});resolve();if(orphaned)void cleanup();});
-    child.once('error',()=>{send({type:'spawn_failed'});closed=true;resolve();void cleanup(true);});
+    child.once('spawn',()=>{ownedPid=child.pid;send({type:'spawned',pid:ownedPid,parentPid:process.pid});resolve();if(orphaned)void cleanup(false,'lost');});
+    child.once('error',()=>{send({type:'spawn_failed'});closed=true;resolve();void cleanup(true,'spawn_error');});
    });
    child.stdin.on('error',()=>{});
    process.stdin.pipe(child.stdin);child.stdout.pipe(process.stdout,{end:false});child.stderr.pipe(process.stderr,{end:false});
-   child.once('exit',(code,signal)=>{rootExited=true;send({type:'root_exit',code,signal});void cleanup();});
+   child.once('exit',(code,signal)=>{rootExited=true;send({type:'root_exit',code,signal});void cleanup(false,'root_exit');});
    child.once('close',()=>{closed=true;});
-  }else if(message.type==='terminate')void cleanup(message.immediate===true);
+  }else if(message.type==='terminate')void cleanup(message.immediate===true,'terminate');
   else lost();
  }
 });
-process.stdout.on('error',()=>{child?.stdout.resume();void cleanup(true);});
-process.stderr.on('error',()=>{child?.stderr.resume();void cleanup(true);});
+process.stdout.on('error',()=>{child?.stdout.resume();void cleanup(true,'output_error');});
+process.stderr.on('error',()=>{child?.stderr.resume();void cleanup(true,'output_error');});
 `;
 export function spawnOwnedGuardian(command, args, options) {
     const child = spawn(process.execPath, ['--input-type=commonjs', '-e', GUARDIAN_SOURCE], {
@@ -73,13 +78,18 @@ export function spawnOwnedGuardian(command, args, options) {
     // Hosts can attach observers after spawn returns without unhandled rejection.
     void started.catch(() => { });
     let confirmed = false, spawnFailed = false, protocolFailed = false, buffer = '';
+    let terminateWritten = false, hostTimeout = false, controlFailed = false, detail;
     let outcome;
     let closePromise;
     const closed = new Promise(resolve => child.once('close', () => { liveness.destroy(); resolve(); }));
+    // A guardian that has confirmed and exited closes its control socket; a terminate written late by a
+    // busy host then fails, possibly before "cleaned" is read. That write error alone is not a protocol
+    // failure: the guardian exits 0 only after it sent "cleaned" (or after a spawn failure owned nothing).
     control.on('error', () => { if (!confirmed && !spawnFailed)
-        protocolFailed = true; });
+        controlFailed = true; });
     liveness.on('error', () => { if (!confirmed && !spawnFailed)
         protocolFailed = true; });
+    const cleanedByExit = () => controlFailed && !protocolFailed && child.exitCode === 0 && child.signalCode === null;
     control.on('data', (chunk) => {
         buffer += chunk;
         if (buffer.length > 8192) {
@@ -109,8 +119,11 @@ export function spawnOwnedGuardian(command, args, options) {
                 outcome = { code: message.code, signal: message.signal };
             else if (message.type === 'cleaned')
                 confirmed = true;
-            else if (message.type === 'unconfirmed')
+            else if (message.type === 'unconfirmed') {
                 protocolFailed = true;
+                if (message.detail && typeof message.detail === 'object' && !Array.isArray(message.detail))
+                    detail = message.detail;
+            }
             else {
                 protocolFailed = true;
                 liveness.destroy();
@@ -121,20 +134,55 @@ export function spawnOwnedGuardian(command, args, options) {
     child.once('close', () => { rejectStarted(new Error('Provider process could not start')); });
     control.write(JSON.stringify({ type: 'launch', command, args, ...options }) + '\n');
     return { child, started, closed, outcome: () => outcome, spawnFailed: () => spawnFailed,
-        signal(immediate) { if (child.exitCode === null && child.signalCode === null && !confirmed && !spawnFailed && !control.destroyed)
-            control.write(JSON.stringify({ type: 'terminate', immediate }) + '\n'); },
+        signal(immediate) { if (child.exitCode === null && child.signalCode === null && !confirmed && !spawnFailed && !control.destroyed) {
+            control.write(JSON.stringify({ type: 'terminate', immediate }) + '\n');
+            terminateWritten = true;
+        } },
         terminate(immediate = false) {
             if (closePromise)
                 return closePromise;
             this.signal(immediate);
             closePromise = new Promise((resolve, reject) => {
-                const timer = setTimeout(() => { protocolFailed = true; liveness.destroy(); reject(new Error('Provider process group cleanup could not be confirmed')); }, 5500);
-                void closed.then(() => { clearTimeout(timer); if (protocolFailed || !confirmed)
+                const timer = setTimeout(() => { protocolFailed = true; hostTimeout = true; liveness.destroy(); reject(new Error('Provider process group cleanup could not be confirmed')); }, 5500);
+                void closed.then(() => { clearTimeout(timer); if (protocolFailed || !(confirmed || cleanedByExit()))
                     reject(new Error('Provider process group cleanup could not be confirmed'));
                 else
                     resolve(); });
             });
             return closePromise;
         },
+        /** Fixed cleanup observations after an unconfirmed cleanup; empty once cleanup is confirmed. */
+        cleanupObservations() {
+            if ((confirmed || cleanedByExit()) && !protocolFailed)
+                return [];
+            if (!protocolFailed && !hostTimeout && !controlFailed)
+                return [];
+            const observed = [terminateWritten ? 'cleanup_guardian_terminate_written' : 'cleanup_guardian_terminate_not_written'];
+            if (hostTimeout)
+                observed.push('cleanup_guardian_host_timeout');
+            if (controlFailed)
+                observed.push('cleanup_guardian_control_failed');
+            if (!detail) {
+                if (!hostTimeout && protocolFailed)
+                    observed.push('cleanup_guardian_protocol_failed');
+                return observed;
+            }
+            const word = (value, allowed) => allowed.find(item => item === value);
+            const trigger = word(detail.trigger, ['terminate', 'root_exit', 'lost', 'output_error', 'spawn_error']);
+            const term = word(detail.term, ['sent', 'eperm', 'esrch', 'not_needed']), kill = word(detail.kill, ['sent', 'eperm', 'esrch', 'not_needed']);
+            if (trigger)
+                observed.push(`cleanup_guardian_trigger_${trigger}`);
+            if (term)
+                observed.push(`cleanup_guardian_term_${term}`);
+            if (kill)
+                observed.push(`cleanup_guardian_kill_${kill}`);
+            if (typeof detail.alive === 'boolean')
+                observed.push(detail.alive ? 'cleanup_guardian_group_alive' : 'cleanup_guardian_group_gone');
+            if (typeof detail.closed === 'boolean')
+                observed.push(detail.closed ? 'cleanup_guardian_pipes_closed' : 'cleanup_guardian_pipes_open');
+            if (typeof detail.root === 'boolean')
+                observed.push(detail.root ? 'cleanup_guardian_root_exited' : 'cleanup_guardian_root_running');
+            return observed;
+        },
     };
 }

package/dist/src/safety/execution-failure.js

@@ -19,6 +19,13 @@ export const STARTUP_OBSERVATIONS = Object.freeze([
     'cleanup_official_exec_close_failed', 'cleanup_official_exec_close_pending',
     'cleanup_exec_tools_revoke_failed', 'cleanup_job_tools_revoke_failed', 'cleanup_stdin_end_failed', 'cleanup_process_terminate_failed',
     'cleanup_tool_server_close_failed', 'cleanup_timeout',
+    // The owned guardian's own account of an unconfirmed cleanup (fixed words, no PIDs or paths).
+    'cleanup_guardian_terminate_written', 'cleanup_guardian_terminate_not_written', 'cleanup_guardian_host_timeout', 'cleanup_guardian_control_failed', 'cleanup_guardian_protocol_failed',
+    'cleanup_guardian_trigger_terminate', 'cleanup_guardian_trigger_root_exit', 'cleanup_guardian_trigger_lost', 'cleanup_guardian_trigger_output_error', 'cleanup_guardian_trigger_spawn_error',
+    'cleanup_guardian_term_sent', 'cleanup_guardian_term_eperm', 'cleanup_guardian_term_esrch', 'cleanup_guardian_term_not_needed',
+    'cleanup_guardian_kill_sent', 'cleanup_guardian_kill_eperm', 'cleanup_guardian_kill_esrch', 'cleanup_guardian_kill_not_needed',
+    'cleanup_guardian_group_alive', 'cleanup_guardian_group_gone', 'cleanup_guardian_pipes_closed', 'cleanup_guardian_pipes_open',
+    'cleanup_guardian_root_exited', 'cleanup_guardian_root_running',
 ]);
 const issued = new WeakMap();
 export function executionFailure(error, stage, fallback = 'EXECUTION_FAILED') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mtech-ai-local-runner",
-  "version": "0.1.39",
+  "version": "0.1.41",
   "private": false,
   "license": "UNLICENSED",
   "bin": {