@jc_stack/ez-agents 0.1.0-beta.50 → 0.1.0-beta.52

package/src/updates/runtime.mjs

@@ -19,13 +19,14 @@ export function execute(command,args,options={}) {
   return new Promise((resolve,reject)=>{
     const invocation=process.env.EZ_DOCKER_COMPOSE==='standalone'&&command==='docker'&&args[0]==='compose'
       ? {command:'docker-compose',args:args.slice(1)} : {command,args};
-    const child=spawn(invocation.command,invocation.args,{cwd:options.cwd,env:environment(),stdio:['ignore','pipe','pipe']});let output='',stdout='';
+    const child=spawn(invocation.command,invocation.args,{cwd:options.cwd,env:environment(),stdio:[options.input===undefined?'ignore':'pipe','pipe','pipe']});let output='',stdout='';
     const timer=setTimeout(()=>child.kill('SIGKILL'),options.timeout||600000);
     const file=options.outputFile?createWriteStream(options.outputFile,{flags:'wx',mode:0o600}):null;
     const written=file?new Promise((resolve,reject)=>{file.once('finish',resolve);file.once('error',reject);}):Promise.resolve();
     if(file)child.stdout.pipe(file);else child.stdout.on('data',b=>{output=(output+b).slice(-16000);if(options.stdoutOnly)stdout=(stdout+b).slice(-16000);});
     child.stderr.on('data',b=>{output=(output+b).slice(-16000);});
     void written.catch(()=>child.kill('SIGKILL'));
+    if(options.input!==undefined){child.stdin.on('error',error=>{child.kill('SIGKILL');reject(error);});child.stdin.end(options.input);}
     child.once('error',error=>{clearTimeout(timer);reject(error);});
     child.once('close',async code=>{clearTimeout(timer);try{await written;if(code!==0)throw Error(`${invocation.command} failed (${code}): ${output}`);resolve(options.stdoutOnly?stdout:output);}catch(error){reject(error);}});
   });
@@ -33,8 +34,8 @@ export function execute(command,args,options={}) {
 export async function textAtomic(file,text) {
   const tmp=file+'.update.tmp';await fs.writeFile(tmp,text,{mode:0o600});await fs.rename(tmp,file);
 }
-export async function backupStateDirectory(source,destination) {
-  await fs.cp(source,destination,{recursive:true,filter:async entry=>!(await fs.lstat(entry)).isSocket()});
+export async function backupStateDirectory(source,destination,omit=[]) {
+  await fs.cp(source,destination,{recursive:true,filter:async entry=>!omit.includes(path.relative(source,entry).split(path.sep)[0])&&!(await fs.lstat(entry)).isSocket()});
 }
 export const pluginArgs = record => ['compose','--project-name',record.project,'--file',record.compose];
 export async function packageManager(root,run=execute) {
@@ -159,7 +160,7 @@ export async function perform(home,job,hooks) {
   const root=path.join(dir,'runtime');await fs.rm(root,{recursive:true,force:true});await extract(archive,root);
   const next=await eligibility(home,job.target,root,job.automatic);
   if(next.old.root!==job.previousRoot||next.old.pkg.version!==job.previousVersion)throw Error('Stale upgrade job');
-  job.status='applying';job.root=root;job.startedAt=new Date().toISOString();await save();
+  job.status='applying';job.preserveRuns=hooks.preserveRuns===true;job.root=root;job.startedAt=new Date().toISOString();await save();
   const complete=async()=>{
     job.status='completed';job.endedAt=new Date().toISOString();await save();
     try { job.cleanup=await cleanupStaleBackups(home);await save(); }
@@ -193,17 +194,30 @@ export async function perform(home,job,hooks) {
       });
       const image=`ez-upgrade-${digest(JSON.stringify([job.sha256,buildArgs])).slice(0,24)}`;
       await run('docker',['build','--target','runtime',...buildArgs,'-t',image,root]);
+      const backup=path.join(dir,'backup');
+      const backupState=async()=>{
+        await fs.mkdir(backup,{mode:0o700});
+        await backupStateDirectory(config.workspace,path.join(backup,'workspace'));
+        await backupStateDirectory(agent.controlDir,path.join(backup,'control'),hooks.preserveRuns ? ['host-executor'] : []);
+        for(const name of ['docker.env','host-executor.json','agent.json','purpose.md','relay.env'])await fs.copyFile(path.join(config.deploymentDir,name),path.join(backup,name));
+      };
+      // Native client state can be many gigabytes. This advisory copy never
+      // rewinds live state, so compatible generations keep serving throughout it.
+      if(hooks.preserveRuns)await backupState();
       const envFile=path.join(config.deploymentDir,'docker.env'),oldEnv=await fs.readFile(envFile,'utf8');
       const cid=(await run('docker',[...relayArgs(config),'ps','-q','relay'])).trim();
       if(!cid||/\s/.test(cid))throw Error('Expected one running relay');
       const oldImage=(await run('docker',['inspect','--format','{{.Image}}',cid])).trim();
       if(!/^sha256:[a-f0-9]{64}$/.test(oldImage))throw Error('Cannot pin rollback image');
       job.rollback={env:envValue(oldEnv,'EZ_RELAY_IMAGE',oldImage),packageRoot:config.packageRoot};await save();
-      await run('docker',[...relayArgs(config),'stop','relay']);await hooks.stopHost();
-      const backup=path.join(dir,'backup');await fs.mkdir(backup,{mode:0o700});
-      await backupStateDirectory(config.workspace,path.join(backup,'workspace'));
-      await backupStateDirectory(agent.controlDir,path.join(backup,'control'));
-      for(const name of ['docker.env','host-executor.json','agent.json','purpose.md','relay.env'])await fs.copyFile(path.join(config.deploymentDir,name),path.join(backup,name));
+      const activate=async()=>{
+        const current=await eligibility(home,job.target,root,job.automatic);
+        if(current.old.root!==job.previousRoot||current.old.pkg.version!==job.previousVersion)throw Error('Stale upgrade job');
+        const pause=path.join(agent.controlDir,'upgrade-pause.json');
+        if(hooks.preserveRuns)await atomic(pause,{id:job.id});
+        try {
+      await run('docker',[...relayArgs(config),'stop','relay']);await (hooks.preserveRuns ? hooks.retireHost() : hooks.stopHost());
+      if(!hooks.preserveRuns)await backupState();
       let env=oldEnv.split(job.previousRoot+path.sep).join(root+path.sep);env=envValue(env,'EZ_RELAY_IMAGE',image);
       await textAtomic(envFile,env);
       await bindUpdates(home,path.join(config.deploymentDir,'host-executor.json'),root);
@@ -213,6 +227,9 @@ export async function perform(home,job,hooks) {
         const brokerImage=(await run('docker',['image','inspect','--format','{{.Id}}',image],{stdoutOnly:true})).trim();
         job.brokerReadiness=await refreshBroker(config,run,brokerImage);
       }
+        }finally{if(hooks.preserveRuns)await fs.rm(pause,{force:true});}
+      };
+      if(hooks.preserveRuns)await locked(home,activate,{drainInvocations:true,signal:hooks.signal});else await activate();
     } else {
       const old=next.old.record;
       const secrets=await read(path.join(home,'packages',job.target,'secrets.json')).catch(e=>{if(e.code==='ENOENT')return {};throw e;});
@@ -266,7 +283,7 @@ export async function recover(home,job,hooks) {
           return /^sha256:[a-f0-9]{64}$/.test(pinned||'')&&current.Image===pinned&&current.State?.Running===true&&current.State?.Health?.Status==='healthy';
         })().catch(()=>false);
         if(!retained)await run('docker',[...relayArgs(config),'stop','relay']);
-        await hooks.stopHost();
+        await (hooks.preserveRuns ? hooks.retireHost() : hooks.stopHost());
         await textAtomic(path.join(config.deploymentDir,'docker.env'),job.rollback.env);
         await bindUpdates(home,path.join(config.deploymentDir,'host-executor.json'),job.rollback.packageRoot);
         await hooks.startHost(job.rollback.packageRoot);

package/src/updates/supervisor.mjs

@@ -5,7 +5,7 @@ import { spawn } from 'node:child_process';
 import { atomic, locked } from '../plugins/manager.mjs';
 import { callDeliverySocket } from '../delivery-socket-client.mjs';
 import { state, read, jobs, jobPath, check, prepare, submit, missing, cleanupStaleBackups } from './control.mjs';
-import { perform, environment, brokerNeedsRefresh } from './runtime.mjs';
+import { perform, environment, brokerNeedsRefresh, execute, relayArgs } from './runtime.mjs';
 
 const reservedProviderKeys=new Set(['HOME','LANG','LC_ALL','LOGNAME','PATH','SHELL','TERM','TMPDIR','USER','CODEX_HOME','NODE_OPTIONS']);
 const providerEnvironmentKey=value=>{
@@ -30,9 +30,9 @@ export async function waitForHostHeartbeat(controlDir,child,failure,{attempts=30
   }
   throw Error('Host transport heartbeat timeout');
 }
-export async function idle(control) {
+export async function idle(control,{pendingRequests=true}={}) {
   const host=await fs.readdir(path.join(control,'host-executor')).catch(e=>{if(e.code==='ENOENT')return [];throw e;});
-  if(host.some(f=>f.endsWith('.running.json')||f.endsWith('.request.json')))return false;
+  if(host.some(f=>f.endsWith('.running.json')||(pendingRequests&&f.endsWith('.request.json'))))return false;
   // Run records live in relay memory: the live socket status is the only
   // cross-process view of active work. A down relay cannot own a run.
   const status=await callDeliverySocket(path.join(control,'delivery.sock'),{op:'status'},2000).catch(()=>null);
@@ -54,7 +54,7 @@ export async function withIdleUpgrade(control,job,signal,apply,{isIdle=idle,wait
 }
 // Plugin replacement drains command leases at activation, not native turns.
 export async function withUpgrade(control,job,signal,apply,options) {
-  if(job.target==='main'||options?.requiresIdle)return withIdleUpgrade(control,job,signal,apply,options);
+  if((job.target==='main'&&!options?.preserveRuns)||options?.requiresIdle)return withIdleUpgrade(control,job,signal,apply,options);
   if(!signal.aborted)return apply();
 }
 export async function queueAutomatic(home,available) {
@@ -73,6 +73,26 @@ export async function queueAutomatic(home,available) {
   }
   return null;
 }
+// Docker archive copy keeps the binding private and readonly without changing
+// host ownership, which may differ from the unprivileged container identity.
+export function executorBindingArchive(binding,gid) {
+  if(!Number.isSafeInteger(gid)||gid<0||gid>0o7777777)throw Error('Invalid executor group');
+  const data=Buffer.from(JSON.stringify(binding)),header=Buffer.alloc(512);
+  const oct=(offset,width,value)=>header.write(value.toString(8).padStart(width-1,'0')+'\0',offset);
+  header.write('executor.json');oct(100,8,0o440);oct(108,8,0);oct(116,8,gid);
+  oct(124,12,data.length);oct(136,12,0);header.fill(32,148,156);
+  header.write('0',156);header.write('ustar\0',257);header.write('00',263);
+  header.write(header.reduce((sum,byte)=>sum+byte,0).toString(8).padStart(6,'0')+'\0 ',148);
+  return Buffer.concat([header,data,Buffer.alloc((512-data.length%512)%512+1024)]);
+}
+export function executorVolumes(deployment,worker) {
+  const names=[...new Set(worker.volumes.filter(m=>m.type==='volume').map(m=>m.source))];
+  return Object.fromEntries(names.map(name=>{
+    const actual=deployment.volumes?.[name]?.name;
+    if(typeof actual!=='string'||!actual)throw Error('Missing executor volume binding');
+    return [name,{name:actual,external:true}];
+  }));
+}
 export async function supervise(deployment,signal,{discover=check}={}) {
   const host=await read(path.join(deployment,'host-executor.json'));
   if(host.agents.length!==1||!host.agents[0].toolsHome)throw Error('Initialize a single deployment plugin registry before starting the supervisor');
@@ -84,20 +104,96 @@ export async function supervise(deployment,signal,{discover=check}={}) {
   if(prior&&(!Number.isSafeInteger(prior.pid)||prior.pid<1))throw Error('Corrupt supervisor lock');
   if(prior){try{process.kill(prior.pid,0);throw Error('Supervisor already running');}catch(e){if(e.code!=='ESRCH')throw e;}await fs.rm(lock);}
   await fs.writeFile(lock,JSON.stringify({pid:process.pid}),{flag:'wx',mode:0o600});
-  let child;
+  let child,container;
+  let retirements=[];
+  let reloadPending=false;
+  const generations=new Map();
+  const removeContainer=async id=>{
+    const generation=generations.get(id);
+    if(generation){await execute('docker',['compose','--project-name',generation.project,'--file',generation.file,'down']);await fs.rm(generation.file,{force:true});generations.delete(id);}
+    else await execute('docker',['rm',id]);
+  };
   const stopHost=async()=>{
+    if(container){await execute('docker',['stop','--time','10',container]);await removeContainer(container);container=undefined;}
     if(!child)return;const running=child;child=undefined;
     if(running.exitCode!==null)return;
     const closed=new Promise(resolve=>running.once('close',resolve));running.kill('SIGTERM');
     const timer=setTimeout(()=>running.kill('SIGKILL'),10000);await closed;clearTimeout(timer);
   };
   const startHost=async root=>{
-    if(host.isolation==='isolated')return
+    if(host.isolation==='isolated') {
+      const pkg=await read(path.join(root,'package.json'));
+      if(pkg.ezRelease?.containerExecutor!==1)return;
+      const lock=path.join(agent.controlDir,'host-executor/worker.lock');
+      const prior=await read(lock).catch(missing);
+      if(prior?.containerId){
+        if(!/^[a-f0-9]{12,64}$/.test(prior.containerId))throw Error('Invalid previous isolated worker identity');
+        const running=await execute('docker',['inspect','--format','{{.State.Running}}',prior.containerId],{stdoutOnly:true}).catch(()=>null);
+        if(running?.trim()==='true')throw Error('Previous isolated worker is still running');
+        // Only the supervisor can prove a foreign container stopped.
+        if(JSON.stringify(await read(lock))!==JSON.stringify(prior))throw Error('Worker lock changed during recovery');
+        await fs.rm(lock);
+      }
+      const config=(await state(home)).config;
+      const deployment=JSON.parse(await execute('docker',[...relayArgs(config),'config','--format','json'],{stdoutOnly:true}));
+      const relay=deployment.services.relay;
+      if(!relay?.image||!Array.isArray(relay.volumes))throw Error('Invalid isolated relay binding');
+      const generation=path.basename(root).replace(/[^a-zA-Z0-9_.-]/g,'-')+'-'+Date.now();
+      const project='ez-executor-'+generation.toLowerCase();
+      const file=path.join(directory,project+'.json');
+      const binding={...host,agents:host.agents.map(a=>({...a,binDir:'/app/bin'}))};
+      const worker={image:relay.image,init:true,restart:'no',user:`${relay.environment.EZ_RUNTIME_UID||1000}:${relay.environment.EZ_RUNTIME_GID||1000}`,
+        cap_drop:['ALL'],security_opt:['no-new-privileges:true'],entrypoint:['node','--import','/app/node_modules/tsx/dist/loader.mjs','/app/src/host-executor.ts','/run/executor.json'],
+        environment:{...relay.environment,EZ_CONTAINER_EXECUTOR:'1',EZ_EXECUTOR_TRANSPORT:'local'},
+        volumes:relay.volumes,tmpfs:['/tmp'],
+        ...Object.fromEntries(['networks','network_mode','extra_hosts','dns','dns_search'].filter(key=>relay[key]!==undefined).map(key=>[key,relay[key]]))};
+      if(worker.networks&&!Array.isArray(worker.networks))worker.networks=Object.fromEntries(Object.keys(worker.networks).map(name=>[name,{}]));
+      const networks=worker.networks?Object.fromEntries(Object.entries(deployment.networks||{}).map(([name,binding])=>[name,{name:binding.name,external:true}])):undefined;
+      const volumes=executorVolumes(deployment,worker);
+      const archive=executorBindingArchive(binding,Number(worker.user.split(':')[1]));
+      await atomic(file,{services:{executor:worker},...(networks?{networks}:{}),...(Object.keys(volumes).length?{volumes}:{})});
+      try {
+        await execute('docker',['compose','--project-name',project,'--file',file,'create','--no-build','executor']);
+        container=(await execute('docker',['compose','--project-name',project,'--file',file,'ps','-aq','executor'],{stdoutOnly:true})).trim();
+        if(!/^[a-f0-9]{12,64}$/.test(container))throw Error('Missing isolated executor');
+      } catch(error) {
+        await execute('docker',['compose','--project-name',project,'--file',file,'down']);
+        await fs.rm(file,{force:true});container=undefined;throw error;
+      }
+      generations.set(container,{project,file});
+      await execute('docker',['cp','-',`${container}:/run`],{input:archive});
+      await execute('docker',['start',container]);
+      for(let n=0;n<300;n++){
+        const running=await execute('docker',['inspect','--format','{{.State.Running}}',container],{stdoutOnly:true});
+        if(running.trim()!=='true')throw Error('Isolated executor exited during startup');
+        const heartbeat=await read(path.join(agent.controlDir,'host-executor/heartbeat.json')).catch(missing);
+        if(heartbeat?.containerExecutor===true&&container.startsWith(heartbeat.containerId||'missing')&&Date.now()-heartbeat.at<1000)return;
+        await sleep(100);
+      }
+      throw Error('Isolated executor heartbeat timeout');
+    }
     child=spawn(process.execPath,['--import',path.join(root,'node_modules/tsx/dist/loader.mjs'),path.join(root,'src/host-executor.ts'),path.join(deployment,'host-executor.json')],
-      {env:{...environment(),...providerEnvironment(host),EZ_HOST_SUPERVISOR_PID:String(process.pid)},stdio:['ignore','inherit','inherit']});
+      {detached:true,env:{...environment(),...providerEnvironment(host),EZ_HOST_SUPERVISOR_PID:String(process.pid)},stdio:['ignore','inherit','inherit']});
     let error;child.once('error',e=>{error=e;});
     await waitForHostHeartbeat(agent.controlDir,child,()=>error);
   };
+  const retireHost=async()=>{
+    if(container){await execute('docker',['kill','--signal','SIGUSR2',container]);retirements.push({container});container=undefined;}
+    if(child){const running=child;child=undefined;if(running.exitCode===null){running.kill('SIGUSR2');retirements.push({child:running});}}
+    // A retiring generation releases its single claim lock before replacement.
+    for(let n=0;n<300;n++){
+      if(!await fs.stat(path.join(agent.controlDir,'host-executor/worker.lock')).then(()=>true,e=>{if(e.code==='ENOENT')return false;throw e;}))return;
+      await sleep(100);
+    }
+    throw Error('Native executor retirement timeout');
+  };
+  const preserveRuns=async job=>{
+    if(job.target!=='main')return false;
+    const old=await read(path.join((await state(home)).config.packageRoot,'package.json'));
+    const candidate=await read(path.join(jobPath(home,job.id),'package/package.json'));
+    return old.ezRelease?.runHandoff===1&&candidate.ezRelease?.runHandoff===1&&
+      (host.isolation!=='isolated'||(container&&old.ezRelease?.containerExecutor===1&&candidate.ezRelease?.containerExecutor===1));
+  };
   const pause=path.join(agent.controlDir,'upgrade-pause.json');
   const refreshRequirement=async job=>{
     if(job.target==='main')return false;
@@ -119,36 +215,58 @@ export async function supervise(deployment,signal,{discover=check}={}) {
     if(owner&&prior&&owner.pid===prior.pid)await fs.rm(registryLock);
     // Wait for an orphaned transport's parent-watch to terminate it first.
     await sleep(1200);
+    const workerOwner=await read(path.join(agent.controlDir,'host-executor/worker.lock')).catch(missing);
+    if(host.isolation==='isolated'&&workerOwner?.containerId){
+      if(!/^[a-f0-9]{12,64}$/.test(workerOwner.containerId))throw Error('Invalid isolated worker identity');
+      if((await execute('docker',['inspect','--format','{{.State.Running}}',workerOwner.containerId],{stdoutOnly:true}).catch(()=>''))?.trim()==='true')container=workerOwner.containerId;
+    }
     const interrupted=(await jobs(home)).find(j=>j.status==='applying');
     if(interrupted){
       const refreshPluginBroker=await refreshRequirement(interrupted);
       const result=refreshPluginBroker===null?undefined:interrupted.target==='main'
-        ? await (async()=>{await atomic(pause,{id:interrupted.id});try{return await locked(home,()=>perform(home,interrupted,{stopHost,startHost}));}finally{await fs.rm(pause,{force:true});}})()
-        : await withUpgrade(agent.controlDir,interrupted,signal,()=>perform(home,interrupted,{stopHost,startHost,signal,refreshPluginBroker}),{requiresIdle:refreshPluginBroker});
-      if(result?.status==='completed'&&interrupted.target==='main')return;
+        ? await (async()=>{await atomic(pause,{id:interrupted.id});try{return await locked(home,()=>perform(home,interrupted,{stopHost,startHost,retireHost,preserveRuns:interrupted.preserveRuns===true}));}finally{await fs.rm(pause,{force:true});}})()
+        : await withUpgrade(agent.controlDir,interrupted,signal,()=>perform(home,interrupted,{stopHost,startHost,retireHost,signal,refreshPluginBroker}),{requiresIdle:refreshPluginBroker});
+      if(['completed','rolled-back'].includes(result?.status)&&interrupted.target==='main')reloadPending=true;
     }
     const isolated=host.isolation==='isolated'
-    if(!isolated && !child)await startHost((await state(home)).config.packageRoot);
+    if(!isolated&&!child)await startHost((await state(home)).config.packageRoot);
+    const initializeIsolated=async()=>{
+      if(!isolated||container)return;
+      const root=(await state(home)).config.packageRoot;
+      if((await read(path.join(root,'package.json'))).ezRelease?.containerExecutor!==1)return;
+      // Queued requests need this worker to drain. Existing native runs still
+      // block migration; the worker lock continues to enforce single ownership.
+      await withIdleUpgrade(agent.controlDir,{id:'executor-migration'},signal,()=>startHost(root),{isIdle:()=>idle(agent.controlDir,{pendingRequests:false})});
+    };
+    await initializeIsolated();
     let nextCheck=0;
     while(!signal.aborted) {
       if(!isolated && child?.exitCode!==null&&child?.exitCode!==undefined)throw Error('Host transport exited; supervisor service should restart');
-      const pending=(await jobs(home)).find(j=>j.status==='queued');
+      await initializeIsolated();
+      if(container&&(await execute('docker',['inspect','--format','{{.State.Running}}',container],{stdoutOnly:true})).trim()!=='true')throw Error('Isolated executor exited; supervisor service should restart');
+      if(reloadPending&&await idle(agent.controlDir))return;
+      // Reap retired containers only after their worker exits.
+      for(const item of retirements.filter(r=>r.container)){
+        if((await execute('docker',['inspect','--format','{{.State.Running}}',item.container],{stdoutOnly:true})).trim()==='false'){await removeContainer(item.container);retirements=retirements.filter(r=>r!==item);}
+      }
+      const pending=!reloadPending&&(await jobs(home)).find(j=>j.status==='queued');
       if(pending) {
         const refreshPluginBroker=await refreshRequirement(pending);
         if(refreshPluginBroker===null)continue;
+        const preserve=await preserveRuns(pending);
         const apply=async()=>{
           try {
-            const activate=async()=>{const latest=await read(path.join(jobPath(home,pending.id),'job.json'));return perform(home,latest,{stopHost,startHost,signal,refreshPluginBroker});};
-            return pending.target==='main'?await locked(home,activate,{drainInvocations:true,signal}):await activate();
+            const activate=async()=>{const latest=await read(path.join(jobPath(home,pending.id),'job.json'));return perform(home,latest,{stopHost,startHost,retireHost,signal,refreshPluginBroker,preserveRuns:preserve});};
+            return pending.target==='main'&&!preserve?await locked(home,activate,{drainInvocations:true,signal}):await activate();
           } catch(error) {
             // A pre-switch rejection is terminal. Applying jobs keep their journal for recovery.
             const latest=await read(path.join(jobPath(home,pending.id),'job.json'));
             if(latest.status==='queued'){latest.status='failed';latest.error=error.message;await atomic(path.join(jobPath(home,pending.id),'job.json'),latest);}else throw error;
           }
         };
-        const result=await withUpgrade(agent.controlDir,pending,signal,apply,{requiresIdle:refreshPluginBroker});
-        if(result?.status==='completed') {
-          if(pending.target==='main')return;
+        const result=await withUpgrade(agent.controlDir,pending,signal,apply,{requiresIdle:refreshPluginBroker,preserveRuns:preserve});
+        if(['completed','rolled-back'].includes(result?.status)) {
+          if(pending.target==='main')reloadPending=true;
           nextCheck=0;
         }
       }
@@ -166,7 +284,8 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       await sleep(500);
     }
   }finally {
-    clearInterval(beat);await stopHost();await fs.rm(lock,{force:true});
+    clearInterval(beat);await stopHost();
+    for(const item of retirements){if(item.container){await execute('docker',['stop','--time','10',item.container]);await removeContainer(item.container);}else if(item.child.exitCode===null)item.child.kill('SIGTERM');}await fs.rm(lock,{force:true});
     if(!(await jobs(home)).some(j=>j.status==='applying'))await fs.rm(pause,{force:true});
   }
 }

package/test/ai-cli.test.ts

@@ -6,6 +6,7 @@ import path from 'node:path'
 import {spawnSync} from 'node:child_process'
 import {fileURLToPath} from 'node:url'
 import {ControlStore} from '../src/control-state.js'
+import {executorJobEnv} from '../src/executor.js'
 import {initialPreset} from '../src/ai.js'
 
 test('explicit CLI/model selection preserves installation default and rejects unavailable choices',async()=>{
@@ -50,12 +51,33 @@ test('an agent can select a declared provider and model without putting the key
   await writeFile(path.join(binDir,'codex'),'#!/bin/sh\nexit 0\n',{mode:0o700})
   await writeFile(path.join(controlDir,'host-executor','models.json'),JSON.stringify([{cli:'codex',provider:'openrouter',model:'google/gemini-3.8-flash',name:'Gemini',efforts:[]}]))
   const store=new ControlStore(controlDir,900000);await store.aiState(initialPreset('codex'))
-  const env={...process.env,HOME:root,PATH:binDir+path.delimiter+process.env.PATH,EZ_CONTROL_DIR:controlDir,OPENROUTER_API_KEY:'must-not-persist'}
+  const env=executorJobEnv({runId:'fixture',controlDir,binDir,catalogTransport:'host'},{...process.env,HOME:root,PATH:binDir+path.delimiter+process.env.PATH,OPENROUTER_API_KEY:'must-not-persist'})
   const bin=fileURLToPath(new URL('../bin/ezenciel-agents-ai.mjs',import.meta.url))
   const result=spawnSync(process.execPath,[bin,'select','--cli','codex','--provider','openrouter','--model','google/gemini-3.8-flash'],{env,encoding:'utf8'})
   assert.equal(result.status,0,result.stderr)
   const state=await store.status(),selected=state.ai?.presets.find(p=>p.id===state.ai?.selectedId)
   assert.equal(selected?.provider,'openrouter');assert.equal(selected?.model,'google/gemini-3.8-flash')
   assert.equal(JSON.stringify(state).includes('must-not-persist'),false)
+  assert.equal(env.EZ_EXECUTOR_TRANSPORT,undefined)
+  assert.equal(env.EZ_ISOLATION,undefined)
+ }finally{await rm(root,{recursive:true,force:true})}
+})
+
+
+test('isolated AI CLI ignores a legacy host catalog and discovers its own Sonnet',async()=>{
+ const root=await mkdtemp(path.join(tmpdir(),'ez-ai-local-catalog-'))
+ try{
+  const controlDir=path.join(root,'control'),binDir=path.join(root,'bin')
+  await mkdir(path.join(controlDir,'host-executor'),{recursive:true});await mkdir(binDir)
+  const stale=JSON.stringify([{cli:'codex',model:'host-only',name:'Host only',efforts:[]}])
+  await writeFile(path.join(controlDir,'host-executor','models.json'),stale)
+  await writeFile(path.join(binDir,'claude'),"#!/bin/sh\nprintf '%s\n' \"  --model <model> Model (e.g. 'sonnet', 'opus')\" \"  --effort <level> Effort (low, high)\"\n",{mode:0o700})
+  const env={...process.env,HOME:root,PATH:binDir,EZ_CONTROL_DIR:controlDir,EZ_ISOLATION:'isolated',EZ_EXECUTOR_TRANSPORT:'local'}
+  const bin=fileURLToPath(new URL('../bin/ezenciel-agents-ai.mjs',import.meta.url))
+  const result=spawnSync(process.execPath,[bin,'list'],{env,encoding:'utf8'})
+  assert.equal(result.status,0,result.stderr)
+  const catalog=JSON.parse(result.stdout)
+  assert.ok(catalog.some((m:{cli:string;model:string})=>m.cli==='claude'&&m.model==='sonnet'))
+  assert.ok(!catalog.some((m:{model:string})=>m.model==='host-only'))
  }finally{await rm(root,{recursive:true,force:true})}
 })

package/test/ai.test.ts

@@ -631,7 +631,7 @@ test('private channel model choices do not consume owner preset slots', async ()
     assert.equal(readback.preset?.model, model.model)
     assert.equal(readback.preset?.effort, 'low')
     assert.deepEqual((await store.status()).ai, before)
-    await assert.rejects(menu.saveSelection(model, 'max'), /at most 12/)
+    assert.equal((await menu.saveSelection(model, 'max')).effort, 'max')
     await assert.rejects(store.changeApplicationSession(scope, {...guard, expectedSession:null}, preset), /Conversation changed/)
     assert.deepEqual((await store.applicationSession(scope))?.preset, readback.preset)
   } finally { await rm(dir, {recursive:true, force:true}) }
@@ -648,7 +648,29 @@ test('detected presets do not count against the saved AI cap', async () => {
     assert.equal((await store.status()).ai!.presets.length, 13)
     const next = {id:'one-more', name:'One more', cli:'claude', model:'sonnet', effort:'medium'}
     await store.savePreset(next)
-    await assert.rejects(store.savePreset({...next, id:'too-many', name:'Too many'}), /at most 12/)
+    await store.savePreset({...next, id:'one-past-cap', name:'One past cap'})
+    const ids = (await store.status()).ai!.presets.map(p => p.id)
+    assert.equal(ids.filter(id => !id.startsWith('detected_')).length, 12)
+    assert.ok(!ids.includes('saved-1') && ids.includes('one-past-cap') && ids.includes(initial.id))
+  } finally { await rm(dir, {recursive:true, force:true}) }
+})
+
+test('a new model choice at the saved AI cap replaces the oldest unused choice', async () => {
+  const dir = await mkdtemp(join(tmpdir(), 'ez-ai-cap-select-'))
+  try {
+    const store = new ControlStore(dir, 1000)
+    const initial = initialPreset('claude')
+    await store.captureChoice(initial)
+    for (let n = 1; n < 12; n++) await store.savePreset({...initial, id:`saved-${n}`, name:`Saved ${n}`, model:'opus', effort:n % 2 ? 'high' : 'xhigh'})
+    await store.selectPreset('saved-1', (await store.getActiveSession())?.sessionId ?? null)
+    const model = {cli:'claude', model:'sonnet', name:'sonnet', efforts:['low', 'high']}
+    const menu = createAiMenu(store, 'claude', async () => [model], dir, undefined, async () => true)
+    const preset = await menu.saveSelection(model, 'high')
+    await menu.select(preset, (await store.getActiveSession())?.sessionId ?? null)
+    const ai = (await store.status()).ai!
+    assert.equal(ai.presets.find(p => p.id === ai.selectedId)?.model, 'sonnet')
+    assert.equal(ai.presets.length, 12)
+    assert.ok(ai.presets.some(p => p.id === 'saved-1') && !ai.presets.some(p => p.id === 'saved-2'))
   } finally { await rm(dir, {recursive:true, force:true}) }
 })
 

package/test/delivery-transport.test.ts

@@ -1,6 +1,6 @@
 import test from 'node:test'
 import assert from 'node:assert/strict'
-import { mkdtemp, rm, unlink } from 'node:fs/promises'
+import { mkdtemp, rm, unlink, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { createConnection } from 'node:net'
@@ -64,9 +64,48 @@ test('a response timeout is not replayed over the loopback endpoint', async t =>
   const dir = await mkdtemp(join(tmpdir(), 'ez-delivery-timeout-'))
   t.after(() => rm(dir, { recursive: true, force: true }))
   let calls = 0
-  const server = await serveDeliverySocket(dir, async () => { calls++; return new Promise(() => {}) }, { tcpPort: 0 })
+  const server = await serveDeliverySocket(dir, async () => { calls++; await new Promise(resolve=>setTimeout(resolve,300)); return {accepted:true} }, { tcpPort: 0 })
   t.after(() => server.stop())
   assert.ok(server.endpoint)
   await assert.rejects(callDeliverySocket(deliverySocketPath(dir), { op: 'enqueue' }, 200), /Delivery relay unavailable/)
   assert.equal(calls, 1)
 })
+
+test('shutdown waits for an admitted enqueue before the final delivery drain',async t=>{
+ const dir=await mkdtemp(join(tmpdir(),'ez-delivery-admission-'))
+ t.after(()=>rm(dir,{recursive:true,force:true}))
+ let release!:()=>void,entered!:()=>void,accepted=0,drained=0
+ const gate=new Promise<void>(resolve=>{release=resolve}),admitted=new Promise<void>(resolve=>{entered=resolve})
+ const server=await serveDeliverySocket(dir,async()=>{entered();await gate;accepted++;return {accepted:true}},{beforeClose:async()=>{drained=accepted}})
+ const request=callDeliverySocket(deliverySocketPath(dir),{op:'enqueue'})
+ await admitted
+ let stopped=false
+ const stop=server.stop().then(()=>{stopped=true})
+ await new Promise(resolve=>setTimeout(resolve,50))
+ assert.equal(stopped,false)
+ assert.equal(drained,0)
+ release()
+ assert.deepEqual(await request,{accepted:true})
+ await stop
+ assert.equal(drained,1)
+})
+
+
+test('a swap connection failure waits for the relay but an accepted timeout is never retried',async t=>{
+ const dir=await mkdtemp(join(tmpdir(),'ez-delivery-swap-'))
+ t.after(()=>rm(dir,{recursive:true,force:true}))
+ await writeFile(join(dir,'upgrade-pause.json'),'{}')
+ const waiting=callDeliverySocket(deliverySocketPath(dir),{op:'enqueue'},1000)
+ await new Promise(r=>setTimeout(r,300))
+ let accepted=0
+ const server=await serveDeliverySocket(dir,async()=>({accepted:++accepted}))
+ t.after(()=>server.stop())
+ assert.deepEqual(await waiting,{accepted:1})
+ assert.equal(accepted,1)
+ await server.stop()
+ let attempted=0
+ const hanging=await serveDeliverySocket(dir,async()=>{attempted++;await new Promise(r=>setTimeout(r,200));return {accepted:true}})
+ t.after(()=>hanging.stop())
+ await assert.rejects(callDeliverySocket(deliverySocketPath(dir),{op:'enqueue'},40),/unavailable/)
+ assert.equal(attempted,1)
+})

package/test/executor.test.ts

@@ -2,11 +2,11 @@ import { ownerRun } from './helpers/owner-run.js'
 import { RunStore } from '../src/runs.js'
 import assert from 'node:assert/strict'
 import test from 'node:test'
-import { mkdtemp, mkdir, rm, writeFile, readFile } from 'node:fs/promises'
+import { mkdtemp, mkdir, rm, writeFile, readFile, rename, stat } from 'node:fs/promises'
 import { spawn } from 'node:child_process'
 import { tmpdir } from 'node:os'
 import path from 'node:path'
-import { EXECUTOR_REGISTRY, executorEnvironment, grokInvocation, grokJobEnv, opencodeDataHome, opencodeInvocation, piAgentDir, resolveExecutor, resolveHostCommand, startExecutorJob, terminateJob, validateCodexProvider } from '../src/executor.js'
+import { EXECUTOR_REGISTRY, attachHostJob, executorEnvironment, grokInvocation, grokJobEnv, opencodeDataHome, opencodeInvocation, piAgentDir, resolveExecutor, resolveHostCommand, startExecutorJob, terminateJob, validateCodexProvider } from '../src/executor.js'
 import { requireOwnerExecution } from '../src/execution-authority.js'
 import { splitTelegramText } from '../src/reply.js'
 import { matchingProcessIds, processSnapshot } from '../src/process-tree.js'
@@ -139,8 +139,10 @@ test('host transport does not throw when selecting the desktop adapter', async (
       workspace: root, controlDir, binDir: path.join(root, 'bin'), cli: 'codex-gui', runId: 'r_hostgui', timeoutMs: 1500,
     })
     assert.ok(job.child.pid)
-    terminateJob(job.child)
-    await new Promise((resolve) => job.child.once('close', resolve))
+    const closed = new Promise((resolve) => job.child.once('close', resolve))
+    await terminateJob(job.child)
+    await closed
+    assert.equal(await readFile(path.join(controlDir,'host-executor/r_hostgui.cancel'),'utf8'),'')
     await job.cleanup()
   } finally {
     if (prior === undefined) delete process.env.EZ_EXECUTOR_TRANSPORT
@@ -149,6 +151,32 @@ test('host transport does not throw when selecting the desktop adapter', async (
   }
 })
 
+test('host cancellation persists before a stalled observer receives a signal', {skip:process.platform==='win32'}, async () => {
+  const root=await mkdtemp(path.join(tmpdir(),'ez-cancel-observer-'))
+  const directory=path.join(root,'host-executor'),id='r_stalled'
+  await mkdir(directory)
+  await assert.rejects(attachHostJob(root,'../escape'),/Invalid host executor run ID/)
+  await writeFile(path.join(directory,id+'.running.json'),'{}')
+  const child=await attachHostJob(root,id)
+  const closed=new Promise(resolve=>child.once('close',resolve))
+  child.kill('SIGSTOP')
+  try {
+    await rename(directory,directory+'.saved')
+    await writeFile(directory,'Blocked transport directory')
+    assert.equal(await terminateJob(child),false)
+    assert.equal(child.exitCode,null)
+    assert.equal(child.signalCode,null)
+    await assert.rejects(readFile(path.join(directory+'.saved',id+'.cancel')),{code:'ENOENT'})
+    await rm(directory);await rename(directory+'.saved',directory)
+    assert.equal(await terminateJob(child),true)
+    assert.equal(await readFile(path.join(directory,id+'.cancel'),'utf8'),'')
+    assert.equal((await stat(path.join(directory,id+'.cancel'))).mode&0o777,0o600)
+  } finally {
+    child.kill('SIGCONT');child.kill('SIGKILL')
+    await closed;await rm(root,{recursive:true,force:true})
+  }
+})
+
 test('Codex jobs disable global memory and host skill discovery', () => {
   const args = EXECUTOR_REGISTRY.codex.buildArgs({workspace:'/agent'},'/prompt','hello')
   assert.ok(args.includes('memories'))

package/test/host-executor.test.ts

@@ -8,7 +8,7 @@ import { serveHostExecutor, refreshAgentCatalog } from '../src/host-executor.js'
 import { spawn } from 'node:child_process'
 import { fileURLToPath } from 'node:url'
 import { isHostRunId } from '../src/host-executor-protocol.js'
-import { EXECUTOR_REGISTRY } from '../src/executor.js'
+import { EXECUTOR_REGISTRY, attachHostJob, terminateJob } from '../src/executor.js'
 import { RunStore } from '../src/runs.js'
 import { packageVersion } from '../src/version.js'
 import { executionDefaults } from '../src/model-policy.js'
@@ -294,11 +294,17 @@ test('one installed CLI executes two agent bindings with separate minds and sani
     assert.equal(await workspaceLease(agents[0].toolsHome),undefined)
     await ownerRun(agents[0].controlDir,'tg_43')
     await writeFile(path.join(directory,'tg_43.request.json'),JSON.stringify({texts:['Hold chat'],options:{cli:'grok'}}))
-    for(let n=0;n<100;n++){try{await readFile(path.join(directory,'tg_43.process.json'));break}catch{await new Promise(r=>setTimeout(r,20))}}
+    for(let n=0;n<100;n++){try{if(JSON.parse(await readFile(path.join(directory,'tg_43.process.json'),'utf8')).pid)break}catch{await new Promise(r=>setTimeout(r,20))}}
     await readFile(path.join(directory,'tg_43.running.json'))
     assert.doesNotMatch(await readFile(path.join(directory,'r_hold.events'),'utf8'),/"stream":"exit"/)
     await readFile(path.join(directory,'r_hold.running.json'))
-    await writeFile(path.join(directory,'r_hold.cancel'),'')
+    // An observer that cannot handle signals must still stop the native run.
+    const observer=await attachHostJob(agents[0].controlDir,'r_hold')
+    const observerClosed=new Promise(resolve=>observer.once('close',resolve))
+    observer.stdout?.resume();observer.stderr?.resume()
+    observer.kill('SIGSTOP')
+    try { assert.equal(await terminateJob(observer),true) }
+    finally { observer.kill('SIGCONT');observer.kill('SIGKILL');await observerClosed }
     let heldOutput=''
     for(let n=0;n<200;n++){heldOutput=await readFile(path.join(directory,'r_hold.events'),'utf8');if(heldOutput.includes('"stream":"exit"'))break;await new Promise(r=>setTimeout(r,20))}
     assert.match(heldOutput, /"stream":"exit","code":1/)
@@ -568,3 +574,64 @@ test('host transport runs a registered script from the run record and refuses ch
     await rm(root,{recursive:true,force:true})
   }
 })
+
+test('a native process that settles during launch records completion without hanging the worker',async t=>{
+ const root=await mkdtemp(path.join(tmpdir(),'ez-fast-native-')),workspace=path.join(root,'mind'),controlDir=path.join(root,'control')
+ await mkdir(workspace);await mkdir(controlDir)
+ const abort=new AbortController(),directory=path.join(controlDir,'host-executor')
+ const launch=async()=>{
+  const child=spawn(process.execPath,['-e','process.exit(0)'],{stdio:['pipe','pipe','pipe']})
+  await new Promise(resolve=>child.once('close',resolve))
+  return {child,cleanup:async()=>{},stdout:''}
+ }
+ const server=serveHostExecutor({cli:'grok',agents:[{name:'test',workspace,controlDir,binDir:root}]},abort.signal,launch,async()=>[])
+ t.after(async()=>{abort.abort();await server;await rm(root,{recursive:true,force:true})})
+ for(let n=0;n<300;n++){try{await readFile(path.join(directory,'heartbeat.json'));break}catch{await new Promise(r=>setTimeout(r,20))}}
+ await ownerRun(controlDir,'r_fast_native')
+ await writeFile(path.join(directory,'r_fast_native.request.json'),JSON.stringify({texts:['fast'],options:{cli:'grok'}}))
+ let events=''
+ for(let n=0;n<300;n++){try{events=await readFile(path.join(directory,'r_fast_native.events'),'utf8');if(events.includes('"stream":"exit"'))break}catch{}await new Promise(r=>setTimeout(r,20))}
+ assert.match(events,/"stream":"exit","code":0/)
+})
+
+test('retiring native generation preserves its run while the replacement claims new work once',async t=>{
+ const root=await mkdtemp(path.join(tmpdir(),'ez-host-generation-'))
+ const workspace=path.join(root,'mind'),controlDir=path.join(root,'control'),toolsHome=path.join(root,'tools')
+ await mkdir(workspace);await mkdir(toolsHome);await mkdir(controlDir)
+ await writeFile(path.join(toolsHome,'config.json'),JSON.stringify({schemaVersion:1,workspace:await realpath(workspace)}))
+ const agent={name:'qa',workspace,controlDir,toolsHome,binDir:root},installation={cli:'grok',agents:[agent]}
+ const stopA=new AbortController(),stopB=new AbortController(),retireA=new AbortController()
+ const children=new Map<string,ReturnType<typeof spawn>>(),launched:string[]=[]
+ const launch=async(_texts:string[],options:any)=>{
+  launched.push(options.runId)
+  const child=spawn(process.execPath,['-e',"process.stdin.resume();process.stdin.once('data',()=>process.exit(0));"],{stdio:['pipe','pipe','pipe']})
+  children.set(options.runId,child)
+  return {child,cleanup:async()=>{},stdout:''}
+ }
+ let a:Promise<void>|undefined,b:Promise<void>|undefined
+ t.after(async()=>{stopA.abort();stopB.abort();for(const child of children.values())child.stdin?.end('done');await Promise.all([a,b]);await rm(root,{recursive:true,force:true})})
+ const directory=path.join(controlDir,'host-executor')
+ const wait=async(check:()=>Promise<boolean>)=>{for(let n=0;n<300;n++){if(await check())return;await new Promise(r=>setTimeout(r,20))}assert.fail('generation did not reach expected state')}
+ const exists=(file:string)=>readFile(file).then(()=>true,()=>false)
+ a=serveHostExecutor(installation,stopA.signal,launch,async()=>[],retireA.signal)
+ await wait(()=>exists(path.join(directory,'heartbeat.json')))
+ await ownerRun(controlDir,'r_old_generation')
+ await writeFile(path.join(directory,'r_old_generation.request.json'),JSON.stringify({texts:['old'],options:{cli:'grok'}}))
+ await wait(async()=>children.has('r_old_generation'))
+ retireA.abort()
+ await wait(async()=>!await exists(path.join(directory,'worker.lock')))
+ assert.equal(children.get('r_old_generation')!.exitCode,null)
+ b=serveHostExecutor(installation,stopB.signal,launch,async()=>[])
+ await wait(()=>exists(path.join(directory,'worker.lock')))
+ await ownerRun(controlDir,'r_new_generation')
+ await writeFile(path.join(directory,'r_new_generation.request.json'),JSON.stringify({texts:['new'],options:{cli:'grok'}}))
+ await wait(async()=>children.has('r_new_generation'))
+ assert.deepEqual(launched,['r_old_generation','r_new_generation'])
+ children.get('r_old_generation')!.stdin!.end('done')
+ await a
+ assert.ok(await exists(path.join(directory,'worker.lock')),'retiring host must not delete replacement lock')
+ assert.match(await readFile(path.join(directory,'r_old_generation.events'),'utf8'),/"stream":"exit","code":0/)
+ children.get('r_new_generation')!.stdin!.end('done')
+ await wait(async()=>(await readFile(path.join(directory,'r_new_generation.events'),'utf8')).includes('"stream":"exit"'))
+ assert.match(await readFile(path.join(directory,'r_new_generation.events'),'utf8'),/"stream":"exit","code":0/)
+})