@jc_stack/ez-agents 0.1.0-beta.51 → 0.1.0-beta.52

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.1.0-beta.52
+
+- Restore isolated executor startup with existing named volumes and a private, root-owned read-only container binding. Host file ownership and permissions remain unchanged.
+- Start a missing isolated executor when native requests are queued, while retaining the live-run drain and single-worker ownership checks.
+
+
 ## 0.1.0-beta.51
 
 - Keep native runs alive during handoff-compatible core upgrades. The retiring executor finishes existing work while its replacement accepts new requests; the relay adopts runs without replay and preserves schedule holds, cancellation and settled delivery receipts. Isolated agents migrate once while idle to a separate, unprivileged executor container; later compatible upgrades preserve its running generation. Older runtimes without the handoff contract retain the full native-work drain.

package/docker/isolated-executor-smoke.mjs

@@ -0,0 +1,35 @@
+// Synthetic private binding and existing-volume acceptance; no provider access.
+import * as fs from 'node:fs/promises';
+import path from 'node:path';
+import {tmpdir} from 'node:os';
+import {randomUUID} from 'node:crypto';
+import {execute} from '../src/updates/runtime.mjs';
+import {executorBindingArchive,executorVolumes} from '../src/updates/supervisor.mjs';
+const root=await fs.mkdtemp(path.join(tmpdir(),'ez-executor-smoke-'));
+const project='ez-executor-smoke-'+randomUUID().slice(0,8),volume=project+'-existing';
+const file=path.join(root,'compose.json'),image=process.env.EZ_RELAY_IMAGE||'ezenciel-agents:local';
+const binding={agents:[{workspace:'/synthetic',binDir:'/app/bin'}]};
+const code=`const fs=require('fs'),assert=require('assert/strict');const p='/run/executor.json',s=fs.statSync(p),d=fs.statSync('/run');assert.equal(process.getuid(),1001);assert.equal(s.uid,0);assert.equal(s.gid,1000);assert.equal(s.mode&511,288);assert.equal(d.uid,0);assert.equal(d.mode&18,0);assert.deepEqual(JSON.parse(fs.readFileSync(p)),${JSON.stringify(binding)});for(const fn of [()=>fs.writeFileSync(p,'changed'),()=>fs.chmodSync(p,384),()=>fs.unlinkSync(p)])assert.throws(fn,e=>['EACCES','EPERM','EROFS'].includes(e.code));assert.equal(fs.readFileSync('/ipc/marker','utf8'),'retained');console.log('Private readonly binding and retained volume passed');`;
+const worker={image,user:'1001:1000',network_mode:'none',cap_drop:['ALL'],security_opt:['no-new-privileges:true'],entrypoint:['node','-e',code],volumes:[{type:'volume',source:'ipc',target:'/ipc'}]};
+const args=['compose','--project-name',project,'--file',file];
+try {
+ await execute('docker',['volume','create',volume]);
+ await execute('docker',['run','--rm','--network','none','--user','0:0','--mount',`type=volume,source=${volume},target=/ipc`,'--entrypoint','node',image,'-e',"require('fs').writeFileSync('/ipc/marker','retained')"]);
+ await fs.writeFile(file,JSON.stringify({services:{executor:worker},volumes:executorVolumes({volumes:{ipc:{name:volume}}},worker)}),{mode:0o600});
+ await execute('docker',[...args,'create','--no-build','executor']);
+ const cid=(await execute('docker',[...args,'ps','-aq','executor'],{stdoutOnly:true})).trim();
+ await execute('docker',['cp','-',`${cid}:/run`],{input:executorBindingArchive(binding,1000)});
+ console.log(await execute('docker',['start','-a',cid]));
+ const exit=(await execute('docker',['inspect','--format','{{.State.ExitCode}}',cid],{stdoutOnly:true})).trim();
+ if(exit!=='0')throw Error('Executor binding acceptance failed');
+ await execute('docker',[...args,'down']);
+ const retained=(await execute('docker',['volume','inspect','--format','{{.Name}}',volume],{stdoutOnly:true})).trim();
+ if(retained!==volume)throw Error('External volume identity changed');
+ // Rejected archive input must fail cleanly, with no uncaught stdin EPIPE.
+ await execute('docker',['cp','-',`${project}-missing:/run`],{input:Buffer.alloc(1024*1024)}).then(()=>{throw Error('Missing container copy succeeded');},()=>{});
+ console.log('PASS: unprivileged executor reads only its private binding; writes, chmod and unlink fail; external volume preserved.');
+} finally {
+ await execute('docker',[...args,'down']).catch(()=>{});
+ await execute('docker',['volume','rm',volume]).catch(()=>{});
+ await fs.rm(root,{recursive:true,force:true});
+}

package/docs/releasing.md

@@ -22,6 +22,7 @@ separate release bot is not required.
    audit advances. Baseline CI does not validate a newly published native version.
    Managed installs and upgrades keep their existing native version selection.
    Main relay: `EZ_RELAY_IMAGE=ez-release-runtime node docker/smoke.mjs`.
+   Isolated executor: `EZ_RELAY_IMAGE=ez-release-runtime node docker/isolated-executor-smoke.mjs`.
    The WhatsApp Compose overlay has no separate relay image; verify its
    provider integration with the synthetic plugin smoke in `docs/plugins.md`.
 4. Create an artifact with `npm pack --ignore-scripts`. Inspect its file list,

package/docs/upgrades.md

@@ -16,7 +16,7 @@ automatic channel for core and plugins without a saved policy. Existing explicit
 stable or manual policies are preserved. The owner may select either per target.
 The main target and installed plugins version independently.
 
-Handoff-compatible core upgrades build while native work continues and pause admission only for activation. The current native executor retires from claiming new work; the replacement claims new requests while the retiring generation finishes its existing runs. The relay adopts those runs without replay, preserving their schedule holds, cancellation and settled delivery receipts. The supervisor reloads only after native work is idle, so OS service process-group cleanup cannot kill a retiring generation. Plugin upgrades build while native work continues. Activation closes the existing registry admission lock, drains in-flight plugin command leases, and replaces the plugin under that lock. New native plugin calls wait for admission; an outdated revision fails before provider execution and must be rediscovered. No admitted provider operation is replayed. The shared broker and native sessions remain running. Older runtimes without the handoff contract retain the full native-work drain. Isolated agents migrate once while idle to a separate, unprivileged executor container using the same file transport; later relay/runtime upgrades preserve that executor generation. No native CLI or provider credentials move to the host. A plugin update also retains that drain if an operator replaced the broker’s single-file host binding (for example to approve new network pins), because that case still requires broker recreation.
+Handoff-compatible core upgrades build while native work continues and pause admission only for activation. The current native executor retires from claiming new work; the replacement claims new requests while the retiring generation finishes its existing runs. The relay adopts those runs without replay, preserving their schedule holds, cancellation and settled delivery receipts. The supervisor reloads only after native work is idle, so OS service process-group cleanup cannot kill a retiring generation. Plugin upgrades build while native work continues. Activation closes the existing registry admission lock, drains in-flight plugin command leases, and replaces the plugin under that lock. New native plugin calls wait for admission; an outdated revision fails before provider execution and must be rediscovered. No admitted provider operation is replayed. The shared broker and native sessions remain running. Older runtimes without the handoff contract retain the full native-work drain. Isolated agents migrate once after live native work finishes to a separate, unprivileged executor container. Pending requests remain queued for that executor; its private binding is copied into the container as root-owned, group-readable and not writable by the configured runtime user. Existing named volumes are attached as external volumes. It uses the same file transport; later relay/runtime upgrades preserve that executor generation. No native CLI or provider credentials move to the host. A plugin update also retains that drain if an operator replaced the broker’s single-file host binding (for example to approve new network pins), because that case still requires broker recreation.
 
 Compatible main upgrades also finish their advisory workspace/native-client
 backup before closing admission or stopping the relay. Large native histories do
@@ -63,6 +63,19 @@ restart and verify. Do not recreate the agent or its registry. Subsequent
 compatible upgrades are agent-owned. Noncanonical/multi-agent host configuration
 files require migration to one canonical deployment per service first.
 
+## Recovering an isolated-executor startup defect
+
+If an older supervisor cannot start its isolated executor, queued requests can
+prevent that same supervisor from installing its correction. Use an exact,
+reviewed newer package's `src/updates/supervisor.mjs DEPLOYMENT_DIRECTORY` as the
+existing OS service's temporary entry point. First verify that no native run is
+active. Keep the same service user, environment, registry, deployment and queued
+requests; do not edit published package files or delete claims. This CLI runs the
+same supervisor and retains the active package binding until normal upgrade
+activation. Let existing queued jobs finish, then prepare and apply the corrected
+release through `ez updates`. Restore the retained bootstrap entry point after
+the active package and running services verify the corrected version.
+
 ## Package-manager preflight and repair
 
 Before installing candidate dependencies or stopping services, the supervisor

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jc_stack/ez-agents",
-  "version": "0.1.0-beta.51",
+  "version": "0.1.0-beta.52",
   "private": false,
   "type": "module",
   "description": "A lightweight foundation for persistent business AI assistants using existing AI harnesses, workspaces and plugins.",

package/src/updates/runtime.mjs

@@ -19,13 +19,14 @@ export function execute(command,args,options={}) {
   return new Promise((resolve,reject)=>{
     const invocation=process.env.EZ_DOCKER_COMPOSE==='standalone'&&command==='docker'&&args[0]==='compose'
       ? {command:'docker-compose',args:args.slice(1)} : {command,args};
-    const child=spawn(invocation.command,invocation.args,{cwd:options.cwd,env:environment(),stdio:['ignore','pipe','pipe']});let output='',stdout='';
+    const child=spawn(invocation.command,invocation.args,{cwd:options.cwd,env:environment(),stdio:[options.input===undefined?'ignore':'pipe','pipe','pipe']});let output='',stdout='';
     const timer=setTimeout(()=>child.kill('SIGKILL'),options.timeout||600000);
     const file=options.outputFile?createWriteStream(options.outputFile,{flags:'wx',mode:0o600}):null;
     const written=file?new Promise((resolve,reject)=>{file.once('finish',resolve);file.once('error',reject);}):Promise.resolve();
     if(file)child.stdout.pipe(file);else child.stdout.on('data',b=>{output=(output+b).slice(-16000);if(options.stdoutOnly)stdout=(stdout+b).slice(-16000);});
     child.stderr.on('data',b=>{output=(output+b).slice(-16000);});
     void written.catch(()=>child.kill('SIGKILL'));
+    if(options.input!==undefined){child.stdin.on('error',error=>{child.kill('SIGKILL');reject(error);});child.stdin.end(options.input);}
     child.once('error',error=>{clearTimeout(timer);reject(error);});
     child.once('close',async code=>{clearTimeout(timer);try{await written;if(code!==0)throw Error(`${invocation.command} failed (${code}): ${output}`);resolve(options.stdoutOnly?stdout:output);}catch(error){reject(error);}});
   });

package/src/updates/supervisor.mjs

@@ -30,9 +30,9 @@ export async function waitForHostHeartbeat(controlDir,child,failure,{attempts=30
   }
   throw Error('Host transport heartbeat timeout');
 }
-export async function idle(control) {
+export async function idle(control,{pendingRequests=true}={}) {
   const host=await fs.readdir(path.join(control,'host-executor')).catch(e=>{if(e.code==='ENOENT')return [];throw e;});
-  if(host.some(f=>f.endsWith('.running.json')||f.endsWith('.request.json')))return false;
+  if(host.some(f=>f.endsWith('.running.json')||(pendingRequests&&f.endsWith('.request.json'))))return false;
   // Run records live in relay memory: the live socket status is the only
   // cross-process view of active work. A down relay cannot own a run.
   const status=await callDeliverySocket(path.join(control,'delivery.sock'),{op:'status'},2000).catch(()=>null);
@@ -73,6 +73,26 @@ export async function queueAutomatic(home,available) {
   }
   return null;
 }
+// Docker archive copy keeps the binding private and readonly without changing
+// host ownership, which may differ from the unprivileged container identity.
+export function executorBindingArchive(binding,gid) {
+  if(!Number.isSafeInteger(gid)||gid<0||gid>0o7777777)throw Error('Invalid executor group');
+  const data=Buffer.from(JSON.stringify(binding)),header=Buffer.alloc(512);
+  const oct=(offset,width,value)=>header.write(value.toString(8).padStart(width-1,'0')+'\0',offset);
+  header.write('executor.json');oct(100,8,0o440);oct(108,8,0);oct(116,8,gid);
+  oct(124,12,data.length);oct(136,12,0);header.fill(32,148,156);
+  header.write('0',156);header.write('ustar\0',257);header.write('00',263);
+  header.write(header.reduce((sum,byte)=>sum+byte,0).toString(8).padStart(6,'0')+'\0 ',148);
+  return Buffer.concat([header,data,Buffer.alloc((512-data.length%512)%512+1024)]);
+}
+export function executorVolumes(deployment,worker) {
+  const names=[...new Set(worker.volumes.filter(m=>m.type==='volume').map(m=>m.source))];
+  return Object.fromEntries(names.map(name=>{
+    const actual=deployment.volumes?.[name]?.name;
+    if(typeof actual!=='string'||!actual)throw Error('Missing executor volume binding');
+    return [name,{name:actual,external:true}];
+  }));
+}
 export async function supervise(deployment,signal,{discover=check}={}) {
   const host=await read(path.join(deployment,'host-executor.json'));
   if(host.agents.length!==1||!host.agents[0].toolsHome)throw Error('Initialize a single deployment plugin registry before starting the supervisor');
@@ -90,7 +110,7 @@ export async function supervise(deployment,signal,{discover=check}={}) {
   const generations=new Map();
   const removeContainer=async id=>{
     const generation=generations.get(id);
-    if(generation){await execute('docker',['compose','--project-name',generation.project,'--file',generation.file,'down']);await fs.rm(generation.file,{force:true});await fs.rm(generation.configFile,{force:true});generations.delete(id);}
+    if(generation){await execute('docker',['compose','--project-name',generation.project,'--file',generation.file,'down']);await fs.rm(generation.file,{force:true});generations.delete(id);}
     else await execute('docker',['rm',id]);
   };
   const stopHost=async()=>{
@@ -121,20 +141,28 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       const generation=path.basename(root).replace(/[^a-zA-Z0-9_.-]/g,'-')+'-'+Date.now();
       const project='ez-executor-'+generation.toLowerCase();
       const file=path.join(directory,project+'.json');
-      const configFile=path.join(directory,project+'-binding.json');
-      await atomic(configFile,{...host,agents:host.agents.map(a=>({...a,binDir:'/app/bin'}))});
+      const binding={...host,agents:host.agents.map(a=>({...a,binDir:'/app/bin'}))};
       const worker={image:relay.image,init:true,restart:'no',user:`${relay.environment.EZ_RUNTIME_UID||1000}:${relay.environment.EZ_RUNTIME_GID||1000}`,
         cap_drop:['ALL'],security_opt:['no-new-privileges:true'],entrypoint:['node','--import','/app/node_modules/tsx/dist/loader.mjs','/app/src/host-executor.ts','/run/executor.json'],
         environment:{...relay.environment,EZ_CONTAINER_EXECUTOR:'1',EZ_EXECUTOR_TRANSPORT:'local'},
-        volumes:[...relay.volumes,{type:'bind',source:configFile,target:'/run/executor.json',read_only:true}],tmpfs:['/tmp'],
+        volumes:relay.volumes,tmpfs:['/tmp'],
         ...Object.fromEntries(['networks','network_mode','extra_hosts','dns','dns_search'].filter(key=>relay[key]!==undefined).map(key=>[key,relay[key]]))};
       if(worker.networks&&!Array.isArray(worker.networks))worker.networks=Object.fromEntries(Object.keys(worker.networks).map(name=>[name,{}]));
       const networks=worker.networks?Object.fromEntries(Object.entries(deployment.networks||{}).map(([name,binding])=>[name,{name:binding.name,external:true}])):undefined;
-      await atomic(file,{services:{executor:worker},...(networks?{networks}:{})});
-      await execute('docker',['compose','--project-name',project,'--file',file,'up','-d','--no-build','executor']);
-      container=(await execute('docker',['compose','--project-name',project,'--file',file,'ps','-q','executor'],{stdoutOnly:true})).trim();
-      if(!/^[a-f0-9]{12,64}$/.test(container))throw Error('Missing isolated executor');
-      generations.set(container,{project,file,configFile});
+      const volumes=executorVolumes(deployment,worker);
+      const archive=executorBindingArchive(binding,Number(worker.user.split(':')[1]));
+      await atomic(file,{services:{executor:worker},...(networks?{networks}:{}),...(Object.keys(volumes).length?{volumes}:{})});
+      try {
+        await execute('docker',['compose','--project-name',project,'--file',file,'create','--no-build','executor']);
+        container=(await execute('docker',['compose','--project-name',project,'--file',file,'ps','-aq','executor'],{stdoutOnly:true})).trim();
+        if(!/^[a-f0-9]{12,64}$/.test(container))throw Error('Missing isolated executor');
+      } catch(error) {
+        await execute('docker',['compose','--project-name',project,'--file',file,'down']);
+        await fs.rm(file,{force:true});container=undefined;throw error;
+      }
+      generations.set(container,{project,file});
+      await execute('docker',['cp','-',`${container}:/run`],{input:archive});
+      await execute('docker',['start',container]);
       for(let n=0;n<300;n++){
         const running=await execute('docker',['inspect','--format','{{.State.Running}}',container],{stdoutOnly:true});
         if(running.trim()!=='true')throw Error('Isolated executor exited during startup');
@@ -206,7 +234,9 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       if(!isolated||container)return;
       const root=(await state(home)).config.packageRoot;
       if((await read(path.join(root,'package.json'))).ezRelease?.containerExecutor!==1)return;
-      await withIdleUpgrade(agent.controlDir,{id:'executor-migration'},signal,()=>startHost(root));
+      // Queued requests need this worker to drain. Existing native runs still
+      // block migration; the worker lock continues to enforce single ownership.
+      await withIdleUpgrade(agent.controlDir,{id:'executor-migration'},signal,()=>startHost(root),{isIdle:()=>idle(agent.controlDir,{pendingRequests:false})});
     };
     await initializeIsolated();
     let nextCheck=0;

package/test/updates.test.mjs

@@ -14,7 +14,7 @@ import { atomic, snapshot, compose, prepareCommand } from '../src/plugins/manage
 import { bindUpdates } from '../src/updates/binding.mjs';
 import { TOOLS_LINE } from '../src/tools-line.mjs';
 import { status as runtimeStatus } from '../src/updates/status.mjs';
-import { providerEnvironment, queueAutomatic, waitForHostHeartbeat, withIdleUpgrade, withUpgrade } from '../src/updates/supervisor.mjs';
+import { idle, executorBindingArchive, executorVolumes, providerEnvironment, queueAutomatic, waitForHostHeartbeat, withIdleUpgrade, withUpgrade } from '../src/updates/supervisor.mjs';
 const exec=promisify(execFile);
 
 test('structured runtime output excludes warnings but failures retain diagnostics',async()=>{
@@ -794,3 +794,30 @@ test('failed handoff-compatible activation retires the candidate and rolls back
  assert.equal(r.calls.filter(c=>c[0]==='retireHost').length,2)
  assert.ok(!r.calls.some(c=>c[0]==='stopHost'))
 })
+
+
+test('missing executor startup ignores queued requests but never live native work',async t=>{
+ const root=await fs.mkdtemp(path.join(tmpdir(),'ez-executor-admission-'));t.after(()=>fs.rm(root,{recursive:true,force:true}));
+ const dir=path.join(root,'host-executor');await fs.mkdir(dir);
+ await fs.writeFile(path.join(dir,'queued.request.json'),'{}');
+ assert.equal(await idle(root),false);assert.equal(await idle(root,{pendingRequests:false}),true);
+ await fs.writeFile(path.join(dir,'live.running.json'),'{}');assert.equal(await idle(root,{pendingRequests:false}),false);
+ assert.equal(await fs.readFile(path.join(dir,'queued.request.json'),'utf8'),'{}');
+ await fs.rm(path.join(dir,'live.running.json'));
+ const server=createServer(socket=>socket.on('data',chunk=>{const req=JSON.parse(chunk.toString());socket.end(JSON.stringify({id:req.id,ok:true,result:{running:1}})+'\n');}));
+ await new Promise(resolve=>server.listen(path.join(root,'delivery.sock'),resolve));t.after(()=>new Promise(resolve=>server.close(resolve)));
+ assert.equal(await idle(root,{pendingRequests:false}),false);
+});
+test('executor preserves exact named volume identities and rejects missing declarations',()=>{
+ const worker={volumes:[{type:'volume',source:'registered-whatsapp-ipc'},{type:'bind',source:'/private/workspace'}]};
+ assert.deepEqual(executorVolumes({volumes:{'registered-whatsapp-ipc':{name:'existing-private-ipc'}}},worker),{'registered-whatsapp-ipc':{name:'existing-private-ipc',external:true}});
+ assert.throws(()=>executorVolumes({},worker),/Missing executor volume/);
+});
+test('executor archive extracts its exact private binding without host ownership changes',async t=>{
+ const root=await fs.mkdtemp(path.join(tmpdir(),'ez-executor-binding-'));t.after(()=>fs.rm(root,{recursive:true,force:true}));
+ const binding={agents:[{workspace:'/private/work',binDir:'/app/bin'}]},file=path.join(root,'binding.tar');
+ await fs.writeFile(file,executorBindingArchive(binding,1000),{mode:0o600});
+ await exec('tar',['-xf',file,'-C',root]);
+ const extracted=path.join(root,'executor.json');assert.deepEqual(await read(extracted),binding);assert.equal((await fs.stat(extracted)).mode&0o777,0o440);
+ for(const invalid of [-1,NaN,1.5,'1000',0o10000000])assert.throws(()=>executorBindingArchive(binding,invalid),/Invalid executor group/);
+});