@jc_stack/ez-agents 0.1.0-beta.46 → 0.1.0-beta.48

package/test/updates.test.mjs

@@ -13,7 +13,7 @@ import { perform, environment, packageManager, backupStateDirectory, execute } f
 import { atomic, snapshot, compose, prepareCommand } from '../src/plugins/manager.mjs';
 import { bindUpdates } from '../src/updates/binding.mjs';
 import { status as runtimeStatus } from '../src/updates/status.mjs';
-import { providerEnvironment, queueAutomatic, waitForHostHeartbeat } from '../src/updates/supervisor.mjs';
+import { providerEnvironment, queueAutomatic, waitForHostHeartbeat, withIdleUpgrade } from '../src/updates/supervisor.mjs';
 const exec=promisify(execFile);
 
 test('structured runtime output excludes warnings but failures retain diagnostics',async()=>{
@@ -113,14 +113,17 @@ test('root preparation leaves private plugin updates readable by the installer',
  const script=`const fs=require('fs');fs.readFileSync(${JSON.stringify(path.join(dir,'job.json'))});fs.readFileSync(${JSON.stringify(path.join(dir,'candidate.tgz'))});`;
  await exec('setpriv',['--reuid=20001','--regid=20002','--clear-groups','--no-new-privs',process.execPath,'-e',script]);
 });
-function runtime(f,{fail,stopped=false,buildArgs={CODEX_CLI_VERSION:'0.156.1'}}={}) {
- const calls=[];let failed=false;
+function runtime(f,{fail,stopped=false,buildArgs={CODEX_CLI_VERSION:'0.156.1'},latest='0.160.0'}={}) {
+ const calls=[];let failed=false,brokerImage='sha256:'+'a'.repeat(64);
  const execute=async(command,args,opts)=>{
-  calls.push([command,...args]);if(fail&&!failed&&fail(command,args)){failed=true;throw Error('Synthetic failure');}
+  calls.push([command,...args]);if(args.includes('view'))return JSON.stringify(latest);if(fail&&!failed&&fail(command,args)){failed=true;throw Error('Synthetic failure');}
   if(args.at(-1)==='--version')return '10.30.3';
   if(args.includes('config'))return JSON.stringify({services:{relay:{build:{args:buildArgs}}}});
-  if(args.includes('ps'))return stopped?'':'container-id';
-  if(args[0]==='inspect')return 'sha256:'+'a'.repeat(64);
+  if(args.includes('ps'))return stopped?'':'a'.repeat(64);
+  if(args.some(arg=>arg.includes('EZ_BROKER_STRUCTURAL_READINESS_FAILED'))) {const [hostSha256,registrySha256,configSha256]=args.slice(-3);return JSON.stringify({version:1,ready:true,hostSha256,registrySha256,configSha256,plugins:[]});}
+  if(args[0]==='image'&&args[1]==='inspect')return 'sha256:'+'b'.repeat(64);
+  if(args.includes('--force-recreate'))brokerImage=(await fs.readFile(args[args.lastIndexOf('--env-file')+1],'utf8')).trim().split('=')[1];
+  if(args[0]==='inspect')return brokerImage;
   if(args[0]==='volume'&&args[1]==='ls')return 'existing';
   if(args[0]==='volume')return JSON.stringify([{Name:args[2]}]);
   if(args[0]==='run')await fs.writeFile(opts.outputFile,'retained-private-state',{mode:0o600});
@@ -306,7 +309,7 @@ test('main transaction stages before stopping, pins rollback image, preserves st
  assert.notEqual(await fs.readFile(path.join(active,'bin/example.mjs'),'utf8'),'tampered');
  assert(r.calls.findIndex(c=>c.includes('build'))<r.calls.findIndex(c=>c[0]==='stopHost'));
  const build=r.calls.find(c=>c[0]==='docker'&&c[1]==='build');
- assert.equal(build[build.indexOf('--build-arg')+1],'CODEX_CLI_VERSION=0.156.1');
+ assert.equal(build[build.indexOf('--build-arg')+1],'CODEX_CLI_VERSION=0.160.0');
  const status=await command(f.home,['status']);assert(!JSON.stringify(status).includes('private-test-token'));assert(!('rollback'in status.jobs[0]));
  assert.equal((await fs.stat(path.join(jobPath(f.home,job.id),'job.json'))).mode&0o777,0o600);
 });
@@ -320,12 +323,14 @@ test('main activation and rollback bind guidance from the selected package root'
  assert.equal((await perform(failed.home,await queued(failed),runtime(failed,{fail:(_command,args)=>args.includes('up')}))).status,'rolled-back');
  assert.equal(await fs.readFile(path.join(failed.agent.workspace,'AGENTS.md'),'utf8'),'old guidance\n');
 });
-test('the same archive with different deployment CLI pins gets distinct image bindings',async t=>{
+test('latest overrides stale deployment pins and changing latest changes image bindings',async t=>{
  const tags=[],hashes=[];
  for(const pin of ['0.153.4','0.156.1']) {
-  const f=await fixture(t),job=await queued(f),r=runtime(f,{buildArgs:{CODEX_CLI_VERSION:pin}});
+  const f=await fixture(t),job=await queued(f),r=runtime(f,{buildArgs:{CODEX_CLI_VERSION:'0.153.4'},latest:pin});
   assert.equal((await perform(f.home,job,r)).status,'completed');
   const build=r.calls.find(c=>c[0]==='docker'&&c[1]==='build'),tag=build[build.indexOf('-t')+1];
+  assert.ok(build.includes(`CODEX_CLI_VERSION=${pin}`));
+  assert.equal((await read(path.join(jobPath(f.home,job.id),'job.json'))).codexVersion,pin);
   assert.match(await fs.readFile(path.join(f.config.deploymentDir,'docker.env'),'utf8'),new RegExp(tag));
   tags.push(tag);hashes.push(job.sha256);
  }
@@ -382,27 +387,60 @@ test('plugin updates register additive command routes on an unchanged deployment
  const dispatch=await prepareCommand(f.home,'query',[]);assert.deepEqual(dispatch.argv.slice(-4),['node','sample','/app/bin/example.mjs','query']);
  assert(r.calls.some(call=>call.includes('up')));
 });
-test('isolated plugin replacement refreshes broker version inventory after registry activation',async t=>{
+test('isolated plugin replacement recreates only the pinned broker and verifies activation and rollback',async t=>{
  const f=await fixture(t,'plugin');
  await fs.appendFile(path.join(f.config.deploymentDir,'docker.env'),'EZ_EXECUTOR_TRANSPORT=local\n');
  const job=await queued(f),r=runtime(f),execute=r.execute;
  let observed;
  r.execute=async(command,args,options)=>{
-  if(args.includes('restart')&&args.at(-1)==='plugin-broker') observed=(await read(path.join(f.home,'registry.json'))).plugins.sample.manifest.version;
+  if(args.some(arg=>arg.includes('EZ_BROKER_STRUCTURAL_READINESS_FAILED'))) observed=(await read(path.join(f.home,'registry.json'))).plugins.sample.manifest.version;
+  if(args.includes('--force-recreate')) {
+   assert.equal(args.at(-1),'plugin-broker');
+   for(const flag of ['--no-deps','--no-build','--wait'])assert(args.includes(flag));
+   assert.equal(args[args.indexOf('--pull')+1],'never');
+   assert.equal(await fs.readFile(args[args.lastIndexOf('--env-file')+1],'utf8'),'EZ_RELAY_IMAGE=sha256:'+ 'a'.repeat(64)+'\n');
+  }
   return execute(command,args,options);
  };
  assert.equal((await perform(f.home,job,r)).status,'completed');
- assert.equal(observed,'0.1.1');
- assert.equal(r.calls.filter(call=>call.includes('restart')).length,1);
+ assert.equal(observed,'0.1.1');assert.equal(job.brokerReadiness.ready,true);
  const interrupted=await read(path.join(jobPath(f.home,job.id),'job.json'));interrupted.status='applying';
  assert.equal((await perform(f.home,interrupted,r)).status,'rolled-back');
- assert.equal(observed,'0.1.0');
- assert.equal(r.calls.filter(call=>call.includes('restart')).length,2);
+ assert.equal(observed,'0.1.0');assert.equal(interrupted.rollbackBrokerReadiness.ready,true);
+ assert.equal(r.calls.filter(call=>call.includes('--force-recreate')).length,2);
+ assert(!r.calls.some(call=>call.includes('restart')||call[0]==='stopHost'));
+});
+for(const boundary of ['recreate','probe','readback'])test(`isolated broker ${boundary} failure cannot complete an update`,async t=>{
+ const f=await fixture(t,'plugin');await fs.appendFile(path.join(f.config.deploymentDir,'docker.env'),'EZ_EXECUTOR_TRANSPORT=local\n');
+ const job=await queued(f),r=runtime(f),base=r.execute;
+ r.execute=async(c,a,o)=>{
+  if(boundary==='recreate'&&a.includes('--force-recreate')||boundary==='probe'&&a.some(arg=>arg.includes('EZ_BROKER_STRUCTURAL_READINESS_FAILED')))throw Error('Synthetic broker failure');
+  if(boundary==='readback'&&a.some(arg=>arg.includes('EZ_BROKER_STRUCTURAL_READINESS_FAILED')))return JSON.stringify({version:1,ready:true,hostSha256:'0'.repeat(64)});
+  return base(c,a,o);
+ };
+ const result=await perform(f.home,job,r);
+ assert.equal(result.status,'recovery-required');assert(result.error);assert(result.recoveryError);
+ assert.equal((await read(path.join(f.home,'registry.json'))).plugins.sample.manifest.version,'0.1.0');
+});
+test('main isolated activation and retained-relay rollback also refresh the broker',async t=>{
+ const f=await fixture(t);await fs.appendFile(path.join(f.config.deploymentDir,'docker.env'),'EZ_EXECUTOR_TRANSPORT=local\n');
+ const job=await queued(f),r=runtime(f);
+ assert.equal((await perform(f.home,job,r)).status,'completed');assert.equal(job.brokerReadiness.ready,true);
+ const base=r.execute;r.execute=async(c,a,o)=>a[0]==='inspect'&&a.includes('{{json .}}')?JSON.stringify({Image:'sha256:'+'a'.repeat(64),State:{Running:true,Health:{Status:'healthy'}}}):base(c,a,o);
+ const interrupted=await read(path.join(jobPath(f.home,job.id),'job.json'));interrupted.status='applying';
+ assert.equal((await perform(f.home,interrupted,r)).status,'rolled-back');
+ assert.equal(interrupted.rollbackBrokerReadiness.ready,true);
+ assert.equal(r.calls.filter(call=>call.includes('--force-recreate')).length,2);
+ assert.equal(r.calls.filter(call=>call.includes('up')&&call.at(-1)==='relay').length,1);
+});
+test('host-capable updates never inspect, recreate or probe an isolated broker',async t=>{
+ const f=await fixture(t,'plugin'),r=runtime(f);assert.equal((await perform(f.home,await queued(f),r)).status,'completed');
+ assert(!r.calls.some(call=>call.includes('plugin-broker')||call.some(arg=>arg.includes('EZ_BROKER_STRUCTURAL_READINESS_FAILED'))));
 });
 test('plugin update preserves a created but never running service as stopped',async t=>{
  const f=await fixture(t,'plugin'),job=await queued(f),r=runtime(f),execute=r.execute;
  r.execute=async(command,args,options)=>{
-  if(args.includes('ps')) {r.calls.push([command,...args]);return args.includes('--status')?'':'created-container';}
+  if(args.includes('ps')) {r.calls.push([command,...args]);if(args.includes('view'))return JSON.stringify(latest);return args.includes('--status')?'':'created-container';}
   return execute(command,args,options);
  };
  assert.equal((await perform(f.home,job,r)).status,'completed');
@@ -495,7 +533,7 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
  // Package archives never contain node_modules; the fake pnpm below provisions the fixture loader.
  await fs.rm(path.join(f.source,'node_modules'),{recursive:true});
  const log=path.join(f.root,'commands.jsonl');
- await fs.writeFile(path.join(fake,provider),`#!${process.execPath}\nif(${JSON.stringify(provider)}==='corepack'&&process.argv[2]!=='pnpm@10.30.3')throw Error('Unpinned manager');if(process.argv.includes('--version')){console.log('10.30.3');process.exit(0)}const fs=require('fs');fs.mkdirSync('node_modules/tsx/dist',{recursive:true});fs.writeFileSync('node_modules/tsx/dist/loader.mjs','');`,{mode:0o755});
+ await fs.writeFile(path.join(fake,provider),`#!${process.execPath}\nif(${JSON.stringify(provider)}==='corepack'&&process.argv[2]!=='pnpm@10.30.3')throw Error('Unpinned manager');if(process.argv.includes('view')){console.log(JSON.stringify('0.160.0'));process.exit(0)}if(process.argv.includes('--version')){console.log('10.30.3');process.exit(0)}const fs=require('fs');fs.mkdirSync('node_modules/tsx/dist',{recursive:true});fs.writeFileSync('node_modules/tsx/dist/loader.mjs','');`,{mode:0o755});
  await fs.writeFile(path.join(fake,'docker'),`#!${process.execPath}\nconst fs=require('fs');const a=process.argv.slice(2);fs.appendFileSync(${JSON.stringify(log)},JSON.stringify(a)+'\\n');if(a.includes('config'))console.log(JSON.stringify({services:{relay:{build:{args:{}}}}}));if(a.includes('ps'))console.log('cid');if(a[0]==='inspect')console.log('sha256:'+'a'.repeat(64));`,{mode:0o755});
  const wrapper=path.join(f.root,'supervisor.mjs'),module=new URL('../src/updates/supervisor.mjs',import.meta.url).href;
  await fs.writeFile(wrapper,`import {supervise} from ${JSON.stringify(module)};const a=new AbortController();process.on('SIGTERM',()=>a.abort());await supervise(${JSON.stringify(f.config.deploymentDir)},a.signal,{discover:async()=>{${provider==='pnpm' ? "throw Error('Synthetic discovery failure')" : 'return []'}}});`);
@@ -510,6 +548,7 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
  const requester=await exec(process.execPath,['--input-type=module','-e',`import {submit} from ${JSON.stringify(new URL('../src/updates/control.mjs',import.meta.url).href)};console.log(JSON.stringify(await submit(${JSON.stringify(f.home)},${JSON.stringify(job.id)},false)));`]);
  assert.equal(JSON.parse(requester.stdout).status,'queued');
  await new Promise(r=>setTimeout(r,1800));assert.equal((await read(path.join(jobPath(f.home,job.id),'job.json'))).status,'queued');
+ await assert.rejects(fs.readFile(path.join(f.agent.controlDir,'upgrade-pause.json')),e=>e.code==='ENOENT');
  await fs.rm(running);
  await wait(async()=>{const j=await read(path.join(jobPath(f.home,job.id),'job.json'));if(j.status==='failed'||j.status==='rolled-back')throw Error(JSON.stringify(j)+first.output());return j.status==='completed';});
  const newBeat=await heartbeat();assert.notEqual(newBeat.pid,oldBeat.pid);assert.equal(await firstClosed,0,first.output());
@@ -529,6 +568,26 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
  const closed3=new Promise(r=>third.p.once('close',r));third.p.kill('SIGTERM');assert.equal(await closed3,0,third.output());
 });
 
+test('queued upgrades keep admission open while busy and release a raced or aborted pause',async t=>{
+ const root=await fs.mkdtemp(path.join(tmpdir(),'ez-upgrade-admission-'));
+ t.after(()=>fs.rm(root,{recursive:true,force:true}));
+ const pause=path.join(root,'upgrade-pause.json'),job={id:'test'},abort=new AbortController();
+ const absent=()=>assert.rejects(fs.readFile(pause),e=>e.code==='ENOENT');
+ let applied=0;
+ const apply=async()=>{assert.equal((await read(pause)).id,job.id);applied++;return 'applied';};
+ await withIdleUpgrade(root,job,abort.signal,apply,{isIdle:async()=>false});
+ await absent();assert.equal(applied,0);
+ let checks=0;
+ await withIdleUpgrade(root,job,abort.signal,apply,{isIdle:async()=>++checks===1,wait:async()=>{}});
+ await absent();assert.equal(applied,0);
+ assert.equal(await withIdleUpgrade(root,job,abort.signal,apply,{isIdle:async()=>true,wait:async()=>{}}),'applied');
+ await absent();assert.equal(applied,1);
+ await assert.rejects(withIdleUpgrade(root,job,abort.signal,async()=>{throw Error('apply failed');},{isIdle:async()=>true,wait:async()=>{}}),/apply failed/);
+ await absent();
+ await withIdleUpgrade(root,job,abort.signal,apply,{isIdle:async()=>true,wait:async()=>abort.abort()});
+ await absent();assert.equal(applied,1);
+});
+
 test('npm candidates verify exact version and integrity; automatic policy is enforced again at execution',async t=>{
  const f=await fixture(t),data=await fs.readFile(await f.pack()),original=globalThis.fetch;
  const {createHash}=await import('node:crypto');
@@ -627,7 +686,7 @@ for(const mode of ['exact','wrong-image','unhealthy']) test(`recovery reuses onl
  const f=await fixture(t),job=await queued(f),r=runtime(f);await perform(f.home,job,r);
  const interrupted=await read(path.join(jobPath(f.home,job.id),'job.json'));interrupted.status='applying';
  const calls=[],execute=async(command,args,options)=>{
-  calls.push([command,...args]);
+  calls.push([command,...args]);if(args.includes('view'))return JSON.stringify(latest);
   if(args.includes('ps'))return 'b'.repeat(64);
   if(args[0]==='inspect'&&args.includes('{{json .}}'))return JSON.stringify({Image:'sha256:'+(mode==='wrong-image'?'c':'a').repeat(64),State:{Running:true,Health:{Status:mode==='unhealthy'?'unhealthy':'healthy'}}});
   if(mode==='exact'&&args.includes('up'))throw Error('Pinned image index no longer exists');