@jc_stack/ez-agents 0.1.0-beta.51 → 0.1.0-beta.58

package/test/setup-fallback.test.ts

@@ -10,7 +10,7 @@ import { ControlStore } from '../src/control-state.js'
 import { RunStore, activeSetup } from '../src/runs.js'
 import { Scheduler } from '../src/scheduler.js'
 import { CliUnavailableError, EXECUTOR_REGISTRY, type ExecutorOptions } from '../src/executor.js'
-import { attemptFailure, codexRejection, nextSetup, quotaScope, startupObserver, startupLine } from '../src/setup-fallback.js'
+import { attemptFailure, codexRejection, nextSetup, quotaScope, setupCandidates, startupObserver, startupLine } from '../src/setup-fallback.js'
 import type { AiPreset } from '../src/ai.js'
 
 const until=async(check:()=>Promise<boolean>)=>{for(let i=0;i<250;i++){if(await check())return;await new Promise(r=>setTimeout(r,20))}throw new Error('Timed out')}
@@ -95,7 +95,22 @@ test('a quota rejection skips models in the same login scope; named accounts hav
   assert.equal(nextSetup([claudeA,claudeB],[{...first,category:'provider-rejected'}]).preset,claudeB)
 })
 
-type Script={lines?:unknown[];code?:number;unavailable?:boolean;hold?:boolean}
+test('same-settings login choices rotate from the last success, preserving explicit model fallback priority',()=>{
+  const a={...claudeA,effort:'xhigh'},b={...a,id:'b',authProfile:'claude2'},c={...a,id:'c',authProfile:'claude3'}
+  const execution={sessionId:'00000000-0000-4000-8000-000000000000',preset:a,fallbacks:[b,c,codexA]}
+  const health={'claude@claude2':{lastSuccessAt:'2026-01-01T00:00:00.000Z'}}
+  assert.deepEqual(setupCandidates(execution,health).map(p=>p.id),['b','c','claude-a','codex-a'])
+  assert.deepEqual(setupCandidates({...execution,fallbacks:[codexA,b]},health).map(p=>p.id),['claude-a','codex-a','b'])
+  const now=Date.parse('2026-01-01T01:00:00.000Z'), resetAt='2026-01-01T02:00:00.000Z'
+  const cooling={claude:{resetAt},'claude@claude2':{...health['claude@claude2'],resetAt}}
+  const next=nextSetup([a,b,c],[],cooling,now)
+  assert.equal(next.preset,c)
+  assert.deepEqual(next.skipped.map(s=>[s.quotaScope,s.category,s.resetAt]),[['claude','same-quota-scope',resetAt],['claude@claude2','same-quota-scope',resetAt]])
+  assert.equal(nextSetup([a,b,c],[],cooling,Date.parse(resetAt)).preset,a,'the native reset expires the cooldown')
+  assert.equal(nextSetup([b,c,a],[{preset:b,quotaScope:'claude@claude2',outcome:'failed',category:'quota'}]).preset,c,'rotation never retries an attempted login')
+})
+
+type Script={lines?:unknown[];code?:number;unavailable?:boolean;hold?:boolean;session?:string;stage?:string}
 const fixture=async(name:string,scripts:Script[],fallbacks=[claudeB,codexA,codexB],primary=claudeA)=>{
   const dir=await mkdtemp(join(tmpdir(),`ez-fallback-${name}-`)),control=new ControlStore(dir,1000),runs=new RunStore(dir),scheduler=new Scheduler(dir)
   const launches:ExecutorOptions[]=[],prompts:string[][]=[],admitted:(AiPreset|undefined)[]=[],sent:string[]=[]
@@ -103,6 +118,8 @@ const fixture=async(name:string,scripts:Script[],fallbacks=[claudeB,codexA,codex
     const script=scripts[launches.length] ?? {code:0}
     launches.push(options);prompts.push(texts);admitted.push(activeSetup((await runs.get(options.runId))!))
     if(script.unavailable)throw new CliUnavailableError(`Native CLI ${options.cli} is not executable on the host PATH`)
+    if(script.stage){const s=await scheduler.get('daily');await scheduler.edit({...s,text:script.stage},s.revision,()=>runs.list(),true)}
+    if (script.session) await options.onSession?.(script.session)
     const out=(script.lines ?? []).map(l=>typeof l==='string'?l:JSON.stringify(l)).join('\n')
     const child=spawn(process.execPath,['-e',`process.stdout.write(${JSON.stringify(out ? out+'\n' : '')});setTimeout(()=>process.exit(${script.code ?? 0}),${script.hold ? 60000 : 10})`],{detached:process.platform!=='win32'})
     await once(child,'spawn');return {child,cleanup:async()=>{},stdout:''}
@@ -110,10 +127,10 @@ const fixture=async(name:string,scripts:Script[],fallbacks=[claudeB,codexA,codex
   relay.bot.api.config.use(async(_prev,method,payload)=>{if(method==='sendMessage')sent.push((payload as {text:string}).text);return {ok:true,result:{message_id:42}} as never})
   await control.requestPairing(101,101);await control.approveOwner(101)
   const owner=(await control.status()).owner!
-  const saved=await scheduler.save({id:'daily',name:'Daily',text:'Saved work',owner,execution:{sessionId:'00000000-0000-4000-8000-000000000000',preset:primary,fallbacks},enabled:true,trigger:{at:'2027-01-01T00:00:00Z'}},true)
+  const saved=await scheduler.save({id:'daily',name:'Daily',text:'Saved work',owner,execution:{sessionId:'00000000-0000-4000-8000-000000000000',preset:primary,...(fallbacks.length ? {fallbacks} : {})},enabled:true,trigger:{at:'2027-01-01T00:00:00Z'}},true)
   const run=await scheduler.trigger(saved.id,saved.revision,'fallback-'+name,owner,runs)
   await relay.drainSources()
-  return {dir,runs,scheduler,relay,run,launches,prompts,admitted,sent,done:async()=>{await until(async()=>['completed','failed','cancelled'].includes((await runs.get(run.id))!.status));return (await runs.get(run.id))!},
+  return {dir,owner,runs,scheduler,relay,run,launches,prompts,admitted,sent,done:async()=>{await until(async()=>['completed','failed','cancelled'].includes((await runs.get(run.id))!.status));return (await runs.get(run.id))!},
     close:async()=>{await relay.stop();await rm(dir,{recursive:true,force:true})}}
 }
 const quota={type:'assistant',error:'rate_limit'},work={type:'assistant',message:{content:[{type:'text',text:'working'}]}}
@@ -209,6 +226,29 @@ test('four Claude accounts advance in order on quota rejection before work',asyn
   }finally{await f.close()}
 })
 
+test('later tasks reuse the last working login and retain quota resets in owner control state',async()=>{
+  const primary={...claudeA,effort:'xhigh'},profiles=['claude2','claude3'].map((authProfile,i)=>({...primary,id:`fallback-${i}`,authProfile}))
+  const resetAt=new Date(Date.now()+3600_000).toISOString(),limit={type:'ez.startup',workBegan:false,rejection:{category:'quota',resetAt}}
+  const f=await fixture('sticky-login',[{lines:[limit],code:1},{lines:[work],code:0},{lines:[limit],code:1},{lines:[work],code:0},{lines:[work],code:0}],profiles,primary)
+  try{
+    await f.done()
+    const control=new ControlStore(f.dir,1000),owner=(await control.status()).owner!
+    assert.equal((await control.scheduledSetupHealth(owner)).claude.resetAt,resetAt)
+    for(const id of ['second','third']){
+      const saved=await f.scheduler.save({id,name:id,text:'Another saved task',owner,execution:{sessionId:'00000000-0000-4000-8000-000000000000',preset:primary,fallbacks:profiles},enabled:true,trigger:{at:'2027-01-01T00:00:00Z'}},true)
+      const run=await f.scheduler.trigger(id,saved.revision,id,owner,f.runs)
+      await f.relay.drainSources()
+      await until(async()=>(await f.runs.get(run.id))?.status==='completed')
+    }
+    assert.deepEqual(f.launches.map(l=>l.authProfile),[undefined,'claude2','claude2','claude3','claude3'])
+    assert.ok(f.launches.every(l=>l.model==='opus'&&l.effort==='xhigh'))
+    const persisted=await new ControlStore(f.dir,1000).scheduledSetupHealth(owner)
+    assert.equal(persisted.claude.resetAt,resetAt)
+    assert.equal(persisted['claude@claude2'].resetAt,resetAt)
+    assert.ok(persisted['claude@claude3'].lastSuccessAt)
+  }finally{await f.close()}
+})
+
 test('arbitrary mixed setup lists preserve CLI, provider, model and effort per attempt',async()=>{
   const candidates:AiPreset[]=[claudeA,
     {...codexA,authProfile:'work',effort:'xhigh'},
@@ -251,13 +291,19 @@ test('a usage limit after work began continues the same run on the next login, t
   }
 })
 
-test('a run that reaches the limit on every login fails with each setup recorded',async()=>{
-  const f=await fixture('limit-everywhere',[{lines:[work,limitEvent,limitMessage],code:1},{lines:[limitEvent,limitMessage],code:1}],[{...claudeA,authProfile:'claude2'}])
+test('a run that reaches the limit on every login waits and manually continues the same occurrence',async()=>{
+  const f=await fixture('limit-everywhere',[{lines:[work,limitEvent,limitMessage],code:0},{lines:[limitEvent,limitMessage],code:1}],[{...claudeA,authProfile:'claude2'}])
   try{
-    const run=await f.done()
-    assert.equal(run.status,'failed');assert.equal(run.failureReason,'setups-unavailable');assert.equal(f.launches.length,2)
+    await until(async()=>Boolean((await f.runs.get(f.run.id))?.quotaWait))
+    const run=(await f.runs.get(f.run.id))!
+    assert.equal(run.status,'queued');assert.equal(run.failure,undefined);assert.equal(f.launches.length,2)
     assert.deepEqual(run.attempts!.map(a=>[a.quotaScope,a.category]),[['claude','quota-after-work-began'],['claude@claude2','quota']])
-    assert.match(run.failure!.error,/quota-after-work-began, resets 2026-10-09T18:10:00\.000Z.*quota, resets/)
+    await f.relay.drainSources();assert.equal(f.launches.length,2)
+    await f.scheduler.retryQuota(f.owner,f.runs,run.id)
+    await f.relay.drainSources()
+    assert.equal((await f.done()).status,'completed')
+    assert.equal(f.launches[2].runId,run.id)
+    assert.ok(f.prompts[2].some(p=>p.includes(f.launches[0].sessionId!) && p.includes('partly applied')))
   }finally{await f.close()}
 })
 
@@ -273,6 +319,18 @@ test('cancellation stops the chain',async()=>{
   }finally{await f.close()}
 })
 
+test('a staged edit lets the running occurrence continue to its saved fallback on captured settings',async()=>{
+  const f=await fixture('staged',[{stage:'Edited work',lines:[quota],code:1},{lines:[startupLine({workBegan:true})],code:0}])
+  try{
+    const run=await f.done()
+    assert.equal(run.status,'completed')
+    assert.deepEqual(run.attempts!.map(a=>[a.preset.id,a.outcome]),[['claude-a','failed'],['claude-b','skipped'],['codex-a','completed']])
+    assert.ok(f.prompts.every(texts=>texts.includes('Saved work')),'every attempt runs the captured instructions')
+    const saved=await f.scheduler.get('daily')
+    assert.equal(saved.text,'Edited work');assert.deepEqual(saved.continuingRunIds,[f.run.id])
+  }finally{await f.close()}
+})
+
 test('when every setup is unavailable the run reports each setup clearly',async()=>{
   const f=await fixture('none',[{unavailable:true},{unavailable:true}],[codexA])
   try{
@@ -300,6 +358,7 @@ test('schedules reject duplicate or invalid setups and the socket cannot rewrite
     try{
       const run=await new RunStore(dir).create({chatId:101,telegramUserId:101,texts:['x'],execution:execution([claudeB])})
       await assert.rejects(callDeliverySocket(socketPathFor(dir),{op:'patch',payload:{runId:run.id,change:{attempts:forged}}}),/relay-owned/)
+      for (const key of ['quotaWait','attemptRound']) await assert.rejects(callDeliverySocket(socketPathFor(dir),{op:'patch',payload:{runId:run.id,change:{[key]:{}}}}),/relay-owned/)
     }finally{await ledger.stop()}
   }finally{await rm(dir,{recursive:true,force:true})}
 })
@@ -357,7 +416,7 @@ test('Codex native session reports typed pre-work rejection and work start',asyn
     const program=`const rl=require('readline').createInterface({input:process.stdin});const send=x=>process.stdout.write(JSON.stringify(x)+'\\n');
 const event=(method,params)=>send({method,params:{threadId:'th',...params}});
 rl.on('line',line=>{const q=JSON.parse(line);if(!q.id)return;
-if(q.method==='initialize')return send({id:q.id,result:{}});
+if(q.method==='initialize' || q.method==='account/rateLimits/read')return send({id:q.id,result:{}});
 if(q.method==='thread/start')return send({id:q.id,result:{thread:{id:'th'}}});
 if(q.method==='turn/start'){send({id:q.id,result:{turn:{id:'one'}}});event('turn/started',{turn:{id:'one'}});
  event('item/started',{item:{type:'userMessage'}});${worked ? "event('item/started',{item:{type:'agentMessage'}});" : ''}
@@ -367,7 +426,7 @@ if(q.method==='turn/start'){send({id:q.id,result:{turn:{id:'one'}}});event('turn
     const code=await runCodexSession({workspace:'/tmp',controlDir:'/tmp/control',model:'gpt-6-sol',prompt:'work'},{launch:()=>spawn(process.execPath,['-e',program],{stdio:['pipe','pipe','pipe'],detached:process.platform!=='win32'}),emit:line=>output.push(line)})
     assert.equal(code,1)
     const o=startupObserver();o.write(output.join('\n')+'\n')
-    assert.deepEqual(o.result(),worked ? {workBegan:true,rejection:undefined} : {workBegan:false,rejection:{category:'quota',resetAt:'2026-10-08T12:00:00.000Z'}})
+    assert.deepEqual(o.result(),{workBegan:worked,rejection:{category:'quota',resetAt:'2026-10-08T12:00:00.000Z'}})
   }
 })
 
@@ -382,8 +441,9 @@ test('schedule CLI saves ordered fallbacks, preserves them on edit and clears th
   const when=['--at','2027-09-09T09:00:00+04:00','--text','Work']
   const saved=await cli('create','chain','--cli','claude','--model','opus',...when,'--fallback','cli=claude,model=sonnet,effort=medium','--fallback','cli=codex,model=gpt-6-sol')
   assert.deepEqual(saved.execution.fallbacks.map((p:AiPreset)=>[p.cli,p.model,p.effort]),[['claude','sonnet','medium'],['codex','gpt-6-sol',undefined]])
-  assert.deepEqual((await cli('edit','chain','--cli','claude','--model','opus',...when)).execution.fallbacks,saved.execution.fallbacks)
-  assert.equal((await cli('edit','chain','--cli','claude','--model','opus',...when,'--clear-fallbacks')).execution.fallbacks,undefined)
+  const kept=await cli('edit','chain','--expected-revision',saved.revision,'--cli','claude','--model','opus',...when)
+  assert.deepEqual(kept.execution.fallbacks,saved.execution.fallbacks)
+  assert.equal((await cli('edit','chain','--expected-revision',kept.revision,'--cli','claude','--model','opus',...when,'--clear-fallbacks')).execution.fallbacks,undefined)
   await assert.rejects(exec(process.execPath,[bin,'create','bad','--cli','claude','--model','opus',...when,'--fallback','cli=codex'],{env}),/cli=EXECUTOR,model=MODEL/)
   await assert.rejects(exec(process.execPath,[bin,'create','dup','--cli','claude','--model','opus',...when,'--fallback','cli=claude,model=opus'],{env}),/distinct/)
 })
@@ -409,7 +469,7 @@ test('schedule CLI saves arbitrary profile/model/effort combinations and validat
     `cli=${cli},model=${model},effort=${effort}${profile?`,auth-profile=${profile}`:''}${provider?`,provider=${provider}`:''}`])
   const saved=await cli('create','chain','--cli','claude','--model','opus','--effort','high',...when,...flags)
   assert.deepEqual(saved.execution.fallbacks.map((p:AiPreset)=>[p.cli,p.authProfile,p.provider,p.model,p.effort]),tuples)
-  assert.deepEqual((await cli('edit','chain',...when)).execution,saved.execution)
+  assert.deepEqual((await cli('edit','chain','--expected-revision',saved.revision,...when)).execution,saved.execution)
   const base=[bin,'create','bad','--cli','claude','--model','opus',...when,'--fallback']
   for(const [spec,error] of [
     ['cli=claude,model=opus,auth-profile=missing',/not provisioned/],
@@ -421,3 +481,140 @@ test('schedule CLI saves arbitrary profile/model/effort combinations and validat
   ] as const)await assert.rejects(exec(process.execPath,[...base,spec],{env}),error)
   await assert.rejects(exec(process.execPath,[...base,'cli=claude,model=opus,auth-profile=claude2','--fallback','cli=claude,model=opus,auth-profile=claude2'],{env}),/distinct/)
 })
+
+
+test('a single saved Codex setup waits for its reported reset and completes without duplicating the occurrence',async()=>{
+  const resetAt=new Date(Date.now()+900).toISOString()
+  const f=await fixture('timed-wake',[{lines:[startupLine({workBegan:false,rejection:{category:'quota',resetAt}})],code:1},{lines:[startupLine({workBegan:true})],code:0}],[],codexA)
+  try {
+    await until(async()=>Boolean((await f.runs.get(f.run.id))?.quotaWait))
+    assert.equal((await f.runs.get(f.run.id))!.quotaWait!.retryAt,resetAt)
+    await f.relay.drainSources();assert.equal(f.launches.length,1)
+    await new Promise(resolve=>setTimeout(resolve,Math.max(0,Date.parse(resetAt)-Date.now()+10)))
+    await f.relay.drainSources()
+    assert.equal((await f.done()).status,'completed')
+    assert.equal(f.launches.length,2)
+    assert.equal((await f.runs.list()).filter(r=>r.scheduled?.id==='daily').length,1)
+    assert.equal((await f.runs.get(f.run.id))!.attempts!.length,2)
+  } finally { await f.close() }
+})
+
+test('credits-required waits for explicit retry; the owner Telegram command wakes it without inference',async()=>{
+  const event={type:'rate_limit_event',rate_limit_info:{status:'rejected',errorCode:'credits_required'}}
+  assert.equal(observe(event,quota).rejection!.category,'credits-required')
+  const f=await fixture('credits',[{lines:[event,quota],code:1},{lines:[work],code:0}],[])
+  try {
+    await until(async()=>Boolean((await f.runs.get(f.run.id))?.quotaWait))
+    assert.deepEqual((await f.runs.get(f.run.id))!.quotaWait,{})
+    assert.equal((await new ControlStore(f.dir,1000).scheduledSetupHealth(f.owner)).claude.manualRetry,true)
+    await f.relay.drainSources();assert.equal(f.launches.length,1)
+    f.relay.bot.botInfo={id:9,is_bot:true,first_name:'Fixture',username:'fixture_bot'} as typeof f.relay.bot.botInfo
+    await f.relay.bot.handleUpdate({update_id:700,message:{message_id:700,date:0,text:'/retry_quota',from:{id:101,is_bot:false,first_name:'Owner'},chat:{id:101,type:'private',first_name:'Owner'}}})
+    await f.relay.drainSources()
+    assert.equal((await f.done()).status,'completed')
+    assert.ok(f.sent.some(text=>text.includes('Quota retry queued')))
+  } finally { await f.close() }
+})
+
+test('waiting tasks share one login probe and leave the execution slot available',async()=>{
+  const f=await fixture('shared-probe',[{lines:[quota],code:1},{hold:true}],[])
+  try {
+    await until(async()=>Boolean((await f.runs.get(f.run.id))?.quotaWait))
+    const first=await f.scheduler.get('daily')
+    const second=await f.scheduler.save({...first,id:'second'})
+    const run=await f.scheduler.trigger(second.id,second.revision,'second',f.owner,f.runs)
+    await f.relay.drainSources()
+    assert.equal((await f.runs.get(run.id))!.status,'queued')
+    assert.equal(f.launches.length,1)
+    await f.scheduler.retryQuota(f.owner,f.runs)
+    await f.relay.drainSources()
+    assert.equal(f.launches.length,2,'one probe for both queued tasks')
+    assert.equal((await f.runs.get(run.id))!.status,'queued')
+    const third=await f.scheduler.save({...first,id:'third',execution:{...first.execution!,preset:codexA}})
+    await f.scheduler.trigger(third.id,third.revision,'third',f.owner,f.runs)
+    await f.relay.drainSources()
+    assert.equal(f.launches.length,3,'another login can use the released slot')
+  } finally { await f.close() }
+})
+
+test('manual quota retry rejects cancelled, edited, foreign and non-quota runs',async()=>{
+  const f=await fixture('retry-authority',[{lines:[quota],code:1}],[])
+  try {
+    await until(async()=>Boolean((await f.runs.get(f.run.id))?.quotaWait))
+    await assert.rejects(f.scheduler.retryQuota({...f.owner,generation:'00000000-0000-4000-8000-000000000000'},f.runs,f.run.id),/not a current/)
+    await f.runs.patch(f.run.id,{status:'failed',quotaWait:undefined,failureReason:'executor-exit'})
+    await assert.rejects(f.scheduler.retryQuota(f.owner,f.runs,f.run.id),/not a current/)
+    await f.runs.patch(f.run.id,{status:'queued',quotaWait:{}})
+    await f.scheduler.cancel(f.run.id)
+    await assert.rejects(f.scheduler.retryQuota(f.owner,f.runs,f.run.id),/not a current/)
+    const saved=await f.scheduler.get('daily')
+    await f.scheduler.save({...saved,text:'Edited work'})
+    await assert.rejects(f.scheduler.retryQuota(f.owner,f.runs,f.run.id),/not a current/)
+    assert.ok((await new ControlStore(f.dir,1000).scheduledSetupHealth(f.owner)).claude.retryAt)
+  } finally { await f.close() }
+})
+
+
+test('installed schedule retry command revives only a typed prior quota failure, retaining run identity',async()=>{
+  const f=await fixture('cli-retry',[{lines:[quota],code:1},{lines:[work],code:0}],[])
+  const {serveTestLedger}=await import('./helpers/ledger.js')
+  const {execFile}=await import('node:child_process'),{promisify}=await import('node:util'),{fileURLToPath}=await import('node:url')
+  const exec=promisify(execFile),bin=fileURLToPath(new URL('../bin/ezenciel-agents-schedule.mjs',import.meta.url))
+  let ledger:Awaited<ReturnType<typeof serveTestLedger>>|undefined
+  try {
+    await until(async()=>Boolean((await f.runs.get(f.run.id))?.quotaWait))
+    await f.runs.patch(f.run.id,{status:'failed',quotaWait:undefined,failureReason:'setups-unavailable',endedAt:new Date().toISOString()})
+    ledger=await serveTestLedger(f.dir)
+    const result=JSON.parse((await exec(process.execPath,[bin,'retry',f.run.id],{env:{...process.env,EZ_CONTROL_DIR:f.dir,EZ_RUN_ID:''}})).stdout)
+    assert.deepEqual(result.runIds,[f.run.id]);assert.deepEqual(result.scopes,['claude'])
+    await f.relay.drainSources()
+    const run=await f.done();assert.equal(run.status,'completed');assert.equal(run.failure,undefined)
+    assert.equal(run.attempts!.length,2)
+    assert.equal((await f.runs.list()).filter(r=>r.scheduled?.id==='daily').length,1)
+  } finally { await ledger?.stop();await f.close() }
+})
+
+
+test('Codex quota continuation retains the actual stopped native thread, not the assigned transport ID',async()=>{
+  const native='019a0000-0000-7000-8000-000000000001'
+  const f=await fixture('native-reference',[{session:native,lines:[startupLine({workBegan:true,rejection:{category:'quota'}})],code:1},{session:'019a0000-0000-7000-8000-000000000002',lines:[startupLine({workBegan:true})]}],[],codexA)
+  try {
+    await until(async()=>Boolean((await f.runs.get(f.run.id))?.quotaWait))
+    assert.notEqual(f.launches[0].sessionId,native)
+    assert.equal((await f.runs.get(f.run.id))!.attempts![0].sessionId,native)
+    await f.scheduler.retryQuota(f.owner,f.runs,f.run.id);await f.relay.drainSources()
+    const ended=await f.done()
+    assert.equal(ended.attempts![0].sessionId,native)
+    assert.ok(f.prompts[1].some(line=>line.includes(native) && line.includes('partly applied')))
+    assert.ok(!f.prompts[1].some(line=>line.includes(f.launches[0].sessionId!)))
+  } finally { await f.close() }
+})
+
+test('socket quota retry shares the admission lock with timed scheduler starts',async t=>{
+  const f=await fixture('retry-start-race',[{lines:[quota],code:1},{hold:true}],[])
+  const {callDeliverySocket,socketPathFor,deliverySocketAlive}=await import('../src/delivery-socket.js')
+  let running:Promise<void>|undefined
+  let release!:()=>void,entered!:()=>void
+  const blocked=new Promise<void>(resolve=>{release=resolve}),atClear=new Promise<void>(resolve=>{entered=resolve})
+  const clear=ControlStore.prototype.clearQuotaWait
+  try {
+    await until(async()=>Boolean((await f.runs.get(f.run.id))?.quotaWait))
+    f.relay.bot.botInfo={id:9,is_bot:true,first_name:'Fixture',username:'fixture_bot'} as typeof f.relay.bot.botInfo
+    t.mock.method(f.relay.bot,'start',async()=>{})
+    running=f.relay.start()
+    await until(async()=>deliverySocketAlive(socketPathFor(f.dir)))
+    t.mock.method(ControlStore.prototype,'clearQuotaWait',async function(this:ControlStore,owner:Parameters<typeof clear>[0],scopes:string[]) {
+      await clear.call(this,owner,scopes)
+      entered();await blocked
+    })
+    const retry=callDeliverySocket(socketPathFor(f.dir),{op:'retryScheduleQuota',payload:{runId:f.run.id}})
+    await atClear
+    const dispatch=f.relay.drainSources()
+    await new Promise(resolve=>setTimeout(resolve,30))
+    assert.equal(f.launches.length,1,'the timed start waits while retry changes admission')
+    release();await retry;await dispatch
+    assert.equal(f.launches.length,2)
+    assert.equal((await f.runs.get(f.run.id))!.status,'running')
+    await f.relay.drainSources();assert.equal(f.launches.length,2,'no duplicate native child')
+  } finally { release();await f.close();await running }
+})

package/test/updates.test.mjs

@@ -14,7 +14,7 @@ import { atomic, snapshot, compose, prepareCommand } from '../src/plugins/manage
 import { bindUpdates } from '../src/updates/binding.mjs';
 import { TOOLS_LINE } from '../src/tools-line.mjs';
 import { status as runtimeStatus } from '../src/updates/status.mjs';
-import { providerEnvironment, queueAutomatic, waitForHostHeartbeat, withIdleUpgrade, withUpgrade } from '../src/updates/supervisor.mjs';
+import { idle, executorBindingArchive, executorVolumes, providerEnvironment, queueAutomatic, waitForHostHeartbeat, withIdleUpgrade, withUpgrade } from '../src/updates/supervisor.mjs';
 const exec=promisify(execFile);
 
 test('structured runtime output excludes warnings but failures retain diagnostics',async()=>{
@@ -531,6 +531,8 @@ test('missing or broken managers fail with repair guidance before installing or
 
 for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider} drains work, replaces host PID and recovers after restart`,async t=>{
  const f=await fixture(t),fake=path.join(f.root,'fake');await fs.mkdir(fake);
+ const server=createServer(socket=>socket.on('data',chunk=>{const req=JSON.parse(chunk.toString());socket.end(JSON.stringify({id:req.id,ok:true,result:{running:0}})+'\n');}));
+ await new Promise(resolve=>server.listen(path.join(f.agent.controlDir,'delivery.sock'),resolve));t.after(()=>new Promise(resolve=>server.close(resolve)));
  // A process may die after creating a job directory but before atomically
  // committing its receipt. That directory must not block transport startup.
  await fs.mkdir(path.join(f.home,'updates','d18e847a-bb59-49c6-96f3-27fdc43ca44f'));
@@ -543,9 +545,9 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
  }
  // Package archives never contain node_modules; the fake pnpm below provisions the fixture loader.
  await fs.rm(path.join(f.source,'node_modules'),{recursive:true});
- const log=path.join(f.root,'commands.jsonl');
+ const log=path.join(f.root,'commands.jsonl'),held=path.join(f.agent.controlDir,'host-executor/held.request.json'),retiring=path.join(f.agent.controlDir,'host-executor/retiring.running.json');
  await fs.writeFile(path.join(fake,provider),`#!${process.execPath}\nif(${JSON.stringify(provider)}==='corepack'&&process.argv[2]!=='pnpm@10.30.3')throw Error('Unpinned manager');if(process.argv.includes('view')){console.log(JSON.stringify('0.160.0'));process.exit(0)}if(process.argv.includes('--version')){console.log('10.30.3');process.exit(0)}const fs=require('fs');fs.mkdirSync('node_modules/tsx/dist',{recursive:true});fs.writeFileSync('node_modules/tsx/dist/loader.mjs','');`,{mode:0o755});
- await fs.writeFile(path.join(fake,'docker'),`#!${process.execPath}\nconst fs=require('fs');const a=process.argv.slice(2);fs.appendFileSync(${JSON.stringify(log)},JSON.stringify(a)+'\\n');if(a.includes('config'))console.log(JSON.stringify({services:{relay:{build:{args:{}}}}}));if(a.includes('ps'))console.log('cid');if(a[0]==='inspect')console.log('sha256:'+'a'.repeat(64));`,{mode:0o755});
+ await fs.writeFile(path.join(fake,'docker'),`#!${process.execPath}\nconst fs=require('fs');const a=process.argv.slice(2);fs.appendFileSync(${JSON.stringify(log)},JSON.stringify(a)+'\\n');if(a.includes('config'))console.log(JSON.stringify({services:{relay:{build:{args:{}}}}}));if(a.includes('ps'))console.log('cid');if(a[0]==='inspect')console.log('sha256:'+'a'.repeat(64));if(a.includes('up')&&!fs.existsSync(${JSON.stringify(held)})){fs.writeFileSync(${JSON.stringify(held)},'queued request');fs.writeFileSync(${JSON.stringify(retiring)},'active retired generation');}`,{mode:0o755});
  const wrapper=path.join(f.root,'supervisor.mjs'),module=new URL('../src/updates/supervisor.mjs',import.meta.url).href;
  await fs.writeFile(wrapper,`import {supervise} from ${JSON.stringify(module)};const a=new AbortController();process.on('SIGTERM',()=>a.abort());await supervise(${JSON.stringify(f.config.deploymentDir)},a.signal,{discover:async()=>{${provider==='pnpm' ? "throw Error('Synthetic discovery failure')" : 'return []'}}});`);
  const start=()=>{const p=spawn(process.execPath,[wrapper],{env:{...process.env,PATH:fake},stdio:['ignore','pipe','pipe']});let output='';p.stdout.on('data',b=>output+=b);p.stderr.on('data',b=>output+=b);return {p,output:()=>output};};
@@ -562,21 +564,38 @@ for(const provider of ['pnpm','corepack']) test(`supervisor with only ${provider
  await assert.rejects(fs.readFile(path.join(f.agent.controlDir,'upgrade-pause.json')),e=>e.code==='ENOENT');
  await fs.rm(running);
  await wait(async()=>{const j=await read(path.join(jobPath(f.home,job.id),'job.json'));if(j.status==='failed'||j.status==='rolled-back')throw Error(JSON.stringify(j)+first.output());return j.status==='completed';});
- const newBeat=await heartbeat();assert.notEqual(newBeat.pid,oldBeat.pid);assert.equal(await firstClosed,0,first.output());
+ const newBeat=await heartbeat();assert.notEqual(newBeat.pid,oldBeat.pid);
+ await wait(async()=>await fs.access(path.join(f.agent.controlDir,'upgrade-pause.json')).then(()=>true,()=>false));
+ assert.equal(first.p.exitCode,null,'active retiring work must finish before service reload');
+ assert.equal(await fs.readFile(retiring,'utf8'),'active retired generation');
+ assert.equal(await fs.readFile(held,'utf8'),'queued request');
+ await fs.rm(retiring);
+ assert.equal(await firstClosed,0,first.output());
+ assert.equal((await read(path.join(f.agent.controlDir,'upgrade-pause.json'))).id,job.id,'admission stays closed across supervisor exit');
  if(provider==='pnpm'){assert.match(first.output(),/Update discovery failed; host remains running/);assert.doesNotMatch(first.output(),/Synthetic discovery failure/);}
  // Adversarial: a completed update must not publish an agent wakeup notice.
  await assert.rejects(fs.readFile(path.join(f.agent.controlDir,'update-attention.json')),error=>error.code==='ENOENT');
  const active=(await read(path.join(f.home,'config.json'))).packageRoot;assert(active.endsWith('/runtime'));
  assert.equal((await read(path.join(jobPath(f.home,job.id),'job.json'))).packageManager.command,provider);
+ // Failed successor startup must not reopen admission or consume pending work.
+ const activeHost=path.join(active,'src/host-executor.ts');await fs.writeFile(activeHost,"throw Error('Synthetic worker startup failure')");
+ const broken=start(),brokenClosed=new Promise(resolve=>broken.p.once('close',resolve));t.after(()=>broken.p.kill('SIGTERM'));
+ assert.equal(await brokenClosed,1,broken.output());
+ assert.equal((await read(path.join(f.agent.controlDir,'upgrade-pause.json'))).id,job.id);
+ assert.equal(await fs.readFile(held,'utf8'),'queued request');
+ await fs.writeFile(activeHost,hostCode);
  const second=start();t.after(()=>second.p.kill('SIGTERM'));
  await wait(async()=>{const h=await heartbeat();return h?.pid!==newBeat.pid&&h?.at>newBeat.at;});
+ await wait(async()=>await fs.access(path.join(f.agent.controlDir,'upgrade-pause.json')).then(()=>false,()=>true));
+ assert.equal(await fs.readFile(held,'utf8'),'queued request','successor resumes without removing queued requests');
  const interrupted=await read(path.join(jobPath(f.home,job.id),'job.json'));interrupted.status='applying';await atomic(path.join(jobPath(f.home,job.id),'job.json'),interrupted);
  await atomic(path.join(f.home,'registry.lock'),{pid:second.p.pid});
  const closed2=new Promise(r=>second.p.once('close',r));second.p.kill('SIGKILL');await closed2;
  const third=start(),closed3=new Promise(r=>third.p.once('close',r));t.after(()=>third.p.kill('SIGTERM'));
  await wait(async()=>{const j=await read(path.join(jobPath(f.home,job.id),'job.json'));return j.status==='rolled-back';});
  assert.equal((await read(path.join(f.home,'config.json'))).packageRoot,f.old);
- // A rolled-back main update reloads automatically once native work is idle.
+ // The exact post-rollback regression: an unclaimed request cannot starve reload.
+ assert.equal(await fs.readFile(held,'utf8'),'queued request');
  await wait(()=>third.p.exitCode!==null||third.p.signalCode!==null);
  assert.equal(await closed3,0,third.output());
 });
@@ -794,3 +813,32 @@ test('failed handoff-compatible activation retires the candidate and rolls back
  assert.equal(r.calls.filter(c=>c[0]==='retireHost').length,2)
  assert.ok(!r.calls.some(c=>c[0]==='stopHost'))
 })
+
+
+test('missing executor startup ignores queued requests but never live native work',async t=>{
+ const root=await fs.mkdtemp(path.join(tmpdir(),'ez-executor-admission-'));t.after(()=>fs.rm(root,{recursive:true,force:true}));
+ const dir=path.join(root,'host-executor');await fs.mkdir(dir);
+ await fs.writeFile(path.join(dir,'queued.request.json'),'{}');
+ assert.equal(await idle(root),false);assert.equal(await idle(root,{pendingRequests:false}),true);
+ assert.equal(await idle(root,{pendingRequests:false,relayRequired:true}),false,'reload requires the live admission barrier');
+ await fs.writeFile(path.join(dir,'live.running.json'),'{}');assert.equal(await idle(root,{pendingRequests:false}),false);
+ assert.equal(await fs.readFile(path.join(dir,'queued.request.json'),'utf8'),'{}');
+ await fs.rm(path.join(dir,'live.running.json'));
+ const server=createServer(socket=>socket.on('data',chunk=>{const req=JSON.parse(chunk.toString());socket.end(JSON.stringify({id:req.id,ok:true,result:{running:1}})+'\n');}));
+ await new Promise(resolve=>server.listen(path.join(root,'delivery.sock'),resolve));t.after(()=>new Promise(resolve=>server.close(resolve)));
+ assert.equal(await idle(root,{pendingRequests:false}),false);
+ assert.equal(await idle(root,{pendingRequests:false,relayRequired:true}),false);
+});
+test('executor preserves exact named volume identities and rejects missing declarations',()=>{
+ const worker={volumes:[{type:'volume',source:'registered-whatsapp-ipc'},{type:'bind',source:'/private/workspace'}]};
+ assert.deepEqual(executorVolumes({volumes:{'registered-whatsapp-ipc':{name:'existing-private-ipc'}}},worker),{'registered-whatsapp-ipc':{name:'existing-private-ipc',external:true}});
+ assert.throws(()=>executorVolumes({},worker),/Missing executor volume/);
+});
+test('executor archive extracts its exact private binding without host ownership changes',async t=>{
+ const root=await fs.mkdtemp(path.join(tmpdir(),'ez-executor-binding-'));t.after(()=>fs.rm(root,{recursive:true,force:true}));
+ const binding={agents:[{workspace:'/private/work',binDir:'/app/bin'}]},file=path.join(root,'binding.tar');
+ await fs.writeFile(file,executorBindingArchive(binding,1000),{mode:0o600});
+ await exec('tar',['-xf',file,'-C',root]);
+ const extracted=path.join(root,'executor.json');assert.deepEqual(await read(extracted),binding);assert.equal((await fs.stat(extracted)).mode&0o777,0o440);
+ for(const invalid of [-1,NaN,1.5,'1000',0o10000000])assert.throws(()=>executorBindingArchive(binding,invalid),/Invalid executor group/);
+});

package/test/upgrade-pause.test.ts

@@ -3,13 +3,76 @@ import test from 'node:test'
 import { access, chmod, mkdir, mkdtemp, rm, stat, symlink, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import { spawnSync } from 'node:child_process'
+import { spawn, spawnSync } from 'node:child_process'
 import { fileURLToPath } from 'node:url'
 import { createRelay } from '../src/index.js'
 import { ControlStore } from '../src/control-state.js'
 import { RunStore } from '../src/runs.js'
 import { initialPreset } from '../src/ai.js'
 import { opencodeDataHome, piAgentDir } from '../src/executor.js'
+// @ts-expect-error The supervisor is a standalone Node module.
+import { idle } from '../src/updates/supervisor.mjs'
+import { initializeWorkspace } from '../src/workspace.js'
+
+test('reload idle status waits for an admission already past the pause check', { timeout: 15000 }, async () => {
+  const dir = await mkdtemp(join(tmpdir(), 'ez-upgrade-barrier-'))
+  const purpose = join(dir, 'purpose.md')
+  await writeFile(purpose, 'Fixture agent\n')
+  await initializeWorkspace(dir, purpose)
+  const control = new ControlStore(dir, 1000), runs = new RunStore(dir)
+  await control.requestPairing(101, 101)
+  await control.approveOwner(101)
+  let launched = 0
+  const relay = createRelay({ controlDir: dir, workspace: dir, pairingTtlMs: 1000,
+    executorTimeoutMs: 0, executorCli: 'grok', telegramBotToken: 'fixture' }, async () => {
+    launched++
+    return { child: spawn(process.execPath, ['-e', 'setTimeout(()=>{},10000)'], { stdio: 'ignore' }), cleanup: async () => {}, stdout: '' }
+  })
+  relay.bot.botInfo = { id: 999, is_bot: true, first_name: 'Fixture', username: 'fixture_bot' } as typeof relay.bot.botInfo
+  let polling!: () => void
+  const ready = new Promise<void>(resolve => { polling = resolve })
+  relay.bot.api.config.use(async (_prev, method, _payload, signal) => {
+    if (method === 'getUpdates') {
+      polling()
+      if (signal && !signal.aborted) await new Promise<void>(resolve => signal.addEventListener('abort', () => resolve(), { once: true }))
+      return { ok: true, result: [] } as never
+    }
+    return { ok: true, result: { message_id: 42 } } as never
+  })
+  const started = relay.start()
+  const originalStatus = ControlStore.prototype.status
+  let release!: () => void, blocked!: () => void
+  const gate = new Promise<void>(resolve => { release = resolve })
+  const entering = new Promise<void>(resolve => { blocked = resolve })
+  try {
+    await ready
+    const run = await runs.create({ chatId: 101, telegramUserId: 101, texts: ['fixture'],
+      execution: await control.captureChoice({ ...initialPreset('grok'), model: 'fixture-model' }) })
+    let reads = 0
+    ControlStore.prototype.status = async function () {
+      if (++reads === 2) { blocked(); await gate }
+      return originalStatus.call(this)
+    }
+    const admission = relay.drainSources()
+    await entering
+    await writeFile(join(dir, 'upgrade-pause.json'), '{}', { mode: 0o600 })
+    let settled = false
+    const snapshot = idle(dir, { pendingRequests: false, relayRequired: true }).then((value: boolean) => { settled = true; return value })
+    await new Promise(resolve => setTimeout(resolve, 50))
+    assert.equal(settled, false, 'status must wait behind the held admission')
+    release()
+    await admission
+    assert.equal(await snapshot, false)
+    assert.equal(launched, 1)
+    assert.equal((await runs.get(run.id))?.status, 'running')
+  } finally {
+    release()
+    ControlStore.prototype.status = originalStatus
+    await relay.stop()
+    await started
+    await rm(dir, { recursive: true, force: true })
+  }
+})
 
 test('private control directory preserves pause, fails closed on errors, and resumes queued work', async () => {
   const dir = await mkdtemp(join(tmpdir(), 'ez-upgrade-pause-'))
@@ -74,5 +137,5 @@ test('Linux relay real/effective UID regression (Docker test target)', {
     '--test', fileURLToPath(import.meta.url),
   ], { encoding: 'utf8', timeout: 15000, env })
   assert.equal(result.status, 0, result.stdout + result.stderr)
-  assert.match(result.stdout, /ok 1 - private control directory/)
+  assert.match(result.stdout, /ok \d+ - private control directory/)
 })