@jc_stack/ez-agents 0.1.0-beta.18 → 0.1.0-beta.21

package/test/failure.test.ts

@@ -17,7 +17,10 @@ import { TelegramSource } from '../src/telegram-source.js'
 import { packageVersion } from '../src/version.js'
 import type { Update } from 'grammy/types'
 const exec=promisify(execFile),bin=fileURLToPath(new URL('../bin/ezenciel-agents-schedule.mjs',import.meta.url))
-const until=async(check:()=>Promise<boolean>)=>{for(let n=0;n<200;n++){if(await check())return;await new Promise(r=>setTimeout(r,20))}throw new Error('Timed out')}
+// This integration test starts subprocesses, persists their receipts, and invokes
+// the schedule CLI. Give a busy CI worker time to settle without changing the
+// production recovery deadline.
+const until=async(check:()=>Promise<boolean>,timeoutMs=15_000)=>{const deadline=Date.now()+timeoutMs;while(!(await check())){if(Date.now()>=deadline)throw new Error(`Timed out after ${timeoutMs}ms`);await new Promise(r=>setTimeout(r,20))}}
 
 test('failure evidence is bounded and redacts configured credentials, headers, tokens, URLs and keys',()=>{
  const token='123456789:abcdefghijklmnopqrstuvwxyz123456789'
@@ -137,7 +140,30 @@ test('relay shutdown waits for executor cleanup and final run state', async () =
  } finally {release();await relay.stop();await rm(dir,{recursive:true,force:true})}
 })
 
-for (const cleanupFails of [false,true]) test(`fatal polling conflict waits for shared shutdown without retrying (cleanup fails: ${cleanupFails})`, async t => {
+test('a fresh relay initializes its bot identity before polling', async () => {
+ const dir=await mkdtemp(join(tmpdir(),'ez-cold-start-'))
+ const relay=createRelay({workspace:dir,controlDir:dir,pairingTtlMs:1000,executorTimeoutMs:0,executorCli:'grok',telegramBotToken:'fixture'},async()=>{throw new Error('No executor expected')})
+ const methods:string[]=[]
+ relay.bot.api.config.use(async(_prev,method,_payload,signal)=>{
+  methods.push(method)
+  if(method==='getMe') return {ok:true,result:{id:999,is_bot:true,first_name:'Fixture',username:'fixture_bot'}} as any
+  if(method==='getUpdates') {
+   if(signal && !signal.aborted) await new Promise<void>(resolve=>signal.addEventListener('abort',()=>resolve(),{once:true}))
+   return {ok:true,result:[]} as any
+  }
+  return {ok:true,result:true} as any
+ })
+ const started=relay.start()
+ try {
+  await until(async()=>methods.includes('getUpdates'))
+  assert.equal(relay.bot.isRunning(),true)
+  assert.equal(methods.filter(method=>method==='getMe').length,1)
+  assert.ok(methods.indexOf('getMe')<methods.indexOf('getUpdates'))
+ } finally {await relay.stop();await started;await rm(dir,{recursive:true,force:true})}
+ assert.equal(relay.bot.isRunning(),false)
+})
+
+for (const cleanupFails of [false,true]) test(`polling conflict preserves work until an explicit shutdown (cleanup fails: ${cleanupFails})`, async t => {
  const dir=await mkdtemp(join(tmpdir(),'ez-polling-conflict-')),control=new ControlStore(dir,1000),runs=new RunStore(dir)
  let release!:()=>void,entered!:()=>void,releaseDelivery!:()=>void,sending!:()=>void,child:ReturnType<typeof spawn>|undefined,polls=0
  const gate=new Promise<void>(resolve=>{release=resolve}),cleaning=new Promise<void>(resolve=>{entered=resolve})
@@ -170,20 +196,26 @@ for (const cleanupFails of [false,true]) test(`fatal polling conflict waits for
   const delivery=relay.drainOutbox()
   await deliveryStarted
   let finished=false
-  const start=assert.rejects(relay.start(),/409.*Conflict/).then(()=>{finished=true})
-  await cleaning
+  const start=relay.start().finally(()=>{finished=true})
+  await until(async()=>polls===1)
+  assert.equal(sourceStops,0,'a polling conflict must not stop the relay')
+  assert.equal((await runs.get('tg_92'))?.status,'running')
+  assert.ok(child && child.exitCode===null && child.signalCode===null)
   const stopping=relay.stop()
   assert.equal(relay.stop(),stopping,'concurrent stop calls share one promise')
   const stopped=cleanupFails?assert.rejects(stopping,/Synthetic shutdown failure/):stopping
-  assert.equal(finished,false,'polling failure must wait for executor cleanup')
+  assert.equal(finished,false,'relay start must wait for the explicit shutdown')
+  await cleaning
   release()
   await until(async()=>(await runs.get('tg_92'))?.status!=='running')
-  assert.equal(finished,false,'polling failure must wait for the in-flight delivery receipt')
-  releaseDelivery();await delivery;await start;await stopped
+  assert.equal(finished,false,'shutdown must wait for the in-flight delivery receipt')
+  releaseDelivery();await delivery
+  if(cleanupFails) await assert.rejects(start,/Synthetic shutdown failure/);else await start
+  await stopped
   assert.equal(relay.stop(),stopping,'finished shutdown remains idempotent')
   assert.equal(sourceStops,1)
   assert.deepEqual(JSON.parse(await readFile(join(dir,'outbox',`${item.id}.sent.json`),'utf8')).receipt.messageIds,[1])
-  assert.equal(polls,1,'a conflict must not start another polling loop')
+  assert.equal(polls,1,'a conflict must not start another polling loop before the retry delay')
   assert.equal(relay.bot.isRunning(),false)
   assert.ok(child && (child.exitCode!==null || child.signalCode!==null))
   assert.notEqual((await runs.get('tg_92'))?.status,'running')

package/test/host-executor.test.ts

@@ -160,6 +160,22 @@ test('client tolerates missing heartbeat and consumes completion before checking
  } finally {client.kill();await closed;await rm(root,{recursive:true,force:true})}
 })
 
+test('client records a relay interruption before exiting 130',async()=>{
+ const root=await mkdtemp(path.join(tmpdir(),'ez-host-interrupt-'))
+ const directory=path.join(root,'host-executor');await mkdir(directory)
+ const client=spawn(process.execPath,['--import',fileURLToPath(new URL('../node_modules/tsx/dist/loader.mjs',import.meta.url)),fileURLToPath(new URL('../src/host-executor-client.ts',import.meta.url)),root,'tg_97'],{stdio:['pipe','pipe','pipe']})
+ let stderr='';client.stderr.on('data',chunk=>stderr+=chunk);client.stdout.resume()
+ client.stdin.end(JSON.stringify({texts:['test'],options:{}}))
+ try {
+  for(let n=0;n<100;n++){try{await readFile(path.join(directory,'tg_97.request.json'));break}catch{await new Promise(r=>setTimeout(r,20))}}
+  const closed=new Promise<number|null>(resolve=>client.once('close',resolve))
+  client.kill('SIGTERM')
+  assert.equal(await closed,130)
+  assert.equal(await readFile(path.join(directory,'tg_97.cancel'),'utf8'),'')
+  assert.match(stderr,/Host executor client interrupted by SIGTERM/)
+ } finally {if(client.exitCode===null && client.signalCode===null)client.kill();await rm(root,{recursive:true,force:true})}
+})
+
 test('client cancels on stale or invalid heartbeat instead of waiting indefinitely',async()=>{
  for(const heartbeat of [{at:Date.now()-60000},{at:'invalid'}]) {
   const root=await mkdtemp(path.join(tmpdir(),'ez-heartbeat-invalid-'))

package/test/intake-relay.test.ts

@@ -201,7 +201,7 @@ test('owner group discovery routes only to the private chat and rechecks identit
   } finally { await f.close() }
 })
 
-test('four-item menu is owner-only; saved AI buttons work and forged/stale buttons cannot change settings', async () => {
+test('four-item menu is owner-only; available AI choices work and forged/stale buttons cannot change settings', async () => {
   const f = await fixture()
   const callback = (id: number, data: string, user = 101): Update => ({
     update_id: id,
@@ -220,11 +220,23 @@ test('four-item menu is owner-only; saved AI buttons work and forged/stale butto
     await f.relay.bot.handleUpdate(callback(4, 'ai:forged'))
     assert.equal(await store.getActiveSession(), null)
     await f.relay.bot.handleUpdate(callback(5, pick))
+    if (!(await store.getActiveSession())) {
+      let useNow = f.keyboards.at(-1)!.flat().find((button) => button.text === 'Use now')?.callback_data
+      if (!useNow) {
+        await f.relay.bot.handleUpdate(callback(6, f.keyboards.at(-1)!.flat()[0].callback_data))
+        useNow = f.keyboards.at(-1)!.flat().find((button) => button.text === 'Use now')!.callback_data
+      }
+      await f.relay.bot.handleUpdate(callback(7, useNow))
+    }
     assert.ok(await store.getActiveSession())
-    await f.relay.bot.handleUpdate(callback(6, pick))
+    await f.relay.bot.handleUpdate(callback(8, pick))
     assert.match(f.replies.at(-1)!, /Menu expired/)
-    await f.relay.bot.handleUpdate(message(7, '/settings'))
+    await f.relay.bot.handleUpdate(message(9, '/settings'))
     assert.match(f.replies.at(-1)!, /Default for new conversations/)
+    await f.relay.bot.handleUpdate(message(8, '/status'))
+    assert.ok(f.keyboards.at(-1)!.flat().some((button) => button.text === 'Scheduled tasks'))
+    await f.relay.bot.handleUpdate(callback(9, 'menu:scheduled-tasks'))
+    assert.match(f.replies.at(-1)!, /No scheduled tasks for this owner/)
     assert.equal(f.launched.length, 0)
   } finally { await f.close() }
 })

package/test/model-policy.test.ts

@@ -11,7 +11,7 @@ import { taskArguments } from '../src/task-executor.js'
 import { runCodexSession } from '../src/codex-session.js'
 import { runDesktopTurn } from '../src/desktop-bridge.js'
 
-test('all model selections and launches reject effort above high before spawning', async () => {
+test('all non-Luna model selections and launches reject effort above high before spawning', async () => {
   for (const cli of ['codex', 'codex-gui', 'grok', 'claude', 'opencode', 'agy']) {
     for (const effort of ['xhigh', 'max', 'ultra', 'unknown']) {
       const preset = { id:'blocked', name:'Blocked', cli, model:'any-model', effort }
@@ -23,6 +23,14 @@ test('all model selections and launches reject effort above high before spawning
   await assert.rejects(runDesktopTurn({workspace:'/unused',controlDir:'/unused',binDir:'/unused',runId:'unused',prompt:'',effort:'ultra'}), /capped at high/)
 })
 
+test('Codex Luna accepts xhigh while every other model and CLI remains capped', async () => {
+  const luna = { id:'luna', name:'Luna', cli:'codex', model:'gpt-5.6-luna', effort:'xhigh' }
+  await validateSelection(luna, [{ cli:'codex', model:'gpt-5.6-luna', name:'Luna', efforts:['high','xhigh'] }], async () => true)
+  assert.deepEqual(executionDefaults('codex', { model:'gpt-5.6-luna', effort:'xhigh' }), { model:'gpt-5.6-luna', effort:'xhigh' })
+  assert.throws(() => executionDefaults('grok', { model:'gpt-5.6-luna', effort:'xhigh' }), /capped at high/)
+  assert.throws(() => executionDefaults('codex', { model:'gpt-5.6-terra', effort:'xhigh' }), /capped at high/)
+})
+
 test('restricted tasks pin Terra high, preserve explicit choices and reject higher effort', () => {
   const args = taskArguments('/unused', ['broker'], 'prompt')
   assert.equal(args[args.indexOf('--model')+1], 'gpt-5.6-terra')

package/test/plugin-manager.test.mjs

@@ -225,3 +225,52 @@ test('standalone rejects relay binding and preserves literal plugin arguments ac
  await fs.writeFile(path.join(f.home,'config.json'),JSON.stringify({schemaVersion:1,workspace:f.workspace,catalog:{},deploymentDir:'/missing'}));
  await assert.rejects(exec(launcher,['status']),/deployment-bound/); // never hide a broken relay binding
 });
+
+test('existing folders are read-only, persistent and fail closed when missing', async t => {
+ const f=await fixture(t);await init(f.home,f.workspace);
+ const inspected=JSON.parse((await f.call('plugins','inspect','sample','--source',f.source)).stdout);
+ await f.call('plugins','install','sample','--source',f.source,'--revision',inspected.revision);
+ const source=await fs.realpath(f.workspace);
+ await f.call('plugins','folder-bind','sample','--service','sample','--source',source,'--target','/data/files');
+ const config=JSON.parse(await fs.readFile(path.join(f.home,'config.json'),'utf8'));
+ assert.deepEqual(config.folders.sample,[{service:'sample',source,target:'/data/files'}]);
+ const c=JSON.parse(await fs.readFile(path.join(f.home,'packages/sample/compose.json'),'utf8'));
+ assert.deepEqual(c.services.sample.volumes.find(v=>v.target==='/data/files'),{type:'bind',source,target:'/data/files',read_only:true,bind:{create_host_path:false}});
+ await assert.rejects(f.call('plugins','folder-bind','sample','--service','sample','--source',source,'--target','/data'),/child of a declared volume|Overlapping/);
+ await assert.rejects(f.call('plugins','folder-bind','sample','--service','sample','--source',source,'--target','/data/files/child'),/Overlapping/);
+ await assert.rejects(f.call('plugins','folder-bind','sample','--service','sample','--source',await fs.realpath(f.home),'--target','/data/private'),/private plugin state/);
+ await assert.rejects(f.call('plugins','folder-bind','sample','--service','sample','--source','/','--target','/data/root'),/private plugin state/);
+ await f.call('plugins','start','sample');
+ await fs.rename(f.workspace,f.workspace+'-moved');
+ await assert.rejects(f.call('plugins','start','sample'),/ENOENT/);
+ await assert.rejects(f.call('sample','read'),/ENOENT/);
+ await f.call('plugins','folder-unbind','sample','--service','sample','--target','/data/files');
+ assert.deepEqual(JSON.parse((await f.call('plugins','folders','sample')).stdout),[]);
+});
+
+test('folder bindings survive compatible descriptors and reject incompatible updates',async t=>{
+ const f=await fixture(t);const p=await snapshot(f.source);
+ const record={...p,project:'ezp-test-sample'};
+ const config={workspace:f.workspace,folders:{sample:[{service:'sample',source:f.workspace,target:'/data/files'}]}};
+ assert.equal(compose(config,record).services.sample.volumes.find(v=>v.target==='/data/files').read_only,true);
+ const next=structuredClone(record);next.deployment.services.sample.volumes={data:'/new-state'};
+ assert.throws(()=>compose(config,next),/child of a declared volume/);
+});
+
+test('folder rebind rejects every live project container including one-shots',async t=>{
+ const f=await fixture(t);await init(f.home,f.workspace);const p=await snapshot(f.source);
+ await f.call('plugins','install','sample','--source',f.source,'--revision',p.revision);
+ await fs.writeFile(path.join(f.fake,'docker'),`#!${process.execPath}\nif(process.argv[2]==='ps')console.log('paused-or-restarting-or-one-shot');\n`,{mode:0o700});
+ await assert.rejects(f.call('plugins','folder-bind','sample','--service','sample','--source',await fs.realpath(f.workspace),'--target','/data/files'),/Stop the plugin/);
+});
+
+test('registered calls honor registry lock and regenerate stale Compose from current folders',async t=>{
+ const f=await fixture(t);await init(f.home,f.workspace);const p=await snapshot(f.source);
+ await f.call('plugins','install','sample','--source',f.source,'--revision',p.revision);
+ await f.call('plugins','folder-bind','sample','--service','sample','--source',await fs.realpath(f.workspace),'--target','/data/files');
+ const file=path.join(f.home,'packages/sample/compose.json');await fs.writeFile(file,'{}');
+ await f.call('sample','read');
+ assert.equal(JSON.parse(await fs.readFile(file,'utf8')).services.sample.volumes.find(v=>v.target==='/data/files').read_only,true);
+ await fs.writeFile(path.join(f.home,'registry.lock'),'test');
+ await assert.rejects(f.call('sample','read'),/busy|EEXIST|locked/i);
+});

package/test/reply.test.ts

@@ -129,3 +129,25 @@ test('a parallel reply delivered during a normal turn is retained for the follow
   assert.deepEqual(await parallelReplyHistory(root,current),[])
  }finally{await rm(root,{recursive:true,force:true})}
 })
+
+
+test('reply handoff accepts independent worker choices and rejects invalid or unauthorized overrides', async () => {
+ const root=await mkdtemp(join(tmpdir(),'ez-reply-worker-')), runs=new RunStore(root)
+ try {
+  await ownerRun(root,'owner')
+  await runs.create({id:'tg_10',chatId:101,telegramUserId:101,texts:['Analyze the report'],execution:{sessionId:'c5dd1edc-be24-47b8-a579-0bc70f44cf43',preset:{id:'chat',name:'Chat',cli:'codex',model:'gpt-5.6-sol',effort:'medium'}}})
+  await runs.patch('tg_10',{status:'running',replyOnly:true})
+  for (const args of [{model:42}, {model:'bad model'}, {effort:'ultra'}, {effort:'invalid'}, {cli:'claude'}])
+   await assert.rejects(replyCall(root,'tg_10',root,'defer',{text:'Analyze and verify the result',...args}))
+  await assert.rejects(replyCall(root,'tg_10',root,'send',{text:'Hello',model:'gpt-6-astra'}),/Unexpected/)
+  await replyCall(root,'tg_10',root,'defer',{text:'Analyze and verify the result',model:'gpt-6-astra',effort:'high'})
+  const file=join(root,'schedules','s_reply_tg_10.json')
+  const saved=JSON.parse(await readFile(file,'utf8'))
+  assert.equal(saved.execution.preset.model,'gpt-6-astra')
+  assert.equal(saved.execution.preset.effort,'high')
+  await replyCall(root,'tg_10',root,'defer',{text:'retry',model:'gpt-5.6-sol',effort:'low'})
+  assert.deepEqual(JSON.parse(await readFile(file,'utf8')),saved)
+  await new ControlStore(root,900000).revokeOwner()
+  await assert.rejects(replyCall(root,'tg_10',root,'defer',{text:'after revocation',model:'gpt-6-astra'}),/owner-mismatch/)
+ } finally { await rm(root,{recursive:true,force:true}) }
+})

package/test/runs.test.ts

@@ -22,6 +22,13 @@ test('creates a queued run and binds chat id outside the workspace', async () =>
   assert.equal((await store.get(created.id))?.chatId, 101)
 }))
 
+test('running does not reap a process from another executor namespace', async () => fixture(async (store) => {
+  const created = await store.create({ chatId: 101, telegramUserId: 101, texts: ['host-backed'] })
+  await store.patch(created.id, { status: 'running', pid: 999_999_999 })
+  assert.equal((await store.running())?.id, created.id)
+  assert.equal((await store.get(created.id))?.status, 'running')
+}))
+
 test('ez message writes an outbox item for the bound run, not a telegram send', async () => fixture(async (store) => {
   const created = await store.create({ chatId: 9, telegramUserId: 9, texts: ['hello'] })
   await store.patch(created.id, { status: 'running' })

package/test/schedule-cli.test.ts

@@ -24,6 +24,8 @@ test('public scheduler CLI saves literal text, reads back, edits, pauses, and re
  assert.equal(saved.text,args.at(-1));assert.equal(saved.execution.preset.cli,'codex')
  assert.equal(saved.execution.preset.model,'gpt-5.6-terra');assert.equal(saved.execution.preset.effort,'high')
  await assert.rejects(exec(process.execPath,[bin,'create','blocked','--at','2027-09-09T09:00:00+04:00','--text','test','--effort','xhigh'],{env}),/capped at high/)
+ const luna=JSON.parse((await exec(process.execPath,[bin,'create','luna','--at','2027-09-10T09:00:00+04:00','--text','Luna xhigh task','--model','gpt-5.6-luna','--effort','xhigh'],{env})).stdout)
+ assert.equal(luna.execution.preset.model,'gpt-5.6-luna');assert.equal(luna.execution.preset.effort,'xhigh')
  assert.equal(saved.nextEligibleAt,'2027-09-09T05:00:00.000Z')
  await assert.rejects(exec(process.execPath,[bin,...args],{env}),/exists/)
  assert.equal(JSON.parse((await exec(process.execPath,[bin,'pause','test'],{env})).stdout).enabled,false)

package/test/scheduled-tasks.test.ts

@@ -0,0 +1,43 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { access, mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
+import { randomUUID } from 'node:crypto'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { Scheduler } from '../src/scheduler.js'
+import { scheduledTasksText } from '../src/scheduled-tasks.js'
+
+const owner = { telegramUserId: 101, telegramChatId: 101, pairedAt: '2026-09-11T00:00:00.000Z' }
+const execution = { sessionId: randomUUID(), preset: { id: 'fixture', name: 'Fixture', cli: 'codex' } }
+
+test('scheduled task view is read-only, owner-bound, and shows stored task contents', async (t) => {
+  const dir = await mkdtemp(join(tmpdir(), 'ez-scheduled-tasks-'))
+  t.after(() => rm(dir, { recursive: true, force: true }))
+  const scheduler = new Scheduler(dir)
+
+  assert.deepEqual(await scheduler.listReadOnly(), [])
+  await assert.rejects(access(join(dir, 'schedules')), /ENOENT/)
+
+  await scheduler.save({
+    id: 'owner-task', name: 'Daily report', text: 'Read the ledger and send the owner a concise report.', owner, execution, enabled: true,
+    trigger: { cron: '0 9 * * 1-5', timezone: 'Asia/Dubai', start: '2026-01-01T00:00:00.000Z' },
+  })
+  await scheduler.save({
+    id: 'other-task', name: 'Other owner task', text: 'This must never be visible.',
+    owner: { ...owner, telegramUserId: 202, telegramChatId: 202 }, execution, enabled: true,
+    trigger: { at: '2027-01-01T00:00:00.000Z' },
+  })
+  const scheduleDir = join(dir, 'schedules')
+  const before = await readFile(join(scheduleDir, 'owner-task.json'), 'utf8')
+  const entries = await readdir(scheduleDir)
+  const text = scheduledTasksText(await scheduler.listReadOnly(), owner, Date.parse('2026-09-11T00:00:00.000Z'))
+
+  assert.match(text, /Title: Daily report/)
+  assert.match(text, /Instructions:\nRead the ledger and send the owner a concise report\./)
+  assert.match(text, /Timing: Cron 0 9 \* \* 1-5 · Asia\/Dubai/)
+  assert.match(text, /State: Scheduled/)
+  assert.match(text, /Next run: 2026-09-11T05:00:00.000Z/)
+  assert.doesNotMatch(text, /Other owner task|This must never be visible/)
+  assert.equal(await readFile(join(scheduleDir, 'owner-task.json'), 'utf8'), before)
+  assert.deepEqual(await readdir(scheduleDir), entries)
+})

package/test/updates.test.mjs

@@ -86,6 +86,16 @@ test('status distinguishes installed, running, legacy and stale main versions wi
  const result=await exec(process.execPath,[new URL('../bin/ezenciel-agents-tools.mjs',import.meta.url).pathname,'--home',f.home,'status']);
  assert.equal(JSON.parse(result.stdout).main.installedVersion,'0.1.0');
 });
+test('relay healthcheck emits bounded predicate evidence without state contents',async t=>{
+ const root=await fs.realpath(await fs.mkdtemp(path.join(tmpdir(),'ez-healthcheck-')));t.after(()=>fs.rm(root,{recursive:true,force:true}));
+ const relay=path.join(root,'relay'),host=path.join(root,'host');await fs.mkdir(relay,{recursive:true});await fs.mkdir(path.join(host,'host-executor'),{recursive:true});
+ const check=async()=>exec(process.execPath,[new URL('../docker/healthcheck.mjs',import.meta.url).pathname],{env:{...process.env,EZ_HEALTH_RELAY_CONTROL_DIR:relay,EZ_EXECUTOR_TRANSPORT:'host',EZ_CONTROL_DIR:host}});
+ await fs.writeFile(path.join(relay,'heartbeat.json'),JSON.stringify({polling:true,at:Date.now()}));await fs.writeFile(path.join(host,'host-executor/heartbeat.json'),JSON.stringify({at:Date.now()}));await check();
+ await fs.writeFile(path.join(relay,'heartbeat.json'),JSON.stringify({polling:false,at:Date.now(),private:'must-not-appear'}));
+ await assert.rejects(check(),error=>{assert.match(error.stderr,/EZ_HEALTH_RELAY_NOT_POLLING/);assert.doesNotMatch(error.stderr,/must-not-appear/);return true;});
+ await fs.writeFile(path.join(relay,'heartbeat.json'),JSON.stringify({polling:true,at:Date.now()}));await fs.writeFile(path.join(host,'host-executor/heartbeat.json'),JSON.stringify({at:Date.now()-20000}));
+ await assert.rejects(check(),error=>{assert.match(error.stderr,/EZ_HEALTH_HOST_STALE/);return true;});
+});
 test('plugin status verifies images and reports stopped, mismatched and unreachable runtimes honestly',async t=>{
  const f=await fixture(t,'plugin'),id='a'.repeat(64),image='sha256:'+'b'.repeat(64);
  for(const mode of ['running','ndjson','stopped','missing','mismatched','offline']) {
@@ -157,6 +167,11 @@ test('failed preparation never stops runtime; failed activation rolls back code
   else assert((await fs.readFile(path.join(f.config.deploymentDir,'docker.env'),'utf8')).includes('sha256:'+'a'.repeat(64)));
  }
 });
+test('main health-gate rollback retains only the recognized sanitized predicate',async t=>{
+ const f=await fixture(t),job=await queued(f),base=runtime(f);let failed=false;
+ const r={...base,execute:async(c,a,o)=>{if(a.includes('up')&&!failed){failed=true;throw Error('container relay is unhealthy');}return a.includes('ps')?'a'.repeat(64):a[0]==='inspect'&&a[2]==='{{json .State.Health}}'?JSON.stringify({Log:[{Output:'EZ_HEALTH_HOST_STALE\n'}]}):base.execute(c,a,o);}};
+ const result=await perform(f.home,job,r);assert.equal(result.status,'rolled-back');assert.match(result.error,/health=host-stale/);assert.doesNotMatch(result.error,/private-test-token/);
+});
 test('plugin transaction preserves named volumes, backs up stopped data and rolls back failed health',async t=>{
  for(const failed of [false,true]) {
   const f=await fixture(t,'plugin'),job=await queued(f),r=runtime(f,{fail:(_c,a)=>failed&&a.includes('up')});

package/test/workforce-watch.test.ts

@@ -0,0 +1,180 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { WorkforceWatch, WorkforceWatchServer } from '../src/workforce-watch.js'
+
+const fixture = async () => {
+  const stateDir = await mkdtemp(join(tmpdir(), 'workforce-watch-'))
+  let now = 0
+  const notices: string[] = []
+  const watch = new WorkforceWatch({ stateDir, enrollmentToken: 'fleet-secret', now: () => now, notify: async text => { notices.push(text) } })
+  return { stateDir, watch, notices, advance: (ms: number) => { now += ms }, close: () => rm(stateDir, { recursive: true, force: true }) }
+}
+
+test('worker enrollment, terminal alert context, and sustained recovery preserve only redacted state', async () => {
+  const f = await fixture()
+  try {
+    await assert.rejects(() => f.watch.enroll('wrong', { workerId: 'stocks-production', checkInSeconds: 10, graceSeconds: 0 }), /Unauthorized/)
+    const enrolled = await f.watch.enroll('fleet-secret', { workerId: 'stocks-production', checkInSeconds: 10, graceSeconds: 0, runbookUrl: 'https://runbooks.example/stocks' })
+    await assert.rejects(() => f.watch.checkIn('stocks-production', 'wrong', { status: 'ok' }), /Unauthorized/)
+    await f.watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'failed', terminal: true, activity: 'IBKR session probe', error: 'No broker server time', runId: 'daily-42', logsHint: 'journalctl --user -u ez-stocks.service --since 30m' })
+    assert.match(f.notices[0], /IBKR session probe/)
+    assert.match(f.notices[0], /No broker server time/)
+    const state = await f.watch.inspect('stocks-production') as Record<string, unknown>
+    assert.equal('tokenHash' in state, false)
+    assert.deepEqual((state.history as Array<Record<string, unknown>>).at(-1)?.runId, 'daily-42')
+    await f.watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok', activity: 'retry one' })
+    await f.watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok', activity: 'retry two' })
+    assert.equal(f.notices.length, 2)
+    assert.match(f.notices[1], /recovered/)
+  } finally { await f.close() }
+})
+
+test('a missed check-in alerts once and requires clean check-ins to recover', async () => {
+  const f = await fixture()
+  try {
+    const enrolled = await f.watch.enroll('fleet-secret', { workerId: 'annie-pa', checkInSeconds: 10, graceSeconds: 2, severity: 'warning' })
+    f.advance(12_001)
+    await f.watch.evaluate(); await f.watch.evaluate()
+    assert.equal(f.notices.length, 1)
+    assert.match(f.notices[0], /missed check-in/)
+    await f.watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok', activity: 'relay available' })
+    assert.equal(f.notices.length, 1)
+    await f.watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok', activity: 'relay available' })
+    assert.equal(f.notices.length, 2)
+  } finally { await f.close() }
+})
+
+test('PagerDuty uses one worker-specific deduplication key and resolves only a triggered incident', async () => {
+  const stateDir = await mkdtemp(join(tmpdir(), 'workforce-watch-'))
+  const pages: Array<{ action: string; dedupKey: string; customDetails: Record<string, string> }> = []
+  const watch = new WorkforceWatch({ stateDir, enrollmentToken: 'fleet-secret', page: async event => { pages.push(event) } })
+  try {
+    const enrolled = await watch.enroll('fleet-secret', { workerId: 'stocks', checkInSeconds: 10, graceSeconds: 0, severity: 'critical' })
+    await watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'failed', terminal: true, activity: 'broker probe', error: 'gateway unavailable', logsHint: 'journalctl -u stocks' })
+    await watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok' })
+    await watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok' })
+    assert.deepEqual(pages.map(page => [page.action, page.dedupKey]), [['trigger', 'ez:workforce:stocks'], ['resolve', 'ez:workforce:stocks']])
+    assert.equal(pages[0]?.customDetails.error, 'gateway unavailable')
+  } finally { await rm(stateDir, { recursive: true, force: true }) }
+})
+
+test('PagerDuty trigger is not duplicated when supplemental Telegram delivery fails', async () => {
+  const stateDir = await mkdtemp(join(tmpdir(), 'workforce-watch-'))
+  const pages: string[] = []
+  const watch = new WorkforceWatch({ stateDir, enrollmentToken: 'fleet-secret', page: async event => { pages.push(event.action) }, notify: async () => { throw new Error('Telegram unavailable') } })
+  try {
+    const enrolled = await watch.enroll('fleet-secret', { workerId: 'aifit', checkInSeconds: 10, graceSeconds: 0 })
+    await assert.rejects(() => watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'failed', terminal: true }), /Telegram unavailable/)
+    await assert.rejects(() => watch.evaluate(), /Telegram unavailable/)
+    assert.deepEqual(pages, ['trigger'])
+  } finally { await rm(stateDir, { recursive: true, force: true }) }
+})
+
+test('a failed recovery notification stays pending and is retried on evaluation', async () => {
+  const stateDir = await mkdtemp(join(tmpdir(), 'workforce-watch-'))
+  let now = 0, failRecovery = true
+  const notices: string[] = []
+  const watch = new WorkforceWatch({ stateDir, enrollmentToken: 'fleet-secret', now: () => now, notify: async text => {
+    if (text.includes('recovered') && failRecovery) throw new Error('Telegram unavailable')
+    notices.push(text)
+  } })
+  try {
+    const enrolled = await watch.enroll('fleet-secret', { workerId: 'dusk-dune', checkInSeconds: 10, graceSeconds: 0 })
+    await watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'failed', terminal: true })
+    await watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok' })
+    await assert.rejects(() => watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok' }), /Telegram unavailable/)
+    assert.ok((await watch.inspect('dusk-dune') as { incident?: unknown }).incident)
+    failRecovery = false; now += 1
+    await watch.evaluate()
+    assert.equal((await watch.inspect('dusk-dune') as { incident?: unknown }).incident, undefined)
+    assert.equal(notices.length, 2)
+  } finally { await rm(stateDir, { recursive: true, force: true }) }
+})
+
+test('an incident whose opening alert never delivered resolves quietly', async () => {
+  const stateDir = await mkdtemp(join(tmpdir(), 'workforce-watch-'))
+  const notices: string[] = []
+  const watch = new WorkforceWatch({ stateDir, enrollmentToken: 'fleet-secret', notify: async text => {
+    if (text.includes('alert')) throw new Error('Telegram unavailable')
+    notices.push(text)
+  } })
+  try {
+    const enrolled = await watch.enroll('fleet-secret', { workerId: 'jc-stack', checkInSeconds: 10, graceSeconds: 0 })
+    await assert.rejects(() => watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'failed', terminal: true }), /Telegram unavailable/)
+    await assert.rejects(() => watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok' }), /Telegram unavailable/)
+    await watch.checkIn(enrolled.workerId, enrolled.workerToken, { status: 'ok' })
+    assert.deepEqual(notices, [])
+    assert.equal((await watch.inspect('jc-stack') as { incident?: unknown }).incident, undefined)
+  } finally { await rm(stateDir, { recursive: true, force: true }) }
+})
+
+test('an enrollment-authorized token rotation invalidates the prior worker secret', async () => {
+  const f = await fixture()
+  try {
+    const enrolled = await f.watch.enroll('fleet-secret', { workerId: 'aifit', checkInSeconds: 10, graceSeconds: 0 })
+    await assert.rejects(() => f.watch.rotate('wrong', 'aifit'), /Unauthorized/)
+    const rotated = await f.watch.rotate('fleet-secret', 'aifit')
+    await assert.rejects(() => f.watch.checkIn('aifit', enrolled.workerToken, { status: 'ok' }), /Unauthorized/)
+    await f.watch.checkIn('aifit', rotated.workerToken, { status: 'ok' })
+  } finally { await f.close() }
+})
+
+test('HTTP enrollment and check-in endpoints reject secrets not owned by the caller', async () => {
+  const f = await fixture(), server = new WorkforceWatchServer(f.watch, 'fleet-secret', () => {})
+  try {
+    await server.listen(0, '127.0.0.1')
+    const base = `http://127.0.0.1:${server.port()}`
+    assert.equal((await fetch(`${base}/v1/workers`)).status, 401)
+    const enrollment = await fetch(`${base}/v1/enroll`, { method: 'POST', headers: { authorization: 'Bearer fleet-secret', 'content-type': 'application/json' }, body: JSON.stringify({ workerId: 'ez-cto', checkInSeconds: 10, graceSeconds: 0 }) })
+    assert.equal(enrollment.status, 201)
+    const { workerToken } = await enrollment.json() as { workerToken: string }
+    assert.equal((await fetch(`${base}/v1/workers/ez-cto/check-in`, { method: 'POST', headers: { authorization: 'Bearer wrong', 'content-type': 'application/json' }, body: '{"status":"ok"}' })).status, 401)
+    assert.equal((await fetch(`${base}/v1/workers/ez-cto/check-in`, { method: 'POST', headers: { authorization: `Bearer ${workerToken}`, 'content-type': 'application/json' }, body: '{"status":"ok","activity":"relay started"}' })).status, 200)
+    const rotation = await fetch(`${base}/v1/workers/ez-cto/rotate`, { method: 'POST', headers: { authorization: 'Bearer fleet-secret' } })
+    assert.equal(rotation.status, 200)
+    const rotated = await rotation.json() as { workerToken: string }
+    assert.equal((await fetch(`${base}/v1/workers/ez-cto/check-in`, { method: 'POST', headers: { authorization: `Bearer ${workerToken}`, 'content-type': 'application/json' }, body: '{"status":"ok"}' })).status, 401)
+    assert.equal((await fetch(`${base}/v1/workers/ez-cto/check-in`, { method: 'POST', headers: { authorization: `Bearer ${rotated.workerToken}`, 'content-type': 'application/json' }, body: '{"status":"ok"}' })).status, 200)
+  } finally { await server.close(); await f.close() }
+})
+
+for (const recoveredChannel of ['pagerduty', 'telegram'] as const) {
+  for (const relapse of ['terminal', 'failed', 'missed'] as const) {
+    test(`a ${relapse} relapse reopens only the recovered ${recoveredChannel} channel after restart`, async () => {
+      const stateDir = await mkdtemp(join(tmpdir(), 'workforce-relapse-'))
+      let now = 0, failRecovery = true
+      const pages: string[] = [], notices: string[] = []
+      const options = { stateDir, enrollmentToken: 'synthetic', now: () => now,
+        page: async (event: { action: string }) => {
+          if (event.action === 'resolve' && recoveredChannel === 'telegram' && failRecovery) throw new Error('recovery unavailable')
+          pages.push(event.action)
+        },
+        notify: async (message: string) => {
+          if (message.includes('recovered') && recoveredChannel === 'pagerduty' && failRecovery) throw new Error('recovery unavailable')
+          notices.push(message.includes('recovered') ? 'resolve' : 'trigger')
+        } }
+      try {
+        let watch = new WorkforceWatch(options)
+        const worker = await watch.enroll('synthetic', { workerId: 'synthetic-worker', checkInSeconds: 10, graceSeconds: 0 })
+        await watch.checkIn(worker.workerId, worker.workerToken, { status: 'failed', terminal: true })
+        await watch.checkIn(worker.workerId, worker.workerToken, { status: 'ok' })
+        await assert.rejects(() => watch.checkIn(worker.workerId, worker.workerToken, { status: 'ok' }), /recovery unavailable/)
+        watch = new WorkforceWatch(options)
+        now = relapse === 'missed' ? 10_001 : 1
+        if (relapse === 'missed') await watch.evaluate()
+        else await watch.checkIn(worker.workerId, worker.workerToken, { status: 'failed', terminal: relapse === 'terminal' })
+        assert.deepEqual(pages, recoveredChannel === 'pagerduty' ? ['trigger', 'resolve', 'trigger'] : ['trigger'])
+        assert.deepEqual(notices, recoveredChannel === 'telegram' ? ['trigger', 'resolve', 'trigger'] : ['trigger'])
+        failRecovery = false
+        await watch.checkIn(worker.workerId, worker.workerToken, { status: 'ok' })
+        await watch.checkIn(worker.workerId, worker.workerToken, { status: 'ok' })
+        assert.equal((await watch.inspect(worker.workerId) as { incident?: unknown }).incident, undefined)
+        assert.equal(pages.at(-1), 'resolve')
+        assert.equal(notices.at(-1), 'resolve')
+      } finally { await rm(stateDir, { recursive: true, force: true }) }
+    })
+  }
+}