@jc_stack/ez-agents 0.1.0-beta.50 → 0.1.0-beta.51

package/src/index.ts

@@ -27,7 +27,7 @@ import { InboxStore, type IncomingItem } from './inbox.js'
 import { loadConfig, type Config } from './config.js'
 import { ControlStore, telegramOwner, ownerId, ownerEpoch, type SessionState } from './control-state.js'
 import { ApprovalStore } from './approval.js'
-import { CliUnavailableError, startExecutorJob, attachHostJob, terminateJob, opencodeDataHome, claudeConfigHome } from './executor.js'
+import { CliUnavailableError, startExecutorJob, attachHostJob, terminateJob, opencodeDataHome, claudeConfigHome, usesFileTransport } from './executor.js'
 import { attemptFailure, continuationLine, nextSetup, quotaScope, setupCandidates, startupObserver, unavailableSummary, SetupsUnavailableError, type SetupAttempt } from './setup-fallback.js'
 import { assertScheduledModel } from './model-policy.js'
 import { RunStore, saveRunHandoff, takeRunHandoff, type RunRecord } from './runs.js'
@@ -96,7 +96,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   let activeBackend = false
   const ownerStopped = new WeakSet<ChildProcess>()
   // Host-transport executors outlive this relay; a graceful stop hands them off.
-  const hostTransport = process.env.EZ_EXECUTOR_TRANSPORT === 'host' && !config.channelBackendUrl
+  const hostTransport = async () => !config.channelBackendUrl && await usesFileTransport(config.controlDir)
   const childRuns = new WeakMap<ChildProcess, string>()
   const handedOff = new WeakSet<ChildProcess>()
   let activeChild: ChildProcess | null = null
@@ -668,8 +668,8 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   }
 
   let outboxWork: Promise<void> | undefined
-  const drainOutbox = (onlyRunId?: string): Promise<void> => {
-    if (shuttingDown) return outboxWork ?? Promise.resolve()
+  const drainOutbox = (onlyRunId?: string, stopping = false): Promise<void> => {
+    if (shuttingDown && !stopping) return outboxWork ?? Promise.resolve()
     return outboxWork ?? (outboxWork = (async () => {
       for (const item of await runs.pendingOutbox()) {
         if (onlyRunId && item.runId !== onlyRunId) continue
@@ -773,6 +773,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   // children, host executor, CLIs) enqueue and query the memory ledger
   // through the shared handler in delivery-socket.ts. The pump below stays
   // the single delivery executor; the socket only admits work into it.
+  let drainingHandoff=false
   let deliveryServer: { socketPath: string; stop: () => Promise<void> } | null = null
   const handleDeliveryOp = createLedgerHandler(config.controlDir, {
     wake: () => { void drainOutbox() },
@@ -835,7 +836,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     const owner = (await control.status()).owner
     if (!owner) return 'Scheduled tasks\n\nNo paired owner.'
     // This intentionally uses the reader that does not create a schedules directory.
-    const active = await scheduler.listActiveReadOnly(await runs.list())
+    const active = await scheduler.listTasksReadOnly(await runs.list())
     return { owner, active, text: scheduledTasksText(active, owner) }
   }
   const replyScheduledTasks = async (ctx: Context, number?: number) => {
@@ -892,6 +893,18 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
         : []),
     ].join('\n')
   }
+  const stopActiveWork = async (): Promise<boolean> => {
+    const children: [string | undefined, ChildProcess][] = [...background]
+    if (activeChild) children.unshift([undefined, activeChild])
+    let requested = true
+    for (const [id, child] of children) {
+      if (!await terminateJob(child)) { requested = false; continue }
+      ownerStopped.add(child)
+      if (id) await scheduler.cancel(id)
+    }
+    return requested
+  }
+
   const cancelPending = async () => {
     let count = 0
     await withStartLock(async () => {
@@ -1030,16 +1043,15 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     if (text === '/stop') {
       const running = await runs.running(false)
       if ((running && running.pid) || background.size) {
-        try {
-          if (activeChild) { ownerStopped.add(activeChild); terminateJob(activeChild) }
-          for (const [id,child] of background) { await scheduler.cancel(id); ownerStopped.add(child); terminateJob(child) }
-        } catch {}
-        if (activeTypingTimer) {
+        let requested = false
+        try { requested = await stopActiveWork() } catch {}
+        if (requested && activeTypingTimer) {
           clearInterval(activeTypingTimer)
           activeTypingTimer = null
         }
         await ctx.reply(
-          '🛑 Stop requested for active work. Queued work remains and will run next. /cancel clears it.',
+          requested ? '🛑 Stop requested for active work. Queued work remains and will run next. /cancel clears it.'
+            : 'A stop request failed; some work remains active.',
         )
       } else {
         await ctx.reply('No active run in progress.')
@@ -1262,11 +1274,12 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
       } else if (action === 'stop') {
         const running = await runs.running(false)
         if ((running && running.pid) || background.size) {
-          if (activeChild) { ownerStopped.add(activeChild); terminateJob(activeChild) }
-          for (const [id,child] of background) { await scheduler.cancel(id); ownerStopped.add(child); terminateJob(child) }
-          await ctx.answerCallbackQuery({ text: 'Run stopped' })
+          let requested = false
+          try { requested = await stopActiveWork() } catch {}
+          await ctx.answerCallbackQuery({ text: requested ? 'Stop requested' : 'Stop request failed' })
           await ctx.reply(
-            '🛑 Stop requested for active work. Queued work remains and will run next. /cancel clears it.',
+            requested ? '🛑 Stop requested for active work. Queued work remains and will run next. /cancel clears it.'
+              : 'A stop request failed; some work remains active.',
           )
         } else {
           await ctx.answerCallbackQuery({ text: 'No run active' })
@@ -1288,7 +1301,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   // unknown outcome is recorded interrupted, which holds its schedule.
   const adoptHandoff = async (): Promise<void> => {
     const handed = await takeRunHandoff(config.controlDir)
-    if (!hostTransport) { if (handed.length) console.error('Ignoring run handoff without host transport', handed.length); return }
+    if (!await hostTransport()) { if (handed.length) console.error('Ignoring run handoff without host transport', handed.length); return }
     // Admit every record before the first attach, so no trigger races a hold.
     const admitted = (await Promise.all(handed.map(run => runs.adopt(run).catch(() => null)))).filter((run): run is RunRecord => run !== null)
     for (const run of admitted) await withStartLock(async () => {
@@ -1353,21 +1366,25 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     shuttingDown = true
     runtimeStarted = false
     pollingAbort.abort()
+    let handedRecords:RunRecord[]=[]
     // Hand off running host work before any wait can outlast the stop grace
     // period. The lock lets an in-flight launch register; later ones see shutdown.
-    if (hostTransport) await withStartLock(async () => {
+    if (await hostTransport()) await withStartLock(async () => {
       const handoff = new Map<ChildProcess, RunRecord>()
       for (const child of [...(activeChild ? [activeChild] : []), ...background.values()]) {
         const run = childRuns.has(child) ? await runs.get(childRuns.get(child)!) : null
         if (run?.status === 'running' && !ownerStopped.has(child) && child.exitCode === null && child.signalCode === null) handoff.set(child, run)
       }
       if (!handoff.size) return
-      try { await saveRunHandoff(config.controlDir, [...handoff.values()]) }
+      try { handedRecords=[...handoff.values()];await saveRunHandoff(config.controlDir, handedRecords) }
       catch (error) { console.error('Run handoff failed; stopping host work', safeError(error)); return }
       // SIGKILL leaves no cancel marker, so the host keeps each executor running.
       for (const child of handoff.keys()) { handedOff.add(child); child.kill('SIGKILL') }
     })
-    await deliveryServer?.stop().catch(() => {})
+    drainingHandoff=handedRecords.length>0
+    if(deliveryServer)await deliveryServer.stop()
+    else if(drainingHandoff){await outboxWork?.catch(() => {});await drainOutbox(undefined,true)}
+    if(handedRecords.length)await saveRunHandoff(config.controlDir,handedRecords)
     deliveryServer = null
     await applicationChannel.stop()
     wakePollRetry?.()
@@ -1402,6 +1419,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
         throw new Error(`Another relay owns ${config.controlDir}; stop it before starting a second one`)
       const deliveryTcpPort = Number(process.env.EZ_DELIVERY_TCP_PORT ?? '')
       deliveryServer = await serveDeliverySocket(config.controlDir, handleDeliveryOp, {
+        beforeClose:async()=>{if(drainingHandoff){await outboxWork?.catch(()=>{});await drainOutbox(undefined,true)}},
         ...(Number.isSafeInteger(deliveryTcpPort) && deliveryTcpPort >= 1024 && deliveryTcpPort <= 65535 ? { tcpPort: deliveryTcpPort } : {}),
       })
       // Reconcile, then adopt handed-off runs, before any intake can launch work.

package/src/plugins/workspace-lease.mjs

@@ -23,7 +23,7 @@ export async function recoverNativeLease(home) {
 export async function workspaceLease(home,owner={kind:'plugin'}) {
   const file=path.join(home,'workspace-writer.lock'),temp=file+'.'+randomUUID()+'.tmp';
   await fs.writeFile(temp,JSON.stringify({...owner,pid:process.pid}),{flag:'wx',mode:0o600});
-  try {await fs.link(temp,file);}catch(error){if(error.code!=='EEXIST')throw error;let prior;try {prior=await leaseOwner(file);}catch(error){if(error.code==='ENOENT')return undefined;throw error;}if(!prior.alive)throw Error('Stale workspace-writer.lock: inspect stopped owner and command containers, then restart for recovery');return undefined;}
+  try {await fs.link(temp,file);}catch(error){if(error.code!=='EEXIST')throw error;let prior;try {prior=await leaseOwner(file);}catch(error){if(error.code==='ENOENT')return undefined;throw error;}if(!prior.alive)throw Error('Stale workspace-writer.lock: inspect stopped owner and command containers, then restart for recovery');if(owner.kind==='native'&&prior.owner.kind==='native')return async()=>{};return undefined;}
   finally {await fs.rm(temp);}
   return async()=>{await fs.rm(file);};
 }

package/src/runs.ts

@@ -162,7 +162,9 @@ const handedRun = (value: unknown): value is RunRecord => {
 
 export const saveRunHandoff = async (controlDir: string, records: RunRecord[]): Promise<void> => {
   const file = handoffFile(controlDir), temporary = `${file}.${process.pid}.tmp`
-  await writeFile(temporary, JSON.stringify({ version: 1, writtenAt: new Date().toISOString(), runs: records }), { mode: 0o600 })
+  const ids=new Set(records.map(run=>run.id))
+  const receipts=[...outboxFor(controlDir).values()].filter(item=>item.runId&&ids.has(item.runId)&&(item.state==='sent'||item.state==='failed'))
+  await writeFile(temporary, JSON.stringify({ version: 1, writtenAt: new Date().toISOString(), runs: records, receipts }), { mode: 0o600 })
   await rename(temporary, file)
 }
 
@@ -176,12 +178,16 @@ export const takeRunHandoff = async (controlDir: string, now = Date.now()): Prom
     try { raw = await readFile(handoffFile(controlDir), 'utf8') }
     catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error }
     await rm(handoffFile(controlDir))
-    const value = JSON.parse(raw) as { version?: unknown; writtenAt?: unknown; runs?: unknown }
+    const value = JSON.parse(raw) as { version?: unknown; writtenAt?: unknown; runs?: unknown; receipts?: unknown }
     const age = now - Date.parse(typeof value.writtenAt === 'string' ? value.writtenAt : '')
     if (value.version !== 1 || !(age >= -60_000 && age <= HANDOFF_MAX_AGE_MS)) throw new Error('Invalid or stale run handoff')
     if (!Array.isArray(value.runs) || value.runs.length > 100) throw new Error('Invalid run handoff')
     const valid = value.runs.filter(handedRun)
     if (valid.length !== value.runs.length) console.error('Ignoring invalid handed-off runs', value.runs.length - valid.length)
+    const ids=new Set(valid.map(run=>run.id))
+    if(Array.isArray(value.receipts)&&value.receipts.length<=MAX_TERMINAL_OUTBOX){
+      for(const item of value.receipts as StoredOutboxItem[])if(item&&typeof item.id==='string'&&/^[a-zA-Z0-9_-]+$/.test(item.id)&&item.runId&&ids.has(item.runId)&&(item.state==='sent'||item.state==='failed'))outboxFor(controlDir).set(item.id,item)
+    }
     return [...new Map(valid.map(run => [run.id, run])).values()]
   } catch (error) {
     console.error('Ignoring unreadable run handoff', error instanceof Error ? error.message : 'Unknown error')

package/src/schedule-cli.ts

@@ -83,7 +83,8 @@ The run records each attempt, setup, failure category and any reported reset tim
 Trigger runs an existing task once with its saved instructions, AI and delivery binding in a fresh session. Reuse the request key after an uncertain result; the regular schedule is unchanged.
 Creates a scheduled task. Instructions are text, never shell commands.
 Use --now to run once. Run completion is not delivery proof.
-Edit replaces the full schedule. Pause/remove affect future work; cancel stops a particular run.
+Edit replaces the full schedule but keeps its paused/enabled state; only pause/resume toggle it.
+Pause/remove affect future work; cancel stops a particular run.
 Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/weekday OR semantics.`);return}
   const config=loadControlConfig(), control=new ControlStore(config.controlDir,config.pairingTtlMs)
   const owner=(await control.status()).owner
@@ -195,13 +196,15 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
       v.cron ? {cron:v.cron,timezone:v.timezone!,start,until:v.until} : {everySeconds:Number(v['every-seconds']),start,until:v.until}
     if(v['clear-preflight'] && v['preflight-file']) throw Error('Choose one preflight operation')
     const previousSchedule = action === 'edit' ? await scheduler.get(id!) : undefined
+    // Edit keeps the paused/enabled state; only pause/resume toggle it.
+    const enabled = previousSchedule?.enabled ?? true
     const preflight = v['clear-preflight'] ? undefined : v['preflight-file'] ? JSON.parse(await readFile(v['preflight-file'],'utf8')) : previousSchedule?.preflight
     const origin = caller?.application ?? caller?.delivery
     const delivery = previousSchedule ? previousSchedule.delivery : (origin ? {bindingId:origin.bindingId,scope:origin.scope} : undefined)
     if (!delivery && !owner.telegramChatId) throw new Error('Create the schedule from an authenticated channel turn to bind its reply destination')
     if (v.script) {
       result=await show(await scheduler.save({id:id || 's_'+randomUUID(),name:v.name || v.script,preflight,
-        originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:'',when:v.when as 'unreviewed-failures' | undefined,trigger,enabled:true,owner,
+        originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:'',when:v.when as 'unreviewed-failures' | undefined,trigger,enabled,owner,
         script:{id:v.script,args:v.arg ?? []}},action==='create'))
       console.log(JSON.stringify(result,null,2));return
     }
@@ -216,7 +219,7 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
       if (setup.authProfile !== undefined) await provisionedHome(config.controlDir, setup.cli, setup.authProfile)
     result=await show(await scheduler.save({id:id || 's_'+randomUUID(),name:v.name || 'Task',
       preflight,
-      originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:v.text || await readFile(v['text-file']!,'utf8'),when:v.when as 'unreviewed-failures' | undefined,trigger,enabled:true,owner,
+      originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:v.text || await readFile(v['text-file']!,'utf8'),when:v.when as 'unreviewed-failures' | undefined,trigger,enabled,owner,
       execution:{sessionId:previous?.sessionId || randomUUID(),preset,...(fallbacks ? {fallbacks} : {})}},action==='create'))
   }else{
     if(!id)throw new Error('ID required')

package/src/scheduled-tasks.ts

@@ -2,7 +2,7 @@ import { executionDefaults } from './model-policy.js'
 import { presetLabel, type AiPreset } from './ai.js'
 import { AUTH_PROFILE_CLIS } from './auth-profile.js'
 import type { Owner } from './control-state.js'
-import type { Schedule, ActiveSchedule } from './scheduler.js'
+import type { Schedule, ScheduledTaskView } from './scheduler.js'
 
 const ownsSchedule = (owner: Owner, schedule: Schedule) =>
   schedule.owner.telegramUserId === owner.telegramUserId &&
@@ -24,7 +24,7 @@ const setupLabel = (preset: AiPreset) => {
 }
 
 // Script schedules have no AI preset; show what core actually invokes.
-const engineLabel = (schedule: ActiveSchedule) => schedule.script
+const engineLabel = (schedule: ScheduledTaskView) => schedule.script
   ? `Script · ${schedule.script.id} · ${schedule.scriptRevision ? `rev ${schedule.scriptRevision.slice(0, 8)}` : 'not registered'}`
   : [schedule.execution!.preset, ...schedule.execution!.fallbacks ?? []].map(setupLabel).join(' → ')
 
@@ -67,28 +67,30 @@ const preview = (schedule: Schedule) => {
   return chars.length > 140 ? chars.slice(0,139).join('')+'…' : sentence
 }
 
-const currentState = (schedule: ActiveSchedule) =>
-  schedule.runState === 'running' ? '▶ Running' : schedule.runState === 'queued' ? '◌ Queued' : '● Active'
+const currentState = (schedule: ScheduledTaskView) =>
+  [schedule.runState === 'running' ? '▶ Running' : schedule.runState === 'queued' ? '◌ Queued' : undefined,
+    !schedule.enabled ? '⏸ Paused' : undefined].filter(Boolean).join(' · ') || '● Active'
 
-const lastRun = (schedule: ActiveSchedule) => {
+const lastRun = (schedule: ScheduledTaskView) => {
   if (!schedule.lastRun) return '— Not run yet'
   const status = schedule.lastRun.status === 'completed' ? '✓ Completed' : schedule.lastRun.status === 'failed' ? '✕ Failed' : '⊘ Cancelled'
   return `${status} · ${time(schedule.lastRun.at)}${schedule.lastRun.currentRevision ? '' : ' · previous version'}`
 }
 
-export const ownedScheduledTasks = (schedules: ActiveSchedule[], owner: Owner) =>
+export const ownedScheduledTasks = (schedules: ScheduledTaskView[], owner: Owner) =>
   schedules.filter((schedule) => ownsSchedule(owner, schedule))
     .sort((a, b) => a.name.localeCompare(b.name))
 
-export const scheduledTasksText = (schedules: ActiveSchedule[], owner: Owner) => {
+export const scheduledTasksText = (schedules: ScheduledTaskView[], owner: Owner) => {
   const owned = ownedScheduledTasks(schedules, owner)
-  if (!owned.length) return '📅 Scheduled tasks\n\nNo active scheduled tasks.'
-  return [`📅 Scheduled tasks · ${owned.length} active`, 'Next/last times UTC. Use /tasks 1 for task details.', ...owned.map((schedule, index) =>
+  if (!owned.length) return '📅 Scheduled tasks\n\nNo scheduled tasks.'
+  const paused = owned.filter(schedule => !schedule.enabled).length
+  return [`📅 Scheduled tasks · ${owned.length - paused} active · ${paused} paused`, 'Next/last times UTC. Use /tasks 1 for task details.', ...owned.map((schedule, index) =>
     `${index + 1} · ${schedule.name}\n  ${currentState(schedule)}\n  Schedule · ${frequency(schedule)}\n  Next · ${schedule.nextAt === null ? '—' : time(schedule.nextAt)}\n  Last · ${lastRun(schedule)}\n  Execution · ${engineLabel(schedule)}\n  ${preview(schedule)}`
   )].join('\n\n')
 }
 
-export const scheduledTaskDetailText = (schedule: ActiveSchedule, number: number, total?: number) => {
+export const scheduledTaskDetailText = (schedule: ScheduledTaskView, number: number, total?: number) => {
   const instructions = schedule.script ? preview(schedule) : schedule.text.trim()
   const capped = instructions.length > 2000 ? instructions.slice(0, 2000) + '… (truncated; full text lives in the workspace)' : instructions
   return [

package/src/scheduler.ts

@@ -29,7 +29,7 @@ export type ScheduledRunSummary = {
   at: string
   currentRevision: boolean
 }
-export type ActiveSchedule = Schedule & { nextAt: number | null; runState?: 'queued' | 'running'; lastRun?: ScheduledRunSummary; scriptRevision?: string }
+export type ScheduledTaskView = Schedule & { nextAt: number | null; runState?: 'queued' | 'running'; lastRun?: ScheduledRunSummary; scriptRevision?: string }
 export type ScheduledOrigin = { id: string; revision: string; dueAt: string; pairedAt: string; originRunId?: string }
 export const validScheduledOrigin = (v: unknown): v is ScheduledOrigin => {
   const s = v as ScheduledOrigin
@@ -102,10 +102,9 @@ export class Scheduler {
       return nextOccurrence(s.trigger,-1)
     }
   }
-  async listActiveReadOnly(runs: RunRecord[]): Promise<ActiveSchedule[]> {
-    const active: ActiveSchedule[] = []
+  async listTasksReadOnly(runs: RunRecord[]): Promise<ScheduledTaskView[]> {
+    const active: ScheduledTaskView[] = []
     for (const s of await this.listReadOnly()) {
-      if (!s.enabled) continue
       const current = runs.filter(r => r.scheduled?.id === s.id && r.scheduled.revision === s.revision && ownsRun(s.owner,r))
       const history = runs.filter(r => r.scheduled?.id === s.id && r.scheduled.pairedAt === s.owner.pairedAt && ownsRun(s.owner,r))
         .filter((r): r is RunRecord & { status: ScheduledRunSummary['status'] } =>
@@ -113,11 +112,11 @@ export class Scheduler {
         .sort((a,b) => (b.endedAt ?? b.startedAt ?? b.createdAt).localeCompare(a.endedAt ?? a.startedAt ?? a.createdAt))
       const last = history[0]
       const runState = current.some(r => r.status === 'running') ? 'running' : current.some(r => r.status === 'queued') ? 'queued' : undefined
-      if (!runState && current.some(r => holdsSchedule(s,r))) continue
+      if (s.enabled && !runState && current.some(r => holdsSchedule(s,r))) continue
       try {
-        const nextAt = await this.pendingOccurrence(s)
+        const nextAt = s.enabled ? await this.pendingOccurrence(s) : null
         const scriptRevision = s.script ? await new Scripts(this.controlDir).get(s.script.id).then(r => r.revision, () => undefined) : undefined
-        if (runState || nextAt !== null) active.push({...s,nextAt,runState,...(scriptRevision ? {scriptRevision} : {}),...(last ? {
+        if (!s.enabled || runState || nextAt !== null) active.push({...s,nextAt,runState,...(scriptRevision ? {scriptRevision} : {}),...(last ? {
           lastRun: {status:last.status,at:last.endedAt ?? last.startedAt ?? last.createdAt,currentRevision:last.scheduled!.revision===s.revision},
         } : {})})
       }

package/src/updates/runtime.mjs

@@ -33,8 +33,8 @@ export function execute(command,args,options={}) {
 export async function textAtomic(file,text) {
   const tmp=file+'.update.tmp';await fs.writeFile(tmp,text,{mode:0o600});await fs.rename(tmp,file);
 }
-export async function backupStateDirectory(source,destination) {
-  await fs.cp(source,destination,{recursive:true,filter:async entry=>!(await fs.lstat(entry)).isSocket()});
+export async function backupStateDirectory(source,destination,omit=[]) {
+  await fs.cp(source,destination,{recursive:true,filter:async entry=>!omit.includes(path.relative(source,entry).split(path.sep)[0])&&!(await fs.lstat(entry)).isSocket()});
 }
 export const pluginArgs = record => ['compose','--project-name',record.project,'--file',record.compose];
 export async function packageManager(root,run=execute) {
@@ -159,7 +159,7 @@ export async function perform(home,job,hooks) {
   const root=path.join(dir,'runtime');await fs.rm(root,{recursive:true,force:true});await extract(archive,root);
   const next=await eligibility(home,job.target,root,job.automatic);
   if(next.old.root!==job.previousRoot||next.old.pkg.version!==job.previousVersion)throw Error('Stale upgrade job');
-  job.status='applying';job.root=root;job.startedAt=new Date().toISOString();await save();
+  job.status='applying';job.preserveRuns=hooks.preserveRuns===true;job.root=root;job.startedAt=new Date().toISOString();await save();
   const complete=async()=>{
     job.status='completed';job.endedAt=new Date().toISOString();await save();
     try { job.cleanup=await cleanupStaleBackups(home);await save(); }
@@ -193,17 +193,30 @@ export async function perform(home,job,hooks) {
       });
       const image=`ez-upgrade-${digest(JSON.stringify([job.sha256,buildArgs])).slice(0,24)}`;
       await run('docker',['build','--target','runtime',...buildArgs,'-t',image,root]);
+      const backup=path.join(dir,'backup');
+      const backupState=async()=>{
+        await fs.mkdir(backup,{mode:0o700});
+        await backupStateDirectory(config.workspace,path.join(backup,'workspace'));
+        await backupStateDirectory(agent.controlDir,path.join(backup,'control'),hooks.preserveRuns ? ['host-executor'] : []);
+        for(const name of ['docker.env','host-executor.json','agent.json','purpose.md','relay.env'])await fs.copyFile(path.join(config.deploymentDir,name),path.join(backup,name));
+      };
+      // Native client state can be many gigabytes. This advisory copy never
+      // rewinds live state, so compatible generations keep serving throughout it.
+      if(hooks.preserveRuns)await backupState();
       const envFile=path.join(config.deploymentDir,'docker.env'),oldEnv=await fs.readFile(envFile,'utf8');
       const cid=(await run('docker',[...relayArgs(config),'ps','-q','relay'])).trim();
       if(!cid||/\s/.test(cid))throw Error('Expected one running relay');
       const oldImage=(await run('docker',['inspect','--format','{{.Image}}',cid])).trim();
       if(!/^sha256:[a-f0-9]{64}$/.test(oldImage))throw Error('Cannot pin rollback image');
       job.rollback={env:envValue(oldEnv,'EZ_RELAY_IMAGE',oldImage),packageRoot:config.packageRoot};await save();
-      await run('docker',[...relayArgs(config),'stop','relay']);await hooks.stopHost();
-      const backup=path.join(dir,'backup');await fs.mkdir(backup,{mode:0o700});
-      await backupStateDirectory(config.workspace,path.join(backup,'workspace'));
-      await backupStateDirectory(agent.controlDir,path.join(backup,'control'));
-      for(const name of ['docker.env','host-executor.json','agent.json','purpose.md','relay.env'])await fs.copyFile(path.join(config.deploymentDir,name),path.join(backup,name));
+      const activate=async()=>{
+        const current=await eligibility(home,job.target,root,job.automatic);
+        if(current.old.root!==job.previousRoot||current.old.pkg.version!==job.previousVersion)throw Error('Stale upgrade job');
+        const pause=path.join(agent.controlDir,'upgrade-pause.json');
+        if(hooks.preserveRuns)await atomic(pause,{id:job.id});
+        try {
+      await run('docker',[...relayArgs(config),'stop','relay']);await (hooks.preserveRuns ? hooks.retireHost() : hooks.stopHost());
+      if(!hooks.preserveRuns)await backupState();
       let env=oldEnv.split(job.previousRoot+path.sep).join(root+path.sep);env=envValue(env,'EZ_RELAY_IMAGE',image);
       await textAtomic(envFile,env);
       await bindUpdates(home,path.join(config.deploymentDir,'host-executor.json'),root);
@@ -213,6 +226,9 @@ export async function perform(home,job,hooks) {
         const brokerImage=(await run('docker',['image','inspect','--format','{{.Id}}',image],{stdoutOnly:true})).trim();
         job.brokerReadiness=await refreshBroker(config,run,brokerImage);
       }
+        }finally{if(hooks.preserveRuns)await fs.rm(pause,{force:true});}
+      };
+      if(hooks.preserveRuns)await locked(home,activate,{drainInvocations:true,signal:hooks.signal});else await activate();
     } else {
       const old=next.old.record;
       const secrets=await read(path.join(home,'packages',job.target,'secrets.json')).catch(e=>{if(e.code==='ENOENT')return {};throw e;});
@@ -266,7 +282,7 @@ export async function recover(home,job,hooks) {
           return /^sha256:[a-f0-9]{64}$/.test(pinned||'')&&current.Image===pinned&&current.State?.Running===true&&current.State?.Health?.Status==='healthy';
         })().catch(()=>false);
         if(!retained)await run('docker',[...relayArgs(config),'stop','relay']);
-        await hooks.stopHost();
+        await (hooks.preserveRuns ? hooks.retireHost() : hooks.stopHost());
         await textAtomic(path.join(config.deploymentDir,'docker.env'),job.rollback.env);
         await bindUpdates(home,path.join(config.deploymentDir,'host-executor.json'),job.rollback.packageRoot);
         await hooks.startHost(job.rollback.packageRoot);

package/src/updates/supervisor.mjs

@@ -5,7 +5,7 @@ import { spawn } from 'node:child_process';
 import { atomic, locked } from '../plugins/manager.mjs';
 import { callDeliverySocket } from '../delivery-socket-client.mjs';
 import { state, read, jobs, jobPath, check, prepare, submit, missing, cleanupStaleBackups } from './control.mjs';
-import { perform, environment, brokerNeedsRefresh } from './runtime.mjs';
+import { perform, environment, brokerNeedsRefresh, execute, relayArgs } from './runtime.mjs';
 
 const reservedProviderKeys=new Set(['HOME','LANG','LC_ALL','LOGNAME','PATH','SHELL','TERM','TMPDIR','USER','CODEX_HOME','NODE_OPTIONS']);
 const providerEnvironmentKey=value=>{
@@ -54,7 +54,7 @@ export async function withIdleUpgrade(control,job,signal,apply,{isIdle=idle,wait
 }
 // Plugin replacement drains command leases at activation, not native turns.
 export async function withUpgrade(control,job,signal,apply,options) {
-  if(job.target==='main'||options?.requiresIdle)return withIdleUpgrade(control,job,signal,apply,options);
+  if((job.target==='main'&&!options?.preserveRuns)||options?.requiresIdle)return withIdleUpgrade(control,job,signal,apply,options);
   if(!signal.aborted)return apply();
 }
 export async function queueAutomatic(home,available) {
@@ -84,20 +84,88 @@ export async function supervise(deployment,signal,{discover=check}={}) {
   if(prior&&(!Number.isSafeInteger(prior.pid)||prior.pid<1))throw Error('Corrupt supervisor lock');
   if(prior){try{process.kill(prior.pid,0);throw Error('Supervisor already running');}catch(e){if(e.code!=='ESRCH')throw e;}await fs.rm(lock);}
   await fs.writeFile(lock,JSON.stringify({pid:process.pid}),{flag:'wx',mode:0o600});
-  let child;
+  let child,container;
+  let retirements=[];
+  let reloadPending=false;
+  const generations=new Map();
+  const removeContainer=async id=>{
+    const generation=generations.get(id);
+    if(generation){await execute('docker',['compose','--project-name',generation.project,'--file',generation.file,'down']);await fs.rm(generation.file,{force:true});await fs.rm(generation.configFile,{force:true});generations.delete(id);}
+    else await execute('docker',['rm',id]);
+  };
   const stopHost=async()=>{
+    if(container){await execute('docker',['stop','--time','10',container]);await removeContainer(container);container=undefined;}
     if(!child)return;const running=child;child=undefined;
     if(running.exitCode!==null)return;
     const closed=new Promise(resolve=>running.once('close',resolve));running.kill('SIGTERM');
     const timer=setTimeout(()=>running.kill('SIGKILL'),10000);await closed;clearTimeout(timer);
   };
   const startHost=async root=>{
-    if(host.isolation==='isolated')return
+    if(host.isolation==='isolated') {
+      const pkg=await read(path.join(root,'package.json'));
+      if(pkg.ezRelease?.containerExecutor!==1)return;
+      const lock=path.join(agent.controlDir,'host-executor/worker.lock');
+      const prior=await read(lock).catch(missing);
+      if(prior?.containerId){
+        if(!/^[a-f0-9]{12,64}$/.test(prior.containerId))throw Error('Invalid previous isolated worker identity');
+        const running=await execute('docker',['inspect','--format','{{.State.Running}}',prior.containerId],{stdoutOnly:true}).catch(()=>null);
+        if(running?.trim()==='true')throw Error('Previous isolated worker is still running');
+        // Only the supervisor can prove a foreign container stopped.
+        if(JSON.stringify(await read(lock))!==JSON.stringify(prior))throw Error('Worker lock changed during recovery');
+        await fs.rm(lock);
+      }
+      const config=(await state(home)).config;
+      const deployment=JSON.parse(await execute('docker',[...relayArgs(config),'config','--format','json'],{stdoutOnly:true}));
+      const relay=deployment.services.relay;
+      if(!relay?.image||!Array.isArray(relay.volumes))throw Error('Invalid isolated relay binding');
+      const generation=path.basename(root).replace(/[^a-zA-Z0-9_.-]/g,'-')+'-'+Date.now();
+      const project='ez-executor-'+generation.toLowerCase();
+      const file=path.join(directory,project+'.json');
+      const configFile=path.join(directory,project+'-binding.json');
+      await atomic(configFile,{...host,agents:host.agents.map(a=>({...a,binDir:'/app/bin'}))});
+      const worker={image:relay.image,init:true,restart:'no',user:`${relay.environment.EZ_RUNTIME_UID||1000}:${relay.environment.EZ_RUNTIME_GID||1000}`,
+        cap_drop:['ALL'],security_opt:['no-new-privileges:true'],entrypoint:['node','--import','/app/node_modules/tsx/dist/loader.mjs','/app/src/host-executor.ts','/run/executor.json'],
+        environment:{...relay.environment,EZ_CONTAINER_EXECUTOR:'1',EZ_EXECUTOR_TRANSPORT:'local'},
+        volumes:[...relay.volumes,{type:'bind',source:configFile,target:'/run/executor.json',read_only:true}],tmpfs:['/tmp'],
+        ...Object.fromEntries(['networks','network_mode','extra_hosts','dns','dns_search'].filter(key=>relay[key]!==undefined).map(key=>[key,relay[key]]))};
+      if(worker.networks&&!Array.isArray(worker.networks))worker.networks=Object.fromEntries(Object.keys(worker.networks).map(name=>[name,{}]));
+      const networks=worker.networks?Object.fromEntries(Object.entries(deployment.networks||{}).map(([name,binding])=>[name,{name:binding.name,external:true}])):undefined;
+      await atomic(file,{services:{executor:worker},...(networks?{networks}:{})});
+      await execute('docker',['compose','--project-name',project,'--file',file,'up','-d','--no-build','executor']);
+      container=(await execute('docker',['compose','--project-name',project,'--file',file,'ps','-q','executor'],{stdoutOnly:true})).trim();
+      if(!/^[a-f0-9]{12,64}$/.test(container))throw Error('Missing isolated executor');
+      generations.set(container,{project,file,configFile});
+      for(let n=0;n<300;n++){
+        const running=await execute('docker',['inspect','--format','{{.State.Running}}',container],{stdoutOnly:true});
+        if(running.trim()!=='true')throw Error('Isolated executor exited during startup');
+        const heartbeat=await read(path.join(agent.controlDir,'host-executor/heartbeat.json')).catch(missing);
+        if(heartbeat?.containerExecutor===true&&container.startsWith(heartbeat.containerId||'missing')&&Date.now()-heartbeat.at<1000)return;
+        await sleep(100);
+      }
+      throw Error('Isolated executor heartbeat timeout');
+    }
     child=spawn(process.execPath,['--import',path.join(root,'node_modules/tsx/dist/loader.mjs'),path.join(root,'src/host-executor.ts'),path.join(deployment,'host-executor.json')],
-      {env:{...environment(),...providerEnvironment(host),EZ_HOST_SUPERVISOR_PID:String(process.pid)},stdio:['ignore','inherit','inherit']});
+      {detached:true,env:{...environment(),...providerEnvironment(host),EZ_HOST_SUPERVISOR_PID:String(process.pid)},stdio:['ignore','inherit','inherit']});
     let error;child.once('error',e=>{error=e;});
     await waitForHostHeartbeat(agent.controlDir,child,()=>error);
   };
+  const retireHost=async()=>{
+    if(container){await execute('docker',['kill','--signal','SIGUSR2',container]);retirements.push({container});container=undefined;}
+    if(child){const running=child;child=undefined;if(running.exitCode===null){running.kill('SIGUSR2');retirements.push({child:running});}}
+    // A retiring generation releases its single claim lock before replacement.
+    for(let n=0;n<300;n++){
+      if(!await fs.stat(path.join(agent.controlDir,'host-executor/worker.lock')).then(()=>true,e=>{if(e.code==='ENOENT')return false;throw e;}))return;
+      await sleep(100);
+    }
+    throw Error('Native executor retirement timeout');
+  };
+  const preserveRuns=async job=>{
+    if(job.target!=='main')return false;
+    const old=await read(path.join((await state(home)).config.packageRoot,'package.json'));
+    const candidate=await read(path.join(jobPath(home,job.id),'package/package.json'));
+    return old.ezRelease?.runHandoff===1&&candidate.ezRelease?.runHandoff===1&&
+      (host.isolation!=='isolated'||(container&&old.ezRelease?.containerExecutor===1&&candidate.ezRelease?.containerExecutor===1));
+  };
   const pause=path.join(agent.controlDir,'upgrade-pause.json');
   const refreshRequirement=async job=>{
     if(job.target==='main')return false;
@@ -119,36 +187,56 @@ export async function supervise(deployment,signal,{discover=check}={}) {
     if(owner&&prior&&owner.pid===prior.pid)await fs.rm(registryLock);
     // Wait for an orphaned transport's parent-watch to terminate it first.
     await sleep(1200);
+    const workerOwner=await read(path.join(agent.controlDir,'host-executor/worker.lock')).catch(missing);
+    if(host.isolation==='isolated'&&workerOwner?.containerId){
+      if(!/^[a-f0-9]{12,64}$/.test(workerOwner.containerId))throw Error('Invalid isolated worker identity');
+      if((await execute('docker',['inspect','--format','{{.State.Running}}',workerOwner.containerId],{stdoutOnly:true}).catch(()=>''))?.trim()==='true')container=workerOwner.containerId;
+    }
     const interrupted=(await jobs(home)).find(j=>j.status==='applying');
     if(interrupted){
       const refreshPluginBroker=await refreshRequirement(interrupted);
       const result=refreshPluginBroker===null?undefined:interrupted.target==='main'
-        ? await (async()=>{await atomic(pause,{id:interrupted.id});try{return await locked(home,()=>perform(home,interrupted,{stopHost,startHost}));}finally{await fs.rm(pause,{force:true});}})()
-        : await withUpgrade(agent.controlDir,interrupted,signal,()=>perform(home,interrupted,{stopHost,startHost,signal,refreshPluginBroker}),{requiresIdle:refreshPluginBroker});
-      if(result?.status==='completed'&&interrupted.target==='main')return;
+        ? await (async()=>{await atomic(pause,{id:interrupted.id});try{return await locked(home,()=>perform(home,interrupted,{stopHost,startHost,retireHost,preserveRuns:interrupted.preserveRuns===true}));}finally{await fs.rm(pause,{force:true});}})()
+        : await withUpgrade(agent.controlDir,interrupted,signal,()=>perform(home,interrupted,{stopHost,startHost,retireHost,signal,refreshPluginBroker}),{requiresIdle:refreshPluginBroker});
+      if(['completed','rolled-back'].includes(result?.status)&&interrupted.target==='main')reloadPending=true;
     }
     const isolated=host.isolation==='isolated'
-    if(!isolated && !child)await startHost((await state(home)).config.packageRoot);
+    if(!isolated&&!child)await startHost((await state(home)).config.packageRoot);
+    const initializeIsolated=async()=>{
+      if(!isolated||container)return;
+      const root=(await state(home)).config.packageRoot;
+      if((await read(path.join(root,'package.json'))).ezRelease?.containerExecutor!==1)return;
+      await withIdleUpgrade(agent.controlDir,{id:'executor-migration'},signal,()=>startHost(root));
+    };
+    await initializeIsolated();
     let nextCheck=0;
     while(!signal.aborted) {
       if(!isolated && child?.exitCode!==null&&child?.exitCode!==undefined)throw Error('Host transport exited; supervisor service should restart');
-      const pending=(await jobs(home)).find(j=>j.status==='queued');
+      await initializeIsolated();
+      if(container&&(await execute('docker',['inspect','--format','{{.State.Running}}',container],{stdoutOnly:true})).trim()!=='true')throw Error('Isolated executor exited; supervisor service should restart');
+      if(reloadPending&&await idle(agent.controlDir))return;
+      // Reap retired containers only after their worker exits.
+      for(const item of retirements.filter(r=>r.container)){
+        if((await execute('docker',['inspect','--format','{{.State.Running}}',item.container],{stdoutOnly:true})).trim()==='false'){await removeContainer(item.container);retirements=retirements.filter(r=>r!==item);}
+      }
+      const pending=!reloadPending&&(await jobs(home)).find(j=>j.status==='queued');
       if(pending) {
         const refreshPluginBroker=await refreshRequirement(pending);
         if(refreshPluginBroker===null)continue;
+        const preserve=await preserveRuns(pending);
         const apply=async()=>{
           try {
-            const activate=async()=>{const latest=await read(path.join(jobPath(home,pending.id),'job.json'));return perform(home,latest,{stopHost,startHost,signal,refreshPluginBroker});};
-            return pending.target==='main'?await locked(home,activate,{drainInvocations:true,signal}):await activate();
+            const activate=async()=>{const latest=await read(path.join(jobPath(home,pending.id),'job.json'));return perform(home,latest,{stopHost,startHost,retireHost,signal,refreshPluginBroker,preserveRuns:preserve});};
+            return pending.target==='main'&&!preserve?await locked(home,activate,{drainInvocations:true,signal}):await activate();
           } catch(error) {
             // A pre-switch rejection is terminal. Applying jobs keep their journal for recovery.
             const latest=await read(path.join(jobPath(home,pending.id),'job.json'));
             if(latest.status==='queued'){latest.status='failed';latest.error=error.message;await atomic(path.join(jobPath(home,pending.id),'job.json'),latest);}else throw error;
           }
         };
-        const result=await withUpgrade(agent.controlDir,pending,signal,apply,{requiresIdle:refreshPluginBroker});
-        if(result?.status==='completed') {
-          if(pending.target==='main')return;
+        const result=await withUpgrade(agent.controlDir,pending,signal,apply,{requiresIdle:refreshPluginBroker,preserveRuns:preserve});
+        if(['completed','rolled-back'].includes(result?.status)) {
+          if(pending.target==='main')reloadPending=true;
           nextCheck=0;
         }
       }
@@ -166,7 +254,8 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       await sleep(500);
     }
   }finally {
-    clearInterval(beat);await stopHost();await fs.rm(lock,{force:true});
+    clearInterval(beat);await stopHost();
+    for(const item of retirements){if(item.container){await execute('docker',['stop','--time','10',item.container]);await removeContainer(item.container);}else if(item.child.exitCode===null)item.child.kill('SIGTERM');}await fs.rm(lock,{force:true});
     if(!(await jobs(home)).some(j=>j.status==='applying'))await fs.rm(pause,{force:true});
   }
 }