@jc_stack/ez-agents 0.1.0-beta.50 → 0.1.0-beta.52

package/src/host-executor.ts

@@ -3,7 +3,7 @@ import { Tasks } from './tasks.js'
 import { ControlStore } from './control-state.js'
 import { authorizeRun, readRun } from './delivery-socket.js'
 import { runPromptSuffix } from './prompt-suffix.js'
-import { mkdir, readFile, writeFile, readdir, rename, rm, appendFile, realpath } from 'node:fs/promises'
+import { mkdir, readFile, writeFile, readdir, rename, rm, appendFile, realpath, utimes, stat } from 'node:fs/promises'
 import path from 'node:path'
 import { isHostRunId } from './host-executor-protocol.js'
 import { fileURLToPath } from 'node:url'
@@ -42,13 +42,16 @@ export const refreshAgentCatalog = async (agent: HostBinding,
 const processStart = async (pid:number) => (await processSnapshot()).get(pid)?.birth
 
 export const serveHostExecutor = async (installation: HostInstallation, signal: AbortSignal, launch = startExecutorJob,
-  readCatalog?: (agent: HostBinding) => Promise<ModelChoice[]>) => {
+  readCatalog?: (agent: HostBinding) => Promise<ModelChoice[]>, retire?: AbortSignal) => {
   if (installation.isolation !== undefined) parseIsolationClass(installation.isolation)
-  if (installation.isolation === 'isolated') throw new Error('Isolated agents run the native CLI in the relay; do not start host transport')
+  if (installation.isolation === 'isolated' && process.env.EZ_CONTAINER_EXECUTOR !== '1') throw new Error('Isolated agents run the native CLI in the relay; do not start host transport')
   resolveExecutor(installation.cli)
   if (new Set(installation.agents.map(a=>a.workspace)).size !== installation.agents.length ||
       new Set(installation.agents.map(a=>a.controlDir)).size !== installation.agents.length)
     throw new Error('Each agent requires a separate workspace and control directory')
+  const containerId = process.env.EZ_CONTAINER_EXECUTOR === '1' ? process.env.HOSTNAME : undefined
+  const hostStarted = await processStart(process.pid)
+  let retiring = false
   const active = new Map<string, ChildProcess>()
   // A fallback attempt reuses its run ID; claim it only after the prior attempt's cleanup.
   const claimed = new Set<string>()
@@ -74,9 +77,29 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
   const freshCatalog = readCatalog ?? catalog
   const beat = async (agent: HostBinding) => {
     const directory = path.join(agent.controlDir, 'host-executor')
-    await writeFile(path.join(directory, 'heartbeat.tmp'), JSON.stringify({ at: Date.now(), pid: process.pid, version: packageVersion, platform: process.platform, arch: process.arch, plugins: await installedPluginVersions(agent.toolsHome) }), { mode: 0o600 })
+    await writeFile(path.join(directory, 'heartbeat.tmp'), JSON.stringify({ at: Date.now(), pid: process.pid, version: packageVersion, platform: process.platform, arch: process.arch, ...(process.env.EZ_CONTAINER_EXECUTOR === '1' ? {containerExecutor:true,containerId} : {}), plugins: await installedPluginVersions(agent.toolsHome) }), { mode: 0o600 })
     await rename(path.join(directory, 'heartbeat.tmp'), path.join(directory, 'heartbeat.json'))
   }
+  const recoverClaims = async (agent: HostBinding) => {
+    const directory=path.join(agent.controlDir,'host-executor')
+    for (const file of await readdir(directory)) if (file.endsWith('.running.json') && isHostRunId(file.slice(0,-13))) {
+      const base=path.join(directory,file.slice(0,-13))
+      if (claimed.has(base)) continue
+      let prior: {pid?:number;started?:string;hostPid?:number;hostStarted?:string;containerId?:string} | undefined
+      try { prior=JSON.parse(await readFile(base+'.process.json','utf8')) } catch(error) { if ((error as NodeJS.ErrnoException).code!=='ENOENT') throw error }
+      if(prior?.containerId && prior.containerId!==containerId){
+        const touched=await stat(base+'.events').then(s=>s.mtimeMs,()=>0)
+        if(Date.now()-touched<15000)continue
+      }
+      if (!prior?.containerId && prior?.hostPid && prior.hostStarted && await processStart(prior.hostPid)===prior.hostStarted) continue
+      if (!prior?.containerId && prior?.pid && await processStart(prior.pid)) {
+        if (!prior.hostPid) throw Error('Previous host CLI is still running; stop it before recovery')
+        if (prior.started && await processStart(prior.pid)===prior.started) terminateJob({pid:prior.pid,exitCode:null,signalCode:null} as ChildProcess)
+      }
+      await appendFile(base+'.events',JSON.stringify({stream:'exit',code:1})+'\n',{mode:0o600})
+      await rm(base+'.process.json',{force:true});await rm(base+'.running.json',{force:true})
+    }
+  }
   try {
     for (const agent of installation.agents) {
       if (![agent.workspace,agent.controlDir,agent.binDir].every(path.isAbsolute)) throw new Error('Host bindings require absolute paths')
@@ -89,7 +112,7 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
         if (!Array.isArray(agent.additionalWorkspaces) || agent.additionalWorkspaces.some(workspace=>typeof workspace!=='string' || !path.isAbsolute(workspace))) throw new Error('Additional workspaces require absolute paths')
         additionalWorkspaces.set(agent,await Promise.all(agent.additionalWorkspaces.map(workspace=>realpath(workspace))))
       }
-      if (agent.toolsHome) {
+      if (agent.toolsHome && !containerId) {
         if (!path.isAbsolute(agent.toolsHome)) throw new Error('Plugin registry binding requires an absolute path')
         const config=JSON.parse(await readFile(path.join(agent.toolsHome,'config.json'),'utf8'))
         if (config.schemaVersion!==1 || config.workspace!==await realpath(agent.workspace)) throw new Error('Plugin registry belongs to another workspace')
@@ -99,6 +122,7 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
       const lock=path.join(directory,'worker.lock')
       try {
         const prior=JSON.parse(await readFile(lock,'utf8'))
+        if(prior.containerId&&prior.containerId!==containerId)throw Error('Previous isolated worker still owns the claim lock')
         const currentStart=await processStart(prior.pid)
         if (!prior.started || !currentStart || prior.started===currentStart) {
           try { process.kill(prior.pid,0); throw new Error('Host executor already running') }
@@ -107,34 +131,29 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
         await rm(lock)
       }
       catch(error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error }
-      await writeFile(lock,JSON.stringify({pid:process.pid,started:await processStart(process.pid)}),{mode:0o600,flag:'wx'})
+      await writeFile(lock,JSON.stringify({pid:process.pid,started:await processStart(process.pid),...(containerId ? {containerId} : {})}),{mode:0o600,flag:'wx'})
       locks.push(lock)
       const models = await freshCatalog(agent)
       await new ControlStore(agent.controlDir, 900000).normalizeProviderBindings(models)
       const modelsFile=path.join(directory,'models.json')
       await writeFile(modelsFile+'.tmp',JSON.stringify(models),{mode:0o600});await rename(modelsFile+'.tmp',modelsFile)
-      // A host crash is terminal for a claimed job. Never replay an action.
-      for (const file of await readdir(directory)) if (file.endsWith('.running.json')) {
-        const base=path.join(directory,file.slice(0,-13))
-        try {
-          const prior=JSON.parse(await readFile(base+'.process.json','utf8'))
-          try { process.kill(prior.pid,0); throw new Error('Previous host CLI is still running; stop it before recovery') }
-          catch(error) { if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error }
-        } catch(error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error }
-        await rm(base+'.process.json',{force:true})
-        await appendFile(base+'.events',JSON.stringify({stream:'exit',code:1})+'\n',{mode:0o600})
-        await rm(path.join(directory,file))
-      }
-      if (agent.toolsHome) await (await import('./plugins/workspace-lease.mjs')).recoverNativeLease(agent.toolsHome)
+      await recoverClaims(agent)
+      if (agent.toolsHome && !containerId) await (await import('./plugins/workspace-lease.mjs')).recoverNativeLease(agent.toolsHome)
     }
     let catalogAt=Date.now()
     const catalogSeen=new Map<HostBinding,string>()
     while (!signal.aborted) {
       const parent=Number(process.env.EZ_HOST_SUPERVISOR_PID)
-      if(parent) { try { process.kill(parent,0) } catch { break } }
+      if(parent && !retiring) { try { process.kill(parent,0) } catch { break } }
+      if (retire?.aborted && !retiring) {
+        retiring=true
+        await catalogRefresh
+        for (const lock of locks.splice(0)) await rm(lock,{force:true})
+      }
+      if (retiring && !tasks.size) break
       // Keep transport liveness independent of native catalog I/O.
-      for (const agent of installation.agents) await beat(agent)
-      if(!catalogRefresh && Date.now()-catalogAt>30000){
+      if (!retiring) for (const agent of installation.agents) await beat(agent)
+      if(!retiring && !catalogRefresh && Date.now()-catalogAt>30000){
         catalogAt=Date.now()
         catalogRefresh=(async()=>{
           for(const agent of installation.agents){
@@ -149,6 +168,8 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
       }
       for (const agent of installation.agents) {
         const directory=path.join(agent.controlDir,'host-executor')
+        if (retiring) continue
+        await recoverClaims(agent)
         for (const file of await readdir(directory)) {
           if (!file.endsWith('.request.json') || !isHostRunId(file.slice(0,-13))) continue
           const id=file.slice(0,-13)
@@ -168,9 +189,9 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
           let workspace = activeWorkspaces.get(agent)
           if (!scheduled && workspace && [...workspace.lanes.values()].includes(false)) continue
           if (!workspace) {
-            const release = agent.toolsHome
+            const release = agent.toolsHome && !containerId
               ? await (await import('./plugins/workspace-lease.mjs')).workspaceLease(agent.toolsHome,{kind:'native',runId:id}) : undefined
-            if (agent.toolsHome && !release) continue
+            if (agent.toolsHome && !containerId && !release) continue
             workspace = { lanes: new Map(), release }
             activeWorkspaces.set(agent, workspace)
           }
@@ -184,7 +205,11 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
               await workspace.release?.()
             }
           }
-          try { await rename(base+'.request.json',base+'.running.json') }
+          try {
+            await writeFile(base+'.process.json',JSON.stringify({hostPid:process.pid,hostStarted,...(containerId ? {containerId} : {})}),{mode:0o600})
+            await appendFile(base+'.events','',{mode:0o600})
+            await rename(base+'.request.json',base+'.running.json')
+          }
           catch (error) { claimed.delete(base); await releaseWorkspace(); throw error }
           const task=(async()=>{
             let job: Awaited<ReturnType<typeof startExecutorJob>> | undefined
@@ -205,7 +230,7 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
               // Script occurrences come from the authorized run record, never the request.
               if (run?.script) {
                 if (!run.scheduled) throw new Error('Script runs require a scheduled occurrence')
-                const options:ExecutorOptions={workspace:agent.workspace,controlDir:agent.controlDir,binDir:agent.binDir,toolsHome:agent.toolsHome,sharedWorkspace,additionalWorkspaces:additionalWorkspaces.get(agent),
+                const options:ExecutorOptions={catalogTransport:containerId?'local':'host',workspace:agent.workspace,controlDir:agent.controlDir,binDir:agent.binDir,toolsHome:agent.toolsHome,sharedWorkspace,additionalWorkspaces:additionalWorkspaces.get(agent),
                   runId:path.basename(base),timeoutMs:0,repairEnabled:opts.repairEnabled,
                   script:{...run.script,scheduleId:run.scheduled.id,scheduleRevision:run.scheduled.revision,dueAt:run.scheduled.dueAt}}
                 job=await launch(request.texts,options)
@@ -237,14 +262,18 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
                 // transient native-client probe failure must not reject a model
                 // that the host just advertised to the relay and application.
                 if (cli !== installation.cli || opts.authProfile !== undefined) await validateSelection({id:'selected',name:'Selected model',cli,provider:opts.provider,authProfile:opts.authProfile,model,effort:opts.effort},JSON.parse(await readFile(path.join(directory,'models.json'),'utf8')))
-                const options:ExecutorOptions={workspace:agent.workspace,controlDir:agent.controlDir,binDir:agent.binDir,toolsHome:agent.toolsHome,sharedWorkspace,additionalWorkspaces:additionalWorkspaces.get(agent),cli,
+                const options:ExecutorOptions={catalogTransport:containerId?'local':'host',workspace:agent.workspace,controlDir:agent.controlDir,binDir:agent.binDir,toolsHome:agent.toolsHome,sharedWorkspace,additionalWorkspaces:additionalWorkspaces.get(agent),cli,
                   runId:path.basename(base),timeoutMs:0,repairEnabled:opts.repairEnabled,
                   sessionId:opts.sessionId,isResume:opts.isResume,model,effort:opts.effort,provider:opts.provider,authProfile:opts.authProfile,codexAutoCompactTokens:opts.codexAutoCompactTokens,codexProvider:provider,
                   promptSuffix:runPromptSuffix(run),taskRun:Boolean(run?.taskId),nativeSession:Boolean(run?.scheduled)}
                 job=await launch(request.texts,options)
               }
               active.set(base,job.child)
-              await writeFile(base+'.process.json',JSON.stringify({pid:job.child.pid}),{mode:0o600})
+              // Observe completion before process metadata I/O: a fast CLI may
+              // already have exited when launch returns, or while that I/O waits.
+              const closed=job.child.exitCode!==null||job.child.signalCode!==null
+                ? Promise.resolve(job.child.exitCode??1)
+                : new Promise<number>(resolve=>job!.child.once('close',code=>resolve(code??1)))
               if(signal.aborted)terminateJob(job.child)
               // Scheduled Claude streams every event; forward only the startup summary.
               const startup = observeClaude ? startupObserver() : undefined
@@ -255,8 +284,9 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
                 if (startup.summary() !== summary) { summary = startup.summary(); emit({stream:'stdout',text:summary+'\n'}) }
               })
               job.child.stderr?.on('data',chunk=>emit({stream:'stderr',text:chunk.toString()}))
-              cancellation=setInterval(()=>{void readFile(base+'.cancel').then(()=>terminateJob(job!.child)).catch(()=>{})},250)
-              const code=await new Promise<number>(resolve=>job!.child.once('close',code=>resolve(code??1)))
+              await writeFile(base+'.process.json',JSON.stringify({pid:job.child.pid,started:job.child.pid ? await processStart(job.child.pid) : undefined,hostPid:process.pid,hostStarted,...(containerId ? {containerId} : {})}),{mode:0o600})
+              cancellation=setInterval(()=>{void readFile(base+'.cancel').then(()=>terminateJob(job!.child)).catch(()=>{}); const now=new Date();void utimes(base+'.events',now,now).catch(()=>{})},250)
+              const code=await closed
               if (cancellation) clearInterval(cancellation)
               await job.cleanup()
               job = undefined
@@ -290,8 +320,9 @@ export const serveHostExecutor = async (installation: HostInstallation, signal:
 }
 
 if (process.argv[1] && path.resolve(process.argv[1])===fileURLToPath(import.meta.url)) {
-  const abort=new AbortController()
+  const abort=new AbortController(),retire=new AbortController()
+  process.once('SIGUSR2',()=>retire.abort())
   for(const signal of ['SIGTERM','SIGINT'] as const)process.once(signal,()=>abort.abort())
   const config=JSON.parse(await readFile(process.argv[2],'utf8'))
-  await serveHostExecutor(config,abort.signal)
+  await serveHostExecutor(config,abort.signal,undefined,undefined,retire.signal)
 }

package/src/index.ts

@@ -27,7 +27,7 @@ import { InboxStore, type IncomingItem } from './inbox.js'
 import { loadConfig, type Config } from './config.js'
 import { ControlStore, telegramOwner, ownerId, ownerEpoch, type SessionState } from './control-state.js'
 import { ApprovalStore } from './approval.js'
-import { CliUnavailableError, startExecutorJob, attachHostJob, terminateJob, opencodeDataHome, claudeConfigHome } from './executor.js'
+import { CliUnavailableError, startExecutorJob, attachHostJob, terminateJob, opencodeDataHome, claudeConfigHome, usesFileTransport } from './executor.js'
 import { attemptFailure, continuationLine, nextSetup, quotaScope, setupCandidates, startupObserver, unavailableSummary, SetupsUnavailableError, type SetupAttempt } from './setup-fallback.js'
 import { assertScheduledModel } from './model-policy.js'
 import { RunStore, saveRunHandoff, takeRunHandoff, type RunRecord } from './runs.js'
@@ -96,7 +96,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   let activeBackend = false
   const ownerStopped = new WeakSet<ChildProcess>()
   // Host-transport executors outlive this relay; a graceful stop hands them off.
-  const hostTransport = process.env.EZ_EXECUTOR_TRANSPORT === 'host' && !config.channelBackendUrl
+  const hostTransport = async () => !config.channelBackendUrl && await usesFileTransport(config.controlDir)
   const childRuns = new WeakMap<ChildProcess, string>()
   const handedOff = new WeakSet<ChildProcess>()
   let activeChild: ChildProcess | null = null
@@ -668,8 +668,8 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   }
 
   let outboxWork: Promise<void> | undefined
-  const drainOutbox = (onlyRunId?: string): Promise<void> => {
-    if (shuttingDown) return outboxWork ?? Promise.resolve()
+  const drainOutbox = (onlyRunId?: string, stopping = false): Promise<void> => {
+    if (shuttingDown && !stopping) return outboxWork ?? Promise.resolve()
     return outboxWork ?? (outboxWork = (async () => {
       for (const item of await runs.pendingOutbox()) {
         if (onlyRunId && item.runId !== onlyRunId) continue
@@ -773,6 +773,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   // children, host executor, CLIs) enqueue and query the memory ledger
   // through the shared handler in delivery-socket.ts. The pump below stays
   // the single delivery executor; the socket only admits work into it.
+  let drainingHandoff=false
   let deliveryServer: { socketPath: string; stop: () => Promise<void> } | null = null
   const handleDeliveryOp = createLedgerHandler(config.controlDir, {
     wake: () => { void drainOutbox() },
@@ -835,7 +836,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     const owner = (await control.status()).owner
     if (!owner) return 'Scheduled tasks\n\nNo paired owner.'
     // This intentionally uses the reader that does not create a schedules directory.
-    const active = await scheduler.listActiveReadOnly(await runs.list())
+    const active = await scheduler.listTasksReadOnly(await runs.list())
     return { owner, active, text: scheduledTasksText(active, owner) }
   }
   const replyScheduledTasks = async (ctx: Context, number?: number) => {
@@ -892,6 +893,18 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
         : []),
     ].join('\n')
   }
+  const stopActiveWork = async (): Promise<boolean> => {
+    const children: [string | undefined, ChildProcess][] = [...background]
+    if (activeChild) children.unshift([undefined, activeChild])
+    let requested = true
+    for (const [id, child] of children) {
+      if (!await terminateJob(child)) { requested = false; continue }
+      ownerStopped.add(child)
+      if (id) await scheduler.cancel(id)
+    }
+    return requested
+  }
+
   const cancelPending = async () => {
     let count = 0
     await withStartLock(async () => {
@@ -1030,16 +1043,15 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     if (text === '/stop') {
       const running = await runs.running(false)
       if ((running && running.pid) || background.size) {
-        try {
-          if (activeChild) { ownerStopped.add(activeChild); terminateJob(activeChild) }
-          for (const [id,child] of background) { await scheduler.cancel(id); ownerStopped.add(child); terminateJob(child) }
-        } catch {}
-        if (activeTypingTimer) {
+        let requested = false
+        try { requested = await stopActiveWork() } catch {}
+        if (requested && activeTypingTimer) {
           clearInterval(activeTypingTimer)
           activeTypingTimer = null
         }
         await ctx.reply(
-          '🛑 Stop requested for active work. Queued work remains and will run next. /cancel clears it.',
+          requested ? '🛑 Stop requested for active work. Queued work remains and will run next. /cancel clears it.'
+            : 'A stop request failed; some work remains active.',
         )
       } else {
         await ctx.reply('No active run in progress.')
@@ -1262,11 +1274,12 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
       } else if (action === 'stop') {
         const running = await runs.running(false)
         if ((running && running.pid) || background.size) {
-          if (activeChild) { ownerStopped.add(activeChild); terminateJob(activeChild) }
-          for (const [id,child] of background) { await scheduler.cancel(id); ownerStopped.add(child); terminateJob(child) }
-          await ctx.answerCallbackQuery({ text: 'Run stopped' })
+          let requested = false
+          try { requested = await stopActiveWork() } catch {}
+          await ctx.answerCallbackQuery({ text: requested ? 'Stop requested' : 'Stop request failed' })
           await ctx.reply(
-            '🛑 Stop requested for active work. Queued work remains and will run next. /cancel clears it.',
+            requested ? '🛑 Stop requested for active work. Queued work remains and will run next. /cancel clears it.'
+              : 'A stop request failed; some work remains active.',
           )
         } else {
           await ctx.answerCallbackQuery({ text: 'No run active' })
@@ -1288,7 +1301,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   // unknown outcome is recorded interrupted, which holds its schedule.
   const adoptHandoff = async (): Promise<void> => {
     const handed = await takeRunHandoff(config.controlDir)
-    if (!hostTransport) { if (handed.length) console.error('Ignoring run handoff without host transport', handed.length); return }
+    if (!await hostTransport()) { if (handed.length) console.error('Ignoring run handoff without host transport', handed.length); return }
     // Admit every record before the first attach, so no trigger races a hold.
     const admitted = (await Promise.all(handed.map(run => runs.adopt(run).catch(() => null)))).filter((run): run is RunRecord => run !== null)
     for (const run of admitted) await withStartLock(async () => {
@@ -1353,21 +1366,25 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     shuttingDown = true
     runtimeStarted = false
     pollingAbort.abort()
+    let handedRecords:RunRecord[]=[]
     // Hand off running host work before any wait can outlast the stop grace
     // period. The lock lets an in-flight launch register; later ones see shutdown.
-    if (hostTransport) await withStartLock(async () => {
+    if (await hostTransport()) await withStartLock(async () => {
       const handoff = new Map<ChildProcess, RunRecord>()
       for (const child of [...(activeChild ? [activeChild] : []), ...background.values()]) {
         const run = childRuns.has(child) ? await runs.get(childRuns.get(child)!) : null
         if (run?.status === 'running' && !ownerStopped.has(child) && child.exitCode === null && child.signalCode === null) handoff.set(child, run)
       }
       if (!handoff.size) return
-      try { await saveRunHandoff(config.controlDir, [...handoff.values()]) }
+      try { handedRecords=[...handoff.values()];await saveRunHandoff(config.controlDir, handedRecords) }
       catch (error) { console.error('Run handoff failed; stopping host work', safeError(error)); return }
       // SIGKILL leaves no cancel marker, so the host keeps each executor running.
       for (const child of handoff.keys()) { handedOff.add(child); child.kill('SIGKILL') }
     })
-    await deliveryServer?.stop().catch(() => {})
+    drainingHandoff=handedRecords.length>0
+    if(deliveryServer)await deliveryServer.stop()
+    else if(drainingHandoff){await outboxWork?.catch(() => {});await drainOutbox(undefined,true)}
+    if(handedRecords.length)await saveRunHandoff(config.controlDir,handedRecords)
     deliveryServer = null
     await applicationChannel.stop()
     wakePollRetry?.()
@@ -1402,6 +1419,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
         throw new Error(`Another relay owns ${config.controlDir}; stop it before starting a second one`)
       const deliveryTcpPort = Number(process.env.EZ_DELIVERY_TCP_PORT ?? '')
       deliveryServer = await serveDeliverySocket(config.controlDir, handleDeliveryOp, {
+        beforeClose:async()=>{if(drainingHandoff){await outboxWork?.catch(()=>{});await drainOutbox(undefined,true)}},
         ...(Number.isSafeInteger(deliveryTcpPort) && deliveryTcpPort >= 1024 && deliveryTcpPort <= 65535 ? { tcpPort: deliveryTcpPort } : {}),
       })
       // Reconcile, then adopt handed-off runs, before any intake can launch work.

package/src/plugins/workspace-lease.mjs

@@ -23,7 +23,7 @@ export async function recoverNativeLease(home) {
 export async function workspaceLease(home,owner={kind:'plugin'}) {
   const file=path.join(home,'workspace-writer.lock'),temp=file+'.'+randomUUID()+'.tmp';
   await fs.writeFile(temp,JSON.stringify({...owner,pid:process.pid}),{flag:'wx',mode:0o600});
-  try {await fs.link(temp,file);}catch(error){if(error.code!=='EEXIST')throw error;let prior;try {prior=await leaseOwner(file);}catch(error){if(error.code==='ENOENT')return undefined;throw error;}if(!prior.alive)throw Error('Stale workspace-writer.lock: inspect stopped owner and command containers, then restart for recovery');return undefined;}
+  try {await fs.link(temp,file);}catch(error){if(error.code!=='EEXIST')throw error;let prior;try {prior=await leaseOwner(file);}catch(error){if(error.code==='ENOENT')return undefined;throw error;}if(!prior.alive)throw Error('Stale workspace-writer.lock: inspect stopped owner and command containers, then restart for recovery');if(owner.kind==='native'&&prior.owner.kind==='native')return async()=>{};return undefined;}
   finally {await fs.rm(temp);}
   return async()=>{await fs.rm(file);};
 }

package/src/runs.ts

@@ -162,7 +162,9 @@ const handedRun = (value: unknown): value is RunRecord => {
 
 export const saveRunHandoff = async (controlDir: string, records: RunRecord[]): Promise<void> => {
   const file = handoffFile(controlDir), temporary = `${file}.${process.pid}.tmp`
-  await writeFile(temporary, JSON.stringify({ version: 1, writtenAt: new Date().toISOString(), runs: records }), { mode: 0o600 })
+  const ids=new Set(records.map(run=>run.id))
+  const receipts=[...outboxFor(controlDir).values()].filter(item=>item.runId&&ids.has(item.runId)&&(item.state==='sent'||item.state==='failed'))
+  await writeFile(temporary, JSON.stringify({ version: 1, writtenAt: new Date().toISOString(), runs: records, receipts }), { mode: 0o600 })
   await rename(temporary, file)
 }
 
@@ -176,12 +178,16 @@ export const takeRunHandoff = async (controlDir: string, now = Date.now()): Prom
     try { raw = await readFile(handoffFile(controlDir), 'utf8') }
     catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error }
     await rm(handoffFile(controlDir))
-    const value = JSON.parse(raw) as { version?: unknown; writtenAt?: unknown; runs?: unknown }
+    const value = JSON.parse(raw) as { version?: unknown; writtenAt?: unknown; runs?: unknown; receipts?: unknown }
     const age = now - Date.parse(typeof value.writtenAt === 'string' ? value.writtenAt : '')
     if (value.version !== 1 || !(age >= -60_000 && age <= HANDOFF_MAX_AGE_MS)) throw new Error('Invalid or stale run handoff')
     if (!Array.isArray(value.runs) || value.runs.length > 100) throw new Error('Invalid run handoff')
     const valid = value.runs.filter(handedRun)
     if (valid.length !== value.runs.length) console.error('Ignoring invalid handed-off runs', value.runs.length - valid.length)
+    const ids=new Set(valid.map(run=>run.id))
+    if(Array.isArray(value.receipts)&&value.receipts.length<=MAX_TERMINAL_OUTBOX){
+      for(const item of value.receipts as StoredOutboxItem[])if(item&&typeof item.id==='string'&&/^[a-zA-Z0-9_-]+$/.test(item.id)&&item.runId&&ids.has(item.runId)&&(item.state==='sent'||item.state==='failed'))outboxFor(controlDir).set(item.id,item)
+    }
     return [...new Map(valid.map(run => [run.id, run])).values()]
   } catch (error) {
     console.error('Ignoring unreadable run handoff', error instanceof Error ? error.message : 'Unknown error')

package/src/schedule-cli.ts

@@ -83,7 +83,8 @@ The run records each attempt, setup, failure category and any reported reset tim
 Trigger runs an existing task once with its saved instructions, AI and delivery binding in a fresh session. Reuse the request key after an uncertain result; the regular schedule is unchanged.
 Creates a scheduled task. Instructions are text, never shell commands.
 Use --now to run once. Run completion is not delivery proof.
-Edit replaces the full schedule. Pause/remove affect future work; cancel stops a particular run.
+Edit replaces the full schedule but keeps its paused/enabled state; only pause/resume toggle it.
+Pause/remove affect future work; cancel stops a particular run.
 Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/weekday OR semantics.`);return}
   const config=loadControlConfig(), control=new ControlStore(config.controlDir,config.pairingTtlMs)
   const owner=(await control.status()).owner
@@ -195,13 +196,15 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
       v.cron ? {cron:v.cron,timezone:v.timezone!,start,until:v.until} : {everySeconds:Number(v['every-seconds']),start,until:v.until}
     if(v['clear-preflight'] && v['preflight-file']) throw Error('Choose one preflight operation')
     const previousSchedule = action === 'edit' ? await scheduler.get(id!) : undefined
+    // Edit keeps the paused/enabled state; only pause/resume toggle it.
+    const enabled = previousSchedule?.enabled ?? true
     const preflight = v['clear-preflight'] ? undefined : v['preflight-file'] ? JSON.parse(await readFile(v['preflight-file'],'utf8')) : previousSchedule?.preflight
     const origin = caller?.application ?? caller?.delivery
     const delivery = previousSchedule ? previousSchedule.delivery : (origin ? {bindingId:origin.bindingId,scope:origin.scope} : undefined)
     if (!delivery && !owner.telegramChatId) throw new Error('Create the schedule from an authenticated channel turn to bind its reply destination')
     if (v.script) {
       result=await show(await scheduler.save({id:id || 's_'+randomUUID(),name:v.name || v.script,preflight,
-        originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:'',when:v.when as 'unreviewed-failures' | undefined,trigger,enabled:true,owner,
+        originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:'',when:v.when as 'unreviewed-failures' | undefined,trigger,enabled,owner,
         script:{id:v.script,args:v.arg ?? []}},action==='create'))
       console.log(JSON.stringify(result,null,2));return
     }
@@ -216,7 +219,7 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
       if (setup.authProfile !== undefined) await provisionedHome(config.controlDir, setup.cli, setup.authProfile)
     result=await show(await scheduler.save({id:id || 's_'+randomUUID(),name:v.name || 'Task',
       preflight,
-      originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:v.text || await readFile(v['text-file']!,'utf8'),when:v.when as 'unreviewed-failures' | undefined,trigger,enabled:true,owner,
+      originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:v.text || await readFile(v['text-file']!,'utf8'),when:v.when as 'unreviewed-failures' | undefined,trigger,enabled,owner,
       execution:{sessionId:previous?.sessionId || randomUUID(),preset,...(fallbacks ? {fallbacks} : {})}},action==='create'))
   }else{
     if(!id)throw new Error('ID required')

package/src/scheduled-tasks.ts

@@ -2,7 +2,7 @@ import { executionDefaults } from './model-policy.js'
 import { presetLabel, type AiPreset } from './ai.js'
 import { AUTH_PROFILE_CLIS } from './auth-profile.js'
 import type { Owner } from './control-state.js'
-import type { Schedule, ActiveSchedule } from './scheduler.js'
+import type { Schedule, ScheduledTaskView } from './scheduler.js'
 
 const ownsSchedule = (owner: Owner, schedule: Schedule) =>
   schedule.owner.telegramUserId === owner.telegramUserId &&
@@ -24,7 +24,7 @@ const setupLabel = (preset: AiPreset) => {
 }
 
 // Script schedules have no AI preset; show what core actually invokes.
-const engineLabel = (schedule: ActiveSchedule) => schedule.script
+const engineLabel = (schedule: ScheduledTaskView) => schedule.script
   ? `Script · ${schedule.script.id} · ${schedule.scriptRevision ? `rev ${schedule.scriptRevision.slice(0, 8)}` : 'not registered'}`
   : [schedule.execution!.preset, ...schedule.execution!.fallbacks ?? []].map(setupLabel).join(' → ')
 
@@ -67,28 +67,30 @@ const preview = (schedule: Schedule) => {
   return chars.length > 140 ? chars.slice(0,139).join('')+'…' : sentence
 }
 
-const currentState = (schedule: ActiveSchedule) =>
-  schedule.runState === 'running' ? '▶ Running' : schedule.runState === 'queued' ? '◌ Queued' : '● Active'
+const currentState = (schedule: ScheduledTaskView) =>
+  [schedule.runState === 'running' ? '▶ Running' : schedule.runState === 'queued' ? '◌ Queued' : undefined,
+    !schedule.enabled ? '⏸ Paused' : undefined].filter(Boolean).join(' · ') || '● Active'
 
-const lastRun = (schedule: ActiveSchedule) => {
+const lastRun = (schedule: ScheduledTaskView) => {
   if (!schedule.lastRun) return '— Not run yet'
   const status = schedule.lastRun.status === 'completed' ? '✓ Completed' : schedule.lastRun.status === 'failed' ? '✕ Failed' : '⊘ Cancelled'
   return `${status} · ${time(schedule.lastRun.at)}${schedule.lastRun.currentRevision ? '' : ' · previous version'}`
 }
 
-export const ownedScheduledTasks = (schedules: ActiveSchedule[], owner: Owner) =>
+export const ownedScheduledTasks = (schedules: ScheduledTaskView[], owner: Owner) =>
   schedules.filter((schedule) => ownsSchedule(owner, schedule))
     .sort((a, b) => a.name.localeCompare(b.name))
 
-export const scheduledTasksText = (schedules: ActiveSchedule[], owner: Owner) => {
+export const scheduledTasksText = (schedules: ScheduledTaskView[], owner: Owner) => {
   const owned = ownedScheduledTasks(schedules, owner)
-  if (!owned.length) return '📅 Scheduled tasks\n\nNo active scheduled tasks.'
-  return [`📅 Scheduled tasks · ${owned.length} active`, 'Next/last times UTC. Use /tasks 1 for task details.', ...owned.map((schedule, index) =>
+  if (!owned.length) return '📅 Scheduled tasks\n\nNo scheduled tasks.'
+  const paused = owned.filter(schedule => !schedule.enabled).length
+  return [`📅 Scheduled tasks · ${owned.length - paused} active · ${paused} paused`, 'Next/last times UTC. Use /tasks 1 for task details.', ...owned.map((schedule, index) =>
     `${index + 1} · ${schedule.name}\n  ${currentState(schedule)}\n  Schedule · ${frequency(schedule)}\n  Next · ${schedule.nextAt === null ? '—' : time(schedule.nextAt)}\n  Last · ${lastRun(schedule)}\n  Execution · ${engineLabel(schedule)}\n  ${preview(schedule)}`
   )].join('\n\n')
 }
 
-export const scheduledTaskDetailText = (schedule: ActiveSchedule, number: number, total?: number) => {
+export const scheduledTaskDetailText = (schedule: ScheduledTaskView, number: number, total?: number) => {
   const instructions = schedule.script ? preview(schedule) : schedule.text.trim()
   const capped = instructions.length > 2000 ? instructions.slice(0, 2000) + '… (truncated; full text lives in the workspace)' : instructions
   return [

package/src/scheduler.ts

@@ -29,7 +29,7 @@ export type ScheduledRunSummary = {
   at: string
   currentRevision: boolean
 }
-export type ActiveSchedule = Schedule & { nextAt: number | null; runState?: 'queued' | 'running'; lastRun?: ScheduledRunSummary; scriptRevision?: string }
+export type ScheduledTaskView = Schedule & { nextAt: number | null; runState?: 'queued' | 'running'; lastRun?: ScheduledRunSummary; scriptRevision?: string }
 export type ScheduledOrigin = { id: string; revision: string; dueAt: string; pairedAt: string; originRunId?: string }
 export const validScheduledOrigin = (v: unknown): v is ScheduledOrigin => {
   const s = v as ScheduledOrigin
@@ -102,10 +102,9 @@ export class Scheduler {
       return nextOccurrence(s.trigger,-1)
     }
   }
-  async listActiveReadOnly(runs: RunRecord[]): Promise<ActiveSchedule[]> {
-    const active: ActiveSchedule[] = []
+  async listTasksReadOnly(runs: RunRecord[]): Promise<ScheduledTaskView[]> {
+    const active: ScheduledTaskView[] = []
     for (const s of await this.listReadOnly()) {
-      if (!s.enabled) continue
       const current = runs.filter(r => r.scheduled?.id === s.id && r.scheduled.revision === s.revision && ownsRun(s.owner,r))
       const history = runs.filter(r => r.scheduled?.id === s.id && r.scheduled.pairedAt === s.owner.pairedAt && ownsRun(s.owner,r))
         .filter((r): r is RunRecord & { status: ScheduledRunSummary['status'] } =>
@@ -113,11 +112,11 @@ export class Scheduler {
         .sort((a,b) => (b.endedAt ?? b.startedAt ?? b.createdAt).localeCompare(a.endedAt ?? a.startedAt ?? a.createdAt))
       const last = history[0]
       const runState = current.some(r => r.status === 'running') ? 'running' : current.some(r => r.status === 'queued') ? 'queued' : undefined
-      if (!runState && current.some(r => holdsSchedule(s,r))) continue
+      if (s.enabled && !runState && current.some(r => holdsSchedule(s,r))) continue
       try {
-        const nextAt = await this.pendingOccurrence(s)
+        const nextAt = s.enabled ? await this.pendingOccurrence(s) : null
         const scriptRevision = s.script ? await new Scripts(this.controlDir).get(s.script.id).then(r => r.revision, () => undefined) : undefined
-        if (runState || nextAt !== null) active.push({...s,nextAt,runState,...(scriptRevision ? {scriptRevision} : {}),...(last ? {
+        if (!s.enabled || runState || nextAt !== null) active.push({...s,nextAt,runState,...(scriptRevision ? {scriptRevision} : {}),...(last ? {
           lastRun: {status:last.status,at:last.endedAt ?? last.startedAt ?? last.createdAt,currentRevision:last.scheduled!.revision===s.revision},
         } : {})})
       }