@jc_stack/ez-agents 0.1.0-beta.48 → 0.1.0-beta.49

package/src/index.ts

@@ -27,7 +27,8 @@ import { InboxStore, type IncomingItem } from './inbox.js'
 import { loadConfig, type Config } from './config.js'
 import { ControlStore, telegramOwner, ownerId, ownerEpoch } from './control-state.js'
 import { ApprovalStore } from './approval.js'
-import { startExecutorJob, terminateJob, opencodeDataHome, claudeConfigHome } from './executor.js'
+import { CliUnavailableError, startExecutorJob, terminateJob, opencodeDataHome, claudeConfigHome } from './executor.js'
+import { advances, nextSetup, quotaScope, setupCandidates, startupObserver, unavailableSummary, SetupsUnavailableError, type SetupAttempt } from './setup-fallback.js'
 import { assertScheduledModel } from './model-policy.js'
 import { RunStore, type RunRecord } from './runs.js'
 import { splitTelegramText } from './reply.js'
@@ -38,7 +39,7 @@ import { normalizeReactionEmoji } from './reaction.js'
 import { downloadTelegramFile } from './read-request.js'
 import { createConversationMenu } from './conversation-menu.js'
 import { createAiMenu, mainCommands, mainKeyboard } from './menu.js'
-import { chatPreset, initialPreset, persistedPreset, presetLabel, statusPreset } from './ai.js'
+import { chatPreset, initialPreset, persistedPreset, presetLabel, statusPreset, type AiPreset } from './ai.js'
 import { discoverDefaults } from './client-defaults.js'
 import { runPromptSuffix } from './prompt-suffix.js'
 import { softwareStatus } from './software-status.js'
@@ -236,6 +237,26 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
         await releaseExternal(run)
         return
       }
+      // Shared by a first launch and a fallback launch that cannot start.
+      const startFailed = async (error: unknown) => {
+        if (!run.scheduled && activeTypingTimer) {
+          clearInterval(activeTypingTimer)
+          activeTypingTimer = null
+        }
+        if (run.scheduled) background.delete(run.id)
+        const unavailable = error instanceof SetupsUnavailableError
+        await runs.patch(run.id, { status: 'failed', failureReason: unavailable ? 'setups-unavailable' : 'executor-start', failure: await failureEvidence(config.controlDir, safeError(error)), endedAt: new Date().toISOString() })
+        await releaseExternal(run)
+        console.error('run start failed', run.id, safeError(error))
+        if (bot && !run.application && !run.delivery && run.chatId !== undefined)
+          await sendChat(run.chatId, unavailable ? `Run ${run.id} could not start. ${safeError(error)}` : `Run ${run.id} failed to start. Check the local relay log.`)
+        setImmediate(() => {
+          void runs
+            .nextQueued()
+            .then((next) => next && startJob(next))
+            .catch(console.error)
+        })
+      }
       try {
         const launchStarted = performance.now()
         const started = await runs.patch(run.id, { status: 'running', startedAt: new Date().toISOString() })
@@ -251,121 +272,194 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
           : await control.executionSession(started.execution!)
         const selected = script ? initialPreset('codex') : run.taskId ? (started.execution?.preset.cli === 'codex' ? started.execution.preset : initialPreset('codex')) : started.execution!.preset
         if (run.scheduled && !script) assertScheduledModel(selected.model)
-        const { child, cleanup } = await launch(texts, script ? {
-          workspace: config.workspace, timeoutMs: 0, repairEnabled: config.repairEnabled, runId: started.id,
-          controlDir: config.controlDir, binDir,
-          script: { ...script.ref, scheduleId: script.scheduled.id, scheduleRevision: script.scheduled.revision, dueAt: script.scheduled.dueAt },
-        } : {
-          workspace: config.workspace,
-          timeoutMs: config.executorTimeoutMs,
-          repairEnabled: config.repairEnabled,
-          runId: started.id,
-          controlDir: config.controlDir,
-          binDir,
-          cli: selected.cli,
-          model: selected.model,
-          effort: selected.effort,
-          provider: selected.provider,
-          codexAutoCompactTokens: config.codexAutoCompactTokens,
-          codexSandbox: !run.taskId && selected.cli === 'codex' ? config.codexSandbox : undefined,
-          sessionId: session.nativeSessionId || session.sessionId,
-          isResume: session.hasStarted,
-          promptSuffix: runPromptSuffix(started),
-          taskRun: Boolean(run.taskId),
-          nativeSession: Boolean(run.scheduled),
-          onSession: run.external || run.taskId ? undefined : async (id) => { await runs.patch(run.id,{nativeSessionId:id}); if (!run.scheduled) await control.saveNativeSession(session.sessionId,id) },
-        })
-        const executionStarted = performance.now()
-        // Attach before disk writes: a fast child can close while PID persistence
-        // is pending, and Node drains its remaining pipes during process close.
-        let failureReason = 'executor-exit', errorTail = '', interrupted = false, outputTail = '', timedOut = false
-        // Script output is bounded diagnostics for run inspection, never chat.
-        if (script) child.stdout?.setEncoding('utf8').on('data', (chunk: string) => { outputTail = (outputTail + chunk).slice(-16384) })
-        const scriptTimer = script ? setTimeout(() => { timedOut = true; terminateJob(child) }, script.registration.timeoutSeconds * 1000) : undefined
-        child.once('close', () => clearTimeout(scriptTimer))
-        child.stderr?.setEncoding('utf8').on('data', (chunk: string) => {
-          errorTail = (errorTail + chunk).slice(-16384)
-          if (script) outputTail = (outputTail + chunk).slice(-16384)
-          if (chunk.includes('Host CLI executor is offline')) failureReason = 'host-executor-offline'
-          if (chunk.includes('Host executor client interrupted by')) {
-            failureReason = 'host-executor-transport-interrupted'
-            interrupted = true
+        // A scheduled occurrence may name ordered fallback setups. They share this
+        // run, workspace and concurrency slot; each is tried at most once.
+        const candidates = run.scheduled && !script ? setupCandidates(started.execution!) : [selected]
+        const ordered = candidates.length > 1
+        let attempts: SetupAttempt[] = []
+        const settle = (change: Partial<SetupAttempt>) => {
+          attempts = [...attempts.slice(0, -1), { ...attempts[attempts.length - 1], ...change, endedAt: new Date().toISOString() }]
+        }
+        // Advance only after an eligible pre-work failure of a current, uncancelled
+        // occurrence. Setups in an exhausted quota scope are skipped and recorded.
+        const advance = async (): Promise<AiPreset | undefined> => {
+          let preset: AiPreset | undefined
+          const current = !shuttingDown && !await scheduler.cancelled(run.id) ? (await control.status()).owner : undefined
+          if (current && ownsRun(current, run) && await scheduler.current(run, current)) {
+            const next = nextSetup(candidates, attempts)
+            attempts = [...attempts, ...next.skipped]
+            if (!next.preset) {
+              await runs.patch(run.id, { attempts })
+              throw new SetupsUnavailableError(unavailableSummary(attempts))
+            }
+            preset = next.preset
           }
-          if (chunk.trim()) console.error('executor stderr', started.id, chunk.trim())
-        })
-        console.info('run timing', { run_id: run.id, phase: 'launch',
-          queue_ms: Math.max(0, Date.parse(started.startedAt!) - Date.parse(run.createdAt)),
-          startup_ms: Math.round(executionStarted - launchStarted), resumed: session.hasStarted })
-        if (run.scheduled) background.set(run.id,child)
-        else activeChild = child
-        const finished = new Promise<number | null>((resolve) => {
-          if (child.exitCode !== null || child.signalCode !== null) resolve(child.exitCode)
-          else child.once('close', resolve)
-        })
-        await runs.patch(started.id, { pid: child.pid })
-        console.info('run started', {
-          run_id: started.id,
-          pid: child.pid,
-          cli: script ? 'script' : selected.cli,
-          ...(script ? { script: script.ref.id, script_revision: script.ref.revision } : {}),
-          session: session.sessionId,
-          isResume: session.hasStarted,
-        })
+          await runs.patch(run.id, { attempts })
+          return preset
+        }
+        // Each attempt is a fresh native session; a rejected Claude attempt already claimed its ID.
+        const launchSetup = async (preset: AiPreset): Promise<{ child: ChildProcess; cleanup: () => Promise<void>; preset: AiPreset }> => {
+          for (;;) {
+            const sessionId = attempts.length ? randomUUID() : session.nativeSessionId || session.sessionId
+            if (ordered) {
+              attempts = [...attempts, { preset, quotaScope: quotaScope(preset), outcome: 'running', startedAt: new Date().toISOString() }]
+              await runs.patch(run.id, { attempts })
+            }
+            try {
+              return { ...await launch(texts, script ? {
+                workspace: config.workspace, timeoutMs: 0, repairEnabled: config.repairEnabled, runId: started.id,
+                controlDir: config.controlDir, binDir,
+                script: { ...script.ref, scheduleId: script.scheduled.id, scheduleRevision: script.scheduled.revision, dueAt: script.scheduled.dueAt },
+              } : {
+                workspace: config.workspace,
+                timeoutMs: config.executorTimeoutMs,
+                repairEnabled: config.repairEnabled,
+                runId: started.id,
+                controlDir: config.controlDir,
+                binDir,
+                cli: preset.cli,
+                model: preset.model,
+                effort: preset.effort,
+                provider: preset.provider,
+                authProfile: preset.authProfile,
+                codexAutoCompactTokens: config.codexAutoCompactTokens,
+                codexSandbox: !run.taskId && preset.cli === 'codex' ? config.codexSandbox : undefined,
+                sessionId,
+                isResume: session.hasStarted,
+                promptSuffix: runPromptSuffix(started),
+                taskRun: Boolean(run.taskId),
+                nativeSession: Boolean(run.scheduled),
+                onSession: run.external || run.taskId ? undefined : async (id) => { await runs.patch(run.id,{nativeSessionId:id}); if (!run.scheduled) await control.saveNativeSession(session.sessionId,id) },
+              }), preset }
+            } catch (error) {
+              if (ordered && !(error instanceof CliUnavailableError)) {
+                settle({ outcome: 'failed', category: 'uncertain' })
+                await runs.patch(run.id, { attempts })
+              }
+              if (!ordered || !(error instanceof CliUnavailableError)) throw error
+              settle({ outcome: 'failed', category: 'cli-unavailable' })
+              const next = await advance()
+              if (!next) throw error
+              preset = next
+            }
+          }
+        }
+        const watch = async ({ child, cleanup, preset }: Awaited<ReturnType<typeof launchSetup>>): Promise<void> => {
+          const executionStarted = performance.now()
+          // Attach before disk writes: a fast child can close while PID persistence
+          // is pending, and Node drains its remaining pipes during process close.
+          let failureReason = 'executor-exit', errorTail = '', interrupted = false, outputTail = '', timedOut = false
+          // Script output is bounded diagnostics for run inspection, never chat.
+          if (script) child.stdout?.setEncoding('utf8').on('data', (chunk: string) => { outputTail = (outputTail + chunk).slice(-16384) })
+          // Structured startup events decide whether a failed setup may advance.
+          const startup = ordered ? startupObserver() : undefined
+          if (startup) child.stdout?.setEncoding('utf8').on('data', startup.write)
+          const scriptTimer = script ? setTimeout(() => { timedOut = true; terminateJob(child) }, script.registration.timeoutSeconds * 1000) : undefined
+          child.once('close', () => clearTimeout(scriptTimer))
+          child.stderr?.setEncoding('utf8').on('data', (chunk: string) => {
+            errorTail = (errorTail + chunk).slice(-16384)
+            if (script) outputTail = (outputTail + chunk).slice(-16384)
+            if (chunk.includes('Host CLI executor is offline')) failureReason = 'host-executor-offline'
+            if (chunk.includes('Host executor client interrupted by')) {
+              failureReason = 'host-executor-transport-interrupted'
+              interrupted = true
+            }
+            if (chunk.trim()) console.error('executor stderr', started.id, chunk.trim())
+          })
+          console.info('run timing', { run_id: run.id, phase: 'launch',
+            queue_ms: Math.max(0, Date.parse(started.startedAt!) - Date.parse(run.createdAt)),
+            startup_ms: Math.round(executionStarted - launchStarted), resumed: session.hasStarted })
+          if (run.scheduled) background.set(run.id,child)
+          else activeChild = child
+          const finished = new Promise<number | null>((resolve) => {
+            if (child.exitCode !== null || child.signalCode !== null) resolve(child.exitCode)
+            else child.once('close', resolve)
+          })
+          await runs.patch(started.id, { pid: child.pid })
+          console.info('run started', {
+            run_id: started.id,
+            pid: child.pid,
+            cli: script ? 'script' : preset.cli,
+            ...(script ? { script: script.ref.id, script_revision: script.ref.revision } : {}),
+            ...(ordered ? { setup: attempts.length } : {}),
+            session: session.sessionId,
+            isResume: session.hasStarted,
+          })
 
-        if (!run.scheduled && activeTypingTimer) clearInterval(activeTypingTimer)
-        if (bot && run.chatId !== undefined && !run.application && !run.external && !run.taskId && !run.scheduled) activeTypingTimer = setInterval(() => {
-          if (performance.now() - executionStarted < 30000) void bot.api.sendChatAction(run.chatId!, 'typing').catch(() => {})
-        }, 4000)
+          if (!run.scheduled && activeTypingTimer) clearInterval(activeTypingTimer)
+          if (bot && run.chatId !== undefined && !run.application && !run.external && !run.taskId && !run.scheduled) activeTypingTimer = setInterval(() => {
+            if (performance.now() - executionStarted < 30000) void bot.api.sendChatAction(run.chatId!, 'typing').catch(() => {})
+          }, 4000)
 
-        const completion = finished.then((code) => {
-          console.info('run timing', { run_id: run.id, phase: 'execution',
-            execution_ms: Math.round(performance.now() - executionStarted), exit_code: code })
-          return (async () => {
-            await withStartLock(async () => {
-              try {
-                await cleanup()
-                if (code === 0 && !run.external && !run.taskId && !run.scheduled) await control.markSessionStarted(session.sessionId)
-                const cancelled = !timedOut && (ownerStopped.has(child) || Boolean(run.scheduled && await scheduler.cancelled(run.id)))
-                if (timedOut) failureReason = 'script-timeout'
-                const failed = !cancelled && (code !== 0 || timedOut)
-                await runs.patch(started.id, { status: cancelled ? 'cancelled' : failed ? 'failed' : 'completed', endedAt: new Date().toISOString(), exitCode: code,
-                  ...(script ? { output: redactFailure(safeError(outputTail)), ...(timedOut ? { timedOut } : {}) } : {}),
-                  ...(failed ? { failureReason, interrupted, failure: await failureEvidence(config.controlDir, safeError(timedOut ? `Script exceeded its ${script!.registration.timeoutSeconds}s timeout and was terminated` : errorTail || `Executor exited with ${code === null ? 'a signal' : `code ${code}`}`)) } : {}) })
-                await releaseExternal(run)
-              } catch (error) {
-                await runs.patch(started.id, { status: ownerStopped.has(child) ? 'cancelled' : 'failed', failureReason: 'session-finalization', failure: await failureEvidence(config.controlDir, safeError(error)), endedAt: new Date().toISOString() })
-                console.error('Session completion failed', safeError(error))
-              } finally {
-                if (run.scheduled) background.delete(run.id)
-                else {
-                  activeChild = null
-                  if (activeTypingTimer) clearInterval(activeTypingTimer)
-                  activeTypingTimer = null
+          const free = () => {
+            if (run.scheduled) background.delete(run.id)
+            else {
+              activeChild = null
+              if (activeTypingTimer) clearInterval(activeTypingTimer)
+              activeTypingTimer = null
+            }
+          }
+          const finalize = async (code: number | null, cancelled: boolean, failed: boolean) => {
+            await runs.patch(started.id, { status: cancelled ? 'cancelled' : failed ? 'failed' : 'completed', endedAt: new Date().toISOString(), exitCode: code,
+              ...(script ? { output: redactFailure(safeError(outputTail)), ...(timedOut ? { timedOut } : {}) } : {}),
+              ...(failed ? { failureReason, interrupted, failure: await failureEvidence(config.controlDir, safeError(timedOut ? `Script exceeded its ${script!.registration.timeoutSeconds}s timeout and was terminated` : errorTail || `Executor exited with ${code === null ? 'a signal' : `code ${code}`}`)) } : {}) })
+            await releaseExternal(run)
+          }
+          const completion = finished.then((code) => {
+            console.info('run timing', { run_id: run.id, phase: 'execution',
+              execution_ms: Math.round(performance.now() - executionStarted), exit_code: code })
+            return (async () => {
+              let eligible = false
+              await withStartLock(async () => {
+                try {
+                  await cleanup()
+                  if (code === 0 && !run.external && !run.taskId && !run.scheduled) await control.markSessionStarted(session.sessionId)
+                  const cancelled = !timedOut && (ownerStopped.has(child) || Boolean(run.scheduled && await scheduler.cancelled(run.id)))
+                  if (timedOut) failureReason = 'script-timeout'
+                  const observed = startup?.result()
+                  const rejected = observed?.rejection && !observed.workBegan ? observed.rejection : undefined
+                  // A rejection before work is a failure even when the CLI exits 0.
+                  const failed = !cancelled && (code !== 0 || timedOut || Boolean(rejected))
+                  if (startup) {
+                    settle({ outcome: cancelled ? 'cancelled' : failed ? 'failed' : 'completed', exitCode: code,
+                      ...(failed ? { category: rejected?.category ?? (observed!.workBegan ? 'after-work-began' : 'uncertain') } : {}),
+                      ...(failed && rejected?.resetAt ? { resetAt: rejected.resetAt } : {}) })
+                    await runs.patch(run.id, { attempts })
+                    eligible = failed && !interrupted && advances(rejected)
+                  }
+                  if (!eligible) await finalize(code, cancelled, failed)
+                } catch (error) {
+                  eligible = false
+                  await runs.patch(started.id, { status: ownerStopped.has(child) ? 'cancelled' : 'failed', failureReason: 'session-finalization', failure: await failureEvidence(config.controlDir, safeError(error)), endedAt: new Date().toISOString() })
+                  console.error('Session completion failed', safeError(error))
+                } finally {
+                  // An advancing occurrence keeps its concurrency slot for the next setup.
+                  if (!eligible) free()
                 }
-              }
-            })
-            console.info('run ended', { run_id: started.id, code })
-            const next = await runs.nextQueued()
-            if (next && !shuttingDown) await startJob(next)
-          })().catch((error) => console.error('Run completion failed', error.message))
-        })
-        completions.add(completion)
-        void completion.finally(() => completions.delete(completion))
-      } catch (error) {
-        if (!run.scheduled && activeTypingTimer) {
-          clearInterval(activeTypingTimer)
-          activeTypingTimer = null
+              })
+              console.info('run ended', { run_id: started.id, code, ...(eligible ? { advancing: true } : {}) })
+              // Re-check cancellation and shutdown under the start lock, so a stop
+              // that already snapshotted children never misses the next setup.
+              if (eligible) return withStartLock(async () => {
+                try {
+                  const next = await advance()
+                  if (next) return await watch(await launchSetup(next))
+                  const cancelled = await scheduler.cancelled(run.id)
+                  await finalize(code, cancelled, !cancelled)
+                  free()
+                  setImmediate(() => { void runs.nextQueued().then(queued => { if (queued && !shuttingDown) return startJob(queued) }).catch(console.error) })
+                } catch (error) { await startFailed(error) }
+              })
+              const next = await runs.nextQueued()
+              if (next && !shuttingDown) await startJob(next)
+            })().catch((error) => console.error('Run completion failed', error.message))
+          })
+          completions.add(completion)
+          void completion.finally(() => completions.delete(completion))
         }
-        await runs.patch(run.id, { status: 'failed', failureReason: 'executor-start', failure: await failureEvidence(config.controlDir, safeError(error)), endedAt: new Date().toISOString() })
-        await releaseExternal(run)
-        console.error('run start failed', run.id, safeError(error))
-        if (bot && !run.application && !run.delivery && run.chatId !== undefined) await sendChat(run.chatId, `Run ${run.id} failed to start. Check the local relay log.`)
-        setImmediate(() => {
-          void runs
-            .nextQueued()
-            .then((next) => next && startJob(next))
-            .catch(console.error)
-        })
+        await watch(await launchSetup(selected))
+      } catch (error) {
+        await startFailed(error)
       }
     })
   }

package/src/menu.ts

@@ -4,7 +4,7 @@ import path from 'node:path'
 import { randomBytes } from 'node:crypto'
 import { InlineKeyboard, type Context } from 'grammy'
 import { ControlStore, type ControlGuard } from './control-state.js'
-import { chatPreset, installed, persistedPreset, presetLabel, presetProvider, readModels, validateSelection, type AiPreset, type ModelChoice } from './ai.js'
+import { chatPreset, installed, persistedPreset, presetLabel, presetProvider, readModels, sameChoice, sameEngine, validateSelection, type AiPreset, type ModelChoice } from './ai.js'
 import { discoverDefaults } from './client-defaults.js'
 import { claudeConfigHome } from './executor.js'
 
@@ -25,7 +25,7 @@ export const mainKeyboard = () => new InlineKeyboard()
 const clientLabel = (cli: string) => cli === 'codex-gui' ? 'codex-gui (desktop)' : cli
 
 const matchesModel = (preset: AiPreset, model: ModelChoice) =>
-  model.cli === preset.cli && model.provider === preset.provider && (preset.model === undefined || model.model === preset.model) &&
+  sameEngine(model, preset) && model.provider === preset.provider && (preset.model === undefined || model.model === preset.model) &&
   (preset.effort === undefined || model.efforts.includes(preset.effort))
 
 // Short-lived opaque button IDs: no model names or executable arguments from callbacks.
@@ -66,7 +66,7 @@ export const createAiMenu = (control: ControlStore, cli: string, catalog = readM
     const state = (await control.status()).ai
     if (!state) throw new Error('AI settings not initialized')
     const current = state.presets.find((p) => p.id === state.selectedId)!
-    const fresh = current.cli !== preset.cli || presetProvider(current) !== presetProvider(preset) || Boolean(session && !session.cli)
+    const fresh = !sameEngine(current, preset) || presetProvider(current) !== presetProvider(preset) || Boolean(session && !session.cli)
     if (!await control.selectPreset(preset.id, expectedSession, fresh, guard)) throw new Error('AI binding changed. Refresh available AIs before trying again.')
     return { preset, fresh }
   }
@@ -127,9 +127,9 @@ export const createAiMenu = (control: ControlStore, cli: string, catalog = readM
     const state = (await control.status()).ai
     if (!state) throw new Error('AI settings not initialized')
     const candidate: AiPreset = { id: randomBytes(8).toString('hex'),
-      name: `${model.name}${effort ? ` · ${effort}` : ''}`.slice(0, 80), cli: model.cli, provider:model.provider, model: model.model, effort }
+      name: `${model.name}${effort ? ` · ${effort}` : ''}`.slice(0, 80), cli: model.cli, provider:model.provider, ...(model.authProfile ? {authProfile:model.authProfile} : {}), model: model.model, effort }
     const stored = persistedPreset(candidate)
-    const existing = state.presets.find((preset) => preset.cli === stored.cli && preset.provider === stored.provider && preset.model === stored.model && preset.effort === stored.effort)
+    const existing = state.presets.find((preset) => sameChoice(preset, stored))
     const preset = existing ?? candidate
     await validateSelection(preset, await catalog(), host ? async name => (await catalog()).some(model => model.cli === name) : isInstalled)
     // Private scopes persist their selection on the session, not the owner menu.

package/src/plugin-broker.mjs

@@ -6,7 +6,7 @@ import { createServer } from 'node:net';
 import { randomBytes, randomUUID, createHash } from 'node:crypto';
 import { spawn } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
-import { prepareCommand, registry, run as dockerRun, stewardOwned } from './plugins/manager.mjs';
+import { prepareCommand, registry, run as dockerRun, stewardOwned, publishBrokerPlugins } from './plugins/manager.mjs';
 
 const MAX_FRAME = 4 * 1024 * 1024;
 const MAX_ARGS = 100;
@@ -165,15 +165,7 @@ const writeReceipt = async (binding, receipt) => {
 // Installed plugin versions for relay status. Rewritten on broker start and
 // after every management call; the relay reads it without Docker access.
 // Failures stay local so a status snapshot never breaks plugin operations.
-const refreshBrokerPlugins = async binding => {
-  const r = await registry(binding.home);
-  const plugins = Object.values(r.plugins).map(record => ({ id: record.manifest.id, version: record.manifest.version }));
-  const file = path.join(binding.controlDir, 'plugin-broker-plugins.json');
-  const temporary = `${file}.${randomUUID()}.tmp`;
-  await fs.writeFile(temporary, JSON.stringify({ version: 1, at: new Date().toISOString(), plugins }) + '\n', { mode: 0o600, flag: 'wx' });
-  await fs.rename(temporary, file);
-  await stewardOwned(file);
-};
+const refreshBrokerPlugins = binding => publishBrokerPlugins(binding.home,binding.controlDir);
 
 const response = (socket, value) => {
   if (socket.destroyed || !socket.writable) return;
@@ -255,6 +247,7 @@ const invokePlugin = async (binding, request, signal, scheduledPreflight = false
     const command = await prepareCommand(binding.home, request.alias, request.args, {
       revision: request.revision,
       invocation: true,
+      signal,
       environment: { EZ_CONTROL_DIR: binding.controlDir, ...(scheduledPreflight ? {} : {EZ_RUN_ID: request.runId}), EZ_DELIVERY_SOCKET: process.env.EZ_DELIVERY_SOCKET ?? deliverySocketPath(binding.controlDir) },
     });
     try {
@@ -360,7 +353,7 @@ export async function servePluginBroker(binding, signal = new AbortController().
               const selected=recordForAlias(await registry(binding.home),check.alias);
               const result=await invokePlugin(binding,{alias:check.alias,args:check.args,revision:selected.record.revision,runId:`preflight_${request.scheduleId}`,id:randomUUID()},abort.signal,true);
               if(!result.ok || result.code!==0) throw Error('Scheduled plugin preflight unavailable');
-              values.push(JSON.parse(result.stdout));
+              try { values.push(JSON.parse(result.stdout)); } catch { throw Error('Scheduled plugin preflight returned invalid output'); }
             }
             response(socket,{version:1,id:request.id,ok:true,result:combinePreflight(values)});
             return;

package/src/plugins/manager.mjs

@@ -78,7 +78,8 @@ async function activeInvocations(home) {
   const directory=invocationDirectory(home),files=await fs.readdir(directory).catch(error=>{if(error.code==='ENOENT')return [];throw error;});
   for(const file of files) {
     if(!/^[0-9a-f-]{36}\.json$/.test(file))throw Error('Invalid command invocation lease; inspect before recovery');
-    const lease=await json(path.join(directory,file));
+    const lease=await json(path.join(directory,file)).catch(error=>{if(error.code==='ENOENT')return null;throw error;});
+    if(!lease)continue;
     if(!Number.isSafeInteger(lease.pid)||lease.pid<1||typeof lease.container!=='string'||!lease.container)throw Error('Invalid command invocation lease; inspect before recovery');
     try {process.kill(lease.pid,0);return true;} catch(error) {if(error.code!=='ESRCH'&&error.code!=='EPERM')throw error;}
     const state=await run(['container','inspect',lease.container],{capture:true});
@@ -94,21 +95,31 @@ async function invocationLease(home,container) {
   await stewardOwned(file);
   return async()=>{await fs.rm(file,{force:true});};
 }
-export async function locked(home, fn, {allowInvocations=false, waitMs=0}={}) {
+export async function locked(home, fn, {allowInvocations=false, waitMs=0, upgradeWaitMs=0, drainInvocations=false, signal}={}) {
   const lock = path.join(home,'registry.lock');
   let handle;
-  const deadline=Date.now()+waitMs;
+  const started=Date.now();
   while(!handle) {
+    signal?.throwIfAborted();
     try { handle=await fs.open(lock,'wx',0o600); }
     catch(error) {
       if(error.code!=='EEXIST')throw error;
-      if(Date.now()>=deadline)throw Error('Registry busy; inspect registry.lock before recovering an interrupted manager');
+      const owner=await json(lock).catch(()=>null);
+      const limit=owner?.kind==='plugin-upgrade'&&upgradeWaitMs?upgradeWaitMs:waitMs;
+      if(Date.now()-started>=limit)throw Error('Registry busy; inspect registry.lock before recovering an interrupted manager');
       await new Promise(resolve=>setTimeout(resolve,25));
     }
   }
   try {
-    await handle.writeFile(JSON.stringify({pid:process.pid}));
-    if(!allowInvocations&&await activeInvocations(home))throw Error('Plugin commands are active; retry the registry or lifecycle change after they finish');
+    await handle.writeFile(JSON.stringify({pid:process.pid,...(drainInvocations?{kind:'plugin-upgrade'}:{})}));
+    // Keep admission closed while already-started calls release their leases.
+    // This is a lifecycle barrier, not an agent-turn drain or a command retry.
+    while(!allowInvocations&&await activeInvocations(home)) {
+      if(!drainInvocations)throw Error('Plugin commands are active; retry the registry or lifecycle change after they finish');
+      signal?.throwIfAborted();
+      await new Promise(resolve=>setTimeout(resolve,25));
+    }
+    signal?.throwIfAborted();
     return await fn();
   }
   finally { await handle.close(); await fs.rm(lock); }
@@ -474,8 +485,16 @@ export async function registry(home) {
   for(const [alias,plugin] of Object.entries(r.commands)) if(!r.plugins[plugin]?.deployment?.commands?.[alias]) throw Error('Corrupt command registry');
   return r;
 }
+// Relay status reads this generated inventory without Docker/registry access.
+export async function publishBrokerPlugins(home,controlDir) {
+  const r=await registry(home);
+  const plugins=Object.values(r.plugins).map(record=>({id:record.manifest.id,version:record.manifest.version}));
+  const file=path.join(controlDir,'plugin-broker-plugins.json');
+  await atomic(file,{version:1,at:new Date().toISOString(),plugins});
+  await stewardOwned(file);
+}
 // The lock protects admission and compose refresh, never a persistent connection.
-export async function prepareCommand(home,alias,args,{revision,exclude,publish,invocation=false,environment=process.env}={}) {
+export async function prepareCommand(home,alias,args,{revision,exclude,publish,invocation=false,environment=process.env,signal}={}) {
   strings(args);
   return locked(home,async()=>{
     const config=await json(path.join(home,'config.json')),r=await registry(home);
@@ -505,7 +524,7 @@ export async function prepareCommand(home,alias,args,{revision,exclude,publish,i
     const invocationRelease=invocation?await invocationLease(home,container):undefined;
     const release=(invocationRelease||contextFile)?(async()=>{try{if(contextFile)await fs.rm(contextFile,{force:true});}finally{await invocationRelease?.();}}):undefined;
     return {container,plugin:record.manifest.id,revision:record.revision,contextFile,argv:[...composeArgs(record),...(contextFile?['--file',contextFile]:[]),'run','--rm','--no-deps','-T','--name',container,...(publish?['--publish',publish]:[]),'--entrypoint',binding.argv[0],binding.service,...binding.argv.slice(1),...record.manifest.commands[alias].args,...args,...(binding.suffix||[])],release};
-  },{allowInvocations:true,waitMs:5000});
+  },{allowInvocations:true,waitMs:5000,upgradeWaitMs:invocation?300000:0,signal});
 }
 export async function init(home,workspace,catalogFile,hostConfig,standalone=false) {
   if(standalone && hostConfig) throw Error('Standalone setup cannot bind a relay host config');

package/src/runs.ts

@@ -6,7 +6,8 @@ import { normalizeReactionEmoji } from './reaction.js'
 import { validScheduledOrigin, type ScheduledOrigin } from './scheduler.js'
 import type { IncomingItem } from './inbox.js'
 import { assertId } from './identity.js'
-import { isExecutionChoice, type ExecutionChoice } from './ai.js'
+import { isExecutionChoice, type AiPreset, type ExecutionChoice } from './ai.js'
+import type { SetupAttempt } from './setup-fallback.js'
 import type { ScriptRunRef } from './scripts.js'
 import {authorizeDeliveryContext,currentDeliveryOwner,type DeliveryContext} from './delivery-context.mjs'
 
@@ -32,6 +33,8 @@ export type RunRecord = {
   pid?: number
   blockReason?: string
   execution?: ExecutionChoice
+  // Ordered task setups tried for this occurrence; the last one is active.
+  attempts?: SetupAttempt[]
   replyOnly?: boolean
   exitCode?: number | null
   failureReason?: string
@@ -129,6 +132,14 @@ const pruneTerminals = (controlDir: string): void => {
   }
 }
 
+// The setup the current attempt runs with: the latest recorded attempt, else the
+// saved choice. Only a setup saved on the task itself is ever admitted.
+export const activeSetup = (run: Pick<RunRecord, 'attempts' | 'execution'>): AiPreset | undefined => {
+  if (!run.execution) return undefined
+  const preset = [...run.attempts ?? []].reverse().find(attempt => attempt.outcome !== 'skipped')?.preset ?? run.execution.preset
+  return [run.execution.preset, ...run.execution.fallbacks ?? []].find(candidate => JSON.stringify(candidate) === JSON.stringify(preset))
+}
+
 export const newRunId = (): string => `r_${Date.now().toString(36)}_${randomBytes(3).toString('hex')}`
 
 export class RunStore {
@@ -198,7 +209,7 @@ export class RunStore {
 
   async patch(
     id: string,
-    change: Partial<Pick<RunRecord, 'status' | 'startedAt' | 'endedAt' | 'pid' | 'nativeSessionId' | 'interrupted' | 'blockReason' | 'backendSubmitted' | 'replyOnly' | 'exitCode' | 'failureReason' | 'failure' | 'failureReview' | 'externalReleased' | 'output' | 'timedOut'>>,
+    change: Partial<Pick<RunRecord, 'status' | 'startedAt' | 'endedAt' | 'pid' | 'nativeSessionId' | 'interrupted' | 'blockReason' | 'backendSubmitted' | 'replyOnly' | 'exitCode' | 'failureReason' | 'failure' | 'failureReview' | 'externalReleased' | 'output' | 'timedOut' | 'attempts'>>,
   ): Promise<RunRecord> {
     const store = runsFor(this.controlDir)
     const run = store.get(id)