@jc_stack/ez-agents 0.1.0-beta.48 → 0.1.0-beta.49

package/src/schedule-cli.ts

@@ -8,18 +8,35 @@ import { ControlStore, sameOwner } from './control-state.js'
 import { ApplicationBindings } from './application-channel.js'
 import type { RunRecord } from './runs.js'
 import { callDeliverySocket, socketPathFor } from './delivery-socket.js'
-import { initialPreset, isPreset } from './ai.js'
+import { initialPreset, isPreset, presetLabel, MAX_FALLBACKS, type AiPreset } from './ai.js'
 import { executionOverrides } from './model-policy.js'
 import { holdsSchedule, Scheduler, executionType } from './scheduler.js'
 import { Scripts, fileSha256, workspaceEntry, DEFAULT_SCRIPT_TIMEOUT_SECONDS } from './scripts.js'
 import { executorEnvironment } from './executor.js'
+import { provisionedHome } from './auth-profile.js'
 import { ownsRun } from './identity.js'
 import { type Trigger } from './schedule-time.js'
 
+// `cli=claude,model=sonnet,effort=medium` names one fallback setup.
+const fallbackSetup = (spec: string, index: number): AiPreset => {
+  const fields = Object.fromEntries(spec.split(',').map(part => {
+    const at = part.indexOf('=')
+    if (at < 1) throw new Error('Fallback uses cli=EXECUTOR,model=MODEL[,effort=EFFORT][,provider=ID]')
+    return [part.slice(0, at), part.slice(at + 1)]
+  }))
+  if (Object.keys(fields).some(key => !['cli','model','effort','provider'].includes(key)) || !fields.cli || !fields.model)
+    throw new Error('Fallback uses cli=EXECUTOR,model=MODEL[,effort=EFFORT][,provider=ID]')
+  const base = initialPreset(fields.cli)
+  const preset = executionOverrides(base.cli, {...base, ...(fields.provider ? {provider: fields.provider} : {})}, fields.model, fields.effort)
+  const named = {...preset, id: `fallback-${index + 1}`, name: presetLabel(preset).slice(0, 80)}
+  if (!isPreset(named)) throw new Error('Invalid fallback setup')
+  return named
+}
+
 async function main() {
   const { values:v, positionals:[action='list',id,scriptId] } = parseArgs({allowPositionals:true,options:{
     script:{type:'string'}, arg:{type:'string',multiple:true}, file:{type:'string'}, interpreter:{type:'string'}, 'timeout-seconds':{type:'string'},
-    cli:{type:'string'}, model:{type:'string'}, effort:{type:'string'},
+    cli:{type:'string'}, 'auth-profile':{type:'string'}, model:{type:'string'}, effort:{type:'string'}, fallback:{type:'string',multiple:true}, 'clear-fallbacks':{type:'boolean'},
     all:{type:'boolean'}, limit:{type:'string'}, offset:{type:'string'}, expected:{type:'string'}, key:{type:'string'}, when:{type:'string'}, status:{type:'string'}, diagnosis:{type:'string'}, recovery:{type:'string'}, outcome:{type:'string'}, 'failed-at':{type:'string'},
     'preflight-file':{type:'string'},'clear-preflight':{type:'boolean'},name:{type:'string'}, text:{type:'string'}, 'text-file':{type:'string'}, at:{type:'string'}, now:{type:'boolean'},
     cron:{type:'string'}, timezone:{type:'string'}, 'every-seconds':{type:'string'}, start:{type:'string'}, until:{type:'string'}, help:{type:'boolean'},
@@ -31,12 +48,14 @@ async function main() {
   review RUN_ID --failed-at ISO --status resolved|attention --diagnosis TEXT --recovery TEXT --outcome TEXT
   create [ID] | edit ID --name NAME (--text TEXT | --text-file FILE | --script SCRIPT_ID [--arg=VALUE ...])
     --now | --at ISO_WITH_OFFSET | --every-seconds N | --cron 'MIN HOUR DAY MONTH WEEKDAY' --timezone IANA
-    [--cli EXECUTOR] [--model MODEL] [--effort <native-effort>]
+    [--cli EXECUTOR] [--auth-profile NAME] [--model MODEL] [--effort <native-effort>]
+    [--fallback cli=EXECUTOR,model=MODEL[,effort=EFFORT][,provider=ID] ... | --clear-fallbacks]
     [--preflight-file FILE | --clear-preflight]
     [--start ISO_WITH_OFFSET] [--until ISO_WITH_OFFSET] [--when unreviewed-failures]
   script list | show SCRIPT_ID | remove SCRIPT_ID
   script register|update SCRIPT_ID --file WORKSPACE_PATH --interpreter COMMAND [--arg=VALUE ...] [--timeout-seconds N]
-Execution types: agent (prompt, engine, model, effort) or script (registered script ID and saved arguments).
+Execution types: agent (prompt, engine, auth profile, model, effort) or script (registered script ID and saved arguments).
+--auth-profile runs a Codex/Claude task under that provisioned login (ezenciel-agents-ai profiles); --cli alone uses the default login.
 A script registration references an entry point inside the agent workspace and records its SHA-256. Core invokes
 the installed interpreter with the entry point and arguments directly, without a shell or a model. Changed entry-point
 bytes refuse to run until an explicit script update. The hash covers that file only, not imported dependencies.
@@ -52,6 +71,10 @@ Failures default to unreviewed owner runs. Review records a diagnosis; it never
 A conditional review schedule consumes no model run when there are no unreviewed failures.
 New tasks capture the selected engine settings; edit preserves existing settings unless overridden.
 Every task requires a concrete saved model. Supply --model if the selected settings have none; existing tasks never fall back to chat or client defaults.
+Fallbacks (at most ${MAX_FALLBACKS}, in order) run only when the previous setup's CLI is unavailable or its provider
+rejected the turn before any work began. Access denial, unknown errors, cancellation and failure after work began stop
+the chain; nothing is replayed. A quota rejection skips later setups of the same client or provider (quota scope),
+even with different credentials. The run records each attempt, setup, failure category and any reported reset time.
 Trigger runs an existing task once with its saved instructions, AI and delivery binding in a fresh session. Reuse the request key after an uncertain result; the regular schedule is unchanged.
 Creates a scheduled task. Instructions are text, never shell commands.
 Use --now to run once. Run completion is not delivery proof.
@@ -159,7 +182,8 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
     if(action==='create' && id && (await scheduler.list()).some(s=>s.id===id))throw new Error('Schedule exists; use edit')
     if([v.now,v.at,v.cron,v['every-seconds']].filter(Boolean).length!==1)throw new Error('Choose exactly one trigger')
     if([v.text,v['text-file'],v.script].filter(Boolean).length!==1)throw new Error('Choose --text, --text-file or --script')
-    if(v.script && (v.cli || v.model || v.effort))throw new Error('Script schedules run no model; --cli, --model and --effort do not apply')
+    if(v.script && (v.cli || v['auth-profile'] || v.model || v.effort || v.fallback || v['clear-fallbacks']))throw new Error('Script schedules run no model; --cli, --auth-profile, --model, --effort and --fallback do not apply')
+    if(v.fallback && v['clear-fallbacks'])throw new Error('Choose --fallback or --clear-fallbacks')
     if(v.arg && !v.script)throw new Error('--arg applies to script schedules only')
     const start=v.start || new Date(Date.now()+1000).toISOString()
     const trigger:Trigger=v.now ? {at:new Date(Date.now()+1000).toISOString()} : v.at ? {at:v.at} :
@@ -180,12 +204,14 @@ Cron uses numeric five-field syntax, lists/ranges/steps, and traditional day/wee
     const state = await control.status()
     const selected = state.ai?.presets.find(p => p.id === state.ai!.selectedId)
     const base = v.cli ? initialPreset(v.cli) : previous?.preset || selected || initialPreset(process.env.EZ_EXECUTOR_CLI || 'codex')
-    const preset = executionOverrides(base.cli, base, v.model, v.effort)
+    const preset = executionOverrides(base.cli, v['auth-profile'] === undefined ? base : {...base, authProfile:v['auth-profile']}, v.model, v.effort)
     if (!isPreset(preset)) throw new Error('Invalid task AI selection')
+    const fallbacks = v['clear-fallbacks'] ? undefined : v.fallback ? v.fallback.map(fallbackSetup) : previous?.fallbacks
+    if (preset.authProfile !== undefined) await provisionedHome(config.controlDir, preset.cli, preset.authProfile)
     result=await show(await scheduler.save({id:id || 's_'+randomUUID(),name:v.name || 'Task',
       preflight,
       originRunId:previousSchedule?.originRunId ?? caller?.scheduled?.originRunId ?? caller?.id,delivery,text:v.text || await readFile(v['text-file']!,'utf8'),when:v.when as 'unreviewed-failures' | undefined,trigger,enabled:true,owner,
-      execution:{sessionId:previous?.sessionId || randomUUID(),preset}},action==='create'))
+      execution:{sessionId:previous?.sessionId || randomUUID(),preset,...(fallbacks ? {fallbacks} : {})}},action==='create'))
   }else{
     if(!id)throw new Error('ID required')
     if(action==='cancel'){

package/src/scheduler.ts

@@ -8,7 +8,7 @@ import { type Owner, sameOwner, validOwner, ownerId, ownerEpoch } from './contro
 import { validApplicationOrigin } from './application-origin.js'
 import { ApplicationBindings } from './application-channel.js'
 import { assertId, ownsRun } from './identity.js'
-import { type ExecutionChoice, isExecutionChoice, persistedPreset } from './ai.js'
+import { type AiPreset, type ExecutionChoice, isExecutionChoice, persistedPreset } from './ai.js'
 import { type Trigger, validateTrigger, nextOccurrence } from './schedule-time.js'
 import { RunStore, type RunRecord } from './runs.js'
 import { Scripts, assertScriptArgs, assertScriptId, type ScriptRunRef } from './scripts.js'
@@ -139,8 +139,13 @@ export class Scheduler {
       if (!input.execution) throw new Error('Schedule needs name, text and an AI selection')
       const execution: ExecutionChoice = {...input.execution, preset: persistedPreset(input.execution.preset)}
       if (!input.name || !input.text?.trim() || !isExecutionChoice(execution)) throw new Error('Schedule needs name, text and an AI selection')
-      assertScheduledModel(execution.preset.model)
-      assertEffort(execution.preset.effort, execution.preset.model, execution.preset.cli)
+      const setups = [execution.preset, ...execution.fallbacks ?? []]
+      for (const preset of setups) {
+        assertScheduledModel(preset.model)
+        assertEffort(preset.effort, preset.model, preset.cli)
+      }
+      const key = (p: AiPreset) => JSON.stringify([p.cli, p.provider, p.model, p.effort])
+      if (new Set(setups.map(key)).size !== setups.length) throw new Error('Task setups must be distinct')
       s = {...input, execution, trigger:validateTrigger(input.trigger),version:1,revision:randomUUID()}
     }
     const now = Date.now()
@@ -272,7 +277,7 @@ export class Scheduler {
             preflight=checked.receipt
             if(!checked.admitted) {await atomic(cursor,{next:future,preflight});continue}
             const current=await this.get(s.id);if(!current.enabled || current.revision!==s.revision)continue
-          } catch {const prior=await readFile(cursor,'utf8').then(v=>JSON.parse(v)).catch(e=>{if(e.code==='ENOENT')return {};throw e});await atomic(cursor,{next:future,preflight:{...prior.preflight,state:'unavailable',observedAt:new Date().toISOString()}});continue}
+          } catch (error) {console.error('Schedule preflight unavailable',s.id,error instanceof Error ? error.message : 'Unknown error');const prior=await readFile(cursor,'utf8').then(v=>JSON.parse(v)).catch(e=>{if(e.code==='ENOENT')return {};throw e});await atomic(cursor,{next:future,preflight:{...prior.preflight,state:'unavailable',observedAt:new Date().toISOString()}});continue}
         }
         const dueAt = new Date(next).toISOString()
         await runs.create({id:scheduledRunId(s,next),

package/src/setup-fallback.ts

@@ -0,0 +1,122 @@
+import { presetLabel, presetProvider, type AiPreset, type ExecutionChoice } from './ai.js'
+
+// Ordered task setups for one scheduled occurrence. Core advances only on typed
+// evidence that the engine never started work: a missing CLI or a provider
+// rejection before any model output or tool item. Anything else stops the chain.
+export type StartupCategory = 'cli-unavailable' | 'quota' | 'provider-rejected' | 'access-denied'
+export type StartupRejection = { category: StartupCategory; resetAt?: string }
+export type AttemptCategory = StartupCategory | 'after-work-began' | 'uncertain' | 'same-quota-scope'
+export type SetupAttempt = {
+  preset: AiPreset
+  quotaScope: string
+  outcome: 'running' | 'completed' | 'failed' | 'cancelled' | 'skipped'
+  category?: AttemptCategory
+  resetAt?: string
+  exitCode?: number | null
+  startedAt?: string
+  endedAt?: string
+}
+
+const categories: StartupCategory[] = ['cli-unavailable', 'quota', 'provider-rejected', 'access-denied']
+export const advances = (rejection?: StartupRejection): boolean => Boolean(rejection && rejection.category !== 'access-denied')
+
+export const setupCandidates = (execution: ExecutionChoice): AiPreset[] => [execution.preset, ...(execution.fallbacks ?? [])]
+
+// Allowance is metered per provider account, not per credential file. Setups on
+// the same native client share a scope; OpenCode/Pi share their upstream
+// provider. A different credential alone never proves independent allowance.
+export const quotaScope = (preset: AiPreset): string => {
+  if (['opencode', 'pi'].includes(preset.cli)) {
+    const provider = preset.cli === 'pi' ? preset.model?.split('/')[0] : presetProvider(preset)
+    return provider ? `provider:${provider}` : preset.cli
+  }
+  return preset.cli === 'codex-gui' ? 'codex' : preset.cli
+}
+
+// The next untried setup, skipping those whose quota scope already rejected
+// this occurrence. Each candidate is tried at most once.
+export const nextSetup = (candidates: AiPreset[], attempts: SetupAttempt[]): { skipped: SetupAttempt[]; preset?: AiPreset } => {
+  const exhausted = new Set(attempts.filter(a => a.category === 'quota').map(a => a.quotaScope))
+  const skipped: SetupAttempt[] = []
+  for (const preset of candidates.slice(attempts.length)) {
+    const scope = quotaScope(preset)
+    if (!exhausted.has(scope)) return { skipped, preset }
+    const at = new Date().toISOString()
+    skipped.push({ preset, quotaScope: scope, outcome: 'skipped', category: 'same-quota-scope', startedAt: at, endedAt: at })
+  }
+  return { skipped }
+}
+
+export class SetupsUnavailableError extends Error {}
+
+export const unavailableSummary = (attempts: SetupAttempt[]): string =>
+  `Every task setup was unavailable: ${attempts.map(a => `${presetLabel(a.preset)} (${a.category}${a.resetAt ? `, resets ${a.resetAt}` : ''})`).join('; ')}`
+
+const epochSeconds = (value: unknown): string | undefined =>
+  typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? new Date(value * 1000).toISOString() : undefined
+
+const claudeErrors: Record<string, StartupCategory> = {
+  rate_limit: 'quota', billing_error: 'quota',
+  authentication_failed: 'access-denied', oauth_org_not_allowed: 'access-denied', account_on_hold: 'access-denied',
+  verification_required: 'access-denied', cloud_credential_error: 'access-denied',
+  overloaded: 'provider-rejected', server_error: 'provider-rejected', model_not_found: 'provider-rejected', invalid_request: 'provider-rejected',
+}
+
+// Codex app-server `codexErrorInfo` of a failed turn.
+export const codexRejection = (info: unknown): StartupCategory | undefined => {
+  if (info === 'usageLimitExceeded' || info === 'rateLimitExceeded') return 'quota'
+  if (info === 'unauthorized') return 'access-denied'
+  if (['serverOverloaded', 'internalServerError', 'badRequest', 'flexUnavailable'].includes(info as string)) return 'provider-rejected'
+  if (!info || typeof info !== 'object') return undefined
+  const status = (Object.values(info)[0] as { httpStatusCode?: unknown } | undefined)?.httpStatusCode
+  if (typeof status !== 'number') return undefined
+  return status === 429 ? 'quota' : [401, 403].includes(status) ? 'access-denied' : [400, 404].includes(status) || status >= 500 ? 'provider-rejected' : undefined
+}
+
+// The latest reset among exhausted Codex rate-limit windows.
+export const codexResetAt = (limits: any): string | undefined => {
+  const resets = [limits?.primary, limits?.secondary].filter(w => w && w.usedPercent >= 100 && typeof w.resetsAt === 'number').map(w => w.resetsAt as number)
+  return resets.length ? epochSeconds(Math.max(...resets)) : undefined
+}
+
+export const startupLine = (state: { workBegan: boolean; rejection?: StartupRejection }): string =>
+  JSON.stringify({ type: 'ez.startup', workBegan: state.workBegan, ...(state.rejection ? { rejection: state.rejection } : {}) })
+
+// Reads structured stdout only: Ez transport summaries (`ez.startup`) and
+// Claude stream-json envelopes. Model text is never inspected. Any event that
+// is not known pre-work metadata counts as work, so ambiguity never advances.
+export const startupObserver = () => {
+  let buffer = '', workBegan = false, rejection: StartupRejection | undefined
+  const line = (text: string) => {
+    if (!text.trim()) return
+    let event: any
+    try { event = JSON.parse(text) } catch { workBegan = true; return }
+    if (event?.type === 'ez.startup') {
+      if (event.workBegan === true) workBegan = true
+      const r = event.rejection
+      if (r && categories.includes(r.category) && (r.resetAt === undefined || Number.isFinite(Date.parse(r.resetAt))))
+        rejection = { category: r.category, ...(r.resetAt ? { resetAt: r.resetAt } : {}) }
+      return
+    }
+    if (event?.type === 'thread.started' || event?.type === 'result' ||
+        (event?.type === 'system' && event.subtype === 'init')) return
+    if (event?.type === 'rate_limit_event') {
+      const info = event.rate_limit_info
+      if (info?.status === 'rejected') rejection = { category: 'quota', ...(epochSeconds(info.resetsAt) ? { resetAt: epochSeconds(info.resetsAt) } : {}) }
+      return
+    }
+    const category = event?.type === 'assistant' && typeof event.error === 'string' ? claudeErrors[event.error] : undefined
+    if (category) rejection = { category, ...(category === 'quota' && rejection?.resetAt ? { resetAt: rejection.resetAt } : {}) }
+    else workBegan = true
+  }
+  return {
+    write: (chunk: string) => {
+      buffer += chunk
+      const lines = buffer.split('\n')
+      buffer = lines.pop() ?? ''
+      for (const text of lines) line(text)
+    },
+    result: () => { line(buffer); buffer = ''; return { workBegan, rejection } },
+    summary: () => startupLine({ workBegan, rejection }),
+  }
+}

package/src/task-executor.ts

@@ -8,6 +8,7 @@ import { promisify } from 'node:util'
 import { readRun } from './delivery-socket.js'
 import {ApplicationBindings} from './application-channel.js'
 import { Tasks } from './tasks.js'
+import { provisionedHome } from './auth-profile.js'
 import { executorEnvironment, executorInvocation, terminateJob, type ExecutorOptions } from './executor.js'
 
 // This adapter is deliberately version-pinned: a new native tool default needs
@@ -58,8 +59,8 @@ export async function startTaskExecutor(options: ExecutorOptions) {
     const catalogInvocation = executorInvocation('codex', ['debug', 'models', '--bundled'])
     const catalog = await promisify(execFile)(catalogInvocation.command, catalogInvocation.args, { env: environment, maxBuffer: 4 * 1024 * 1024 })
     await writeFile(join(temporary, 'models.json'), JSON.stringify(taskModelCatalog(JSON.parse(catalog.stdout))), { mode: 0o600 })
-    const auth = process.env.EZ_ISOLATION === 'isolated'
-      ? join(options.controlDir, 'cli', 'codex', 'auth.json')
+    const auth = process.env.EZ_ISOLATION === 'isolated' || options.authProfile !== undefined
+      ? join(await provisionedHome(options.controlDir, 'codex', options.authProfile), 'auth.json')
       : join(homedir(), '.codex', 'auth.json')
     await symlink(auth, join(home, 'auth.json'))
     const broker = [process.execPath, '--import', fileURLToPath(new URL('../node_modules/tsx/dist/loader.mjs', import.meta.url)),

package/src/updates/runtime.mjs

@@ -7,7 +7,7 @@ import { pathToFileURL } from 'node:url';
 import { spawn } from 'node:child_process';
 import { parseEnv } from 'node:util';
 import { tmpdir } from 'node:os';
-import { atomic, compose, snapshot, checkFolders } from '../plugins/manager.mjs';
+import { atomic, compose, snapshot, checkFolders, locked, publishBrokerPlugins } from '../plugins/manager.mjs';
 import { read, state, eligibility, jobPath, cleanupStaleBackups } from './control.mjs';
 import { bindUpdates } from './binding.mjs';
 import { extract, digest } from './artifact.mjs';
@@ -82,24 +82,44 @@ export async function refreshBroker(config, run=execute, image) {
     const current=(await run('docker',[...args,'ps','-q','plugin-broker'],{stdoutOnly:true,timeout:15000})).trim();
     if(!/^[a-f0-9]{12,64}$/i.test(current)||(await run('docker',['inspect','--format','{{.Image}}',current],{stdoutOnly:true,timeout:15000})).trim()!==image)
       throw Error('Recreated broker image does not match the intended installed image');
-    const hostConfig=config.hostConfig||path.join(config.deploymentDir,'host-executor.json');
-    const home=path.dirname(hostConfig)===config.deploymentDir
-      ? (await read(hostConfig)).agents?.find(agent=>agent.workspace===config.workspace)?.toolsHome : undefined;
-    if(typeof home!=='string'||!path.isAbsolute(home))throw Error('Missing broker registry binding');
-    const files=[hostConfig,path.join(home,'registry.json'),path.join(home,'config.json')];
-    const hashes=await Promise.all(files.map(async file=>digest(await fs.readFile(file))));
-    // Supply trusted read-only code so rollback can check an older image that
-    // predates this diagnostic. Its validators still come from that image.
-    const diagnostic=await fs.readFile(new URL('../../docker/broker-readiness.mjs',import.meta.url),'utf8');
-    const probe=JSON.parse(await run('docker',[...args,'exec','-T','plugin-broker','node','--import','/app/node_modules/tsx/dist/loader.mjs',
-      '--input-type=module','-e',diagnostic,...hashes],{stdoutOnly:true,timeout:60000}));
-    if(probe.version!==1||probe.ready!==true||probe.hostSha256!==hashes[0]||probe.registrySha256!==hashes[1]||probe.configSha256!==hashes[2])
-      throw Error('Invalid isolated broker readiness readback');
-    // A concurrent writer invalidates the readback; never silently accept it.
-    for(let i=0;i<files.length;i++)if(digest(await fs.readFile(files[i]))!==hashes[i])throw Error('Broker binding changed during verification');
-    return probe;
+    return verifyBroker(config,run);
   }finally {await fs.rm(temporary,{recursive:true,force:true});}
 }
+// A plugin update normally leaves this single-file binding unchanged. If an
+// operator replaced it (for example to approve new network pins), recreation
+// remains necessary and must retain the existing full-turn drain.
+export async function brokerNeedsRefresh(config,run=execute) {
+  if (!(await relayServices(config)).includes('plugin-broker')) return false;
+  const hostConfig=config.hostConfig||path.join(config.deploymentDir,'host-executor.json');
+  const expected=digest(await fs.readFile(hostConfig));
+  const script="const fs=require('fs'),crypto=require('crypto');console.log(crypto.createHash('sha256').update(fs.readFileSync(process.env.EZ_PLUGIN_BROKER_HOST_CONFIG)).digest('hex'));";
+  const mounted=(await run('docker',[...relayArgs(config),'exec','-T','plugin-broker','node','-e',script],{stdoutOnly:true,timeout:15000})).trim();
+  if(!/^[a-f0-9]{64}$/.test(mounted))throw Error('Invalid broker host binding fingerprint');
+  return mounted!==expected;
+}
+// Plugin upgrades change the directory-mounted registry, not the broker's
+// single-file host binding. Verify the live broker without replacing its socket.
+export async function verifyBroker(config,run=execute) {
+  if (!(await relayServices(config)).includes('plugin-broker')) return null;
+  const args=relayArgs(config);
+  const hostConfig=config.hostConfig||path.join(config.deploymentDir,'host-executor.json');
+  const home=path.dirname(hostConfig)===config.deploymentDir
+    ? (await read(hostConfig)).agents?.find(agent=>agent.workspace===config.workspace)?.toolsHome : undefined;
+  if(typeof home!=='string'||!path.isAbsolute(home))throw Error('Missing broker registry binding');
+  const files=[hostConfig,path.join(home,'registry.json'),path.join(home,'config.json')];
+  const hashes=await Promise.all(files.map(async file=>digest(await fs.readFile(file))));
+  // Supply trusted read-only code so rollback can check an older image that
+  // predates this diagnostic. Its validators still come from that image.
+  const diagnostic=await fs.readFile(new URL('../../docker/broker-readiness.mjs',import.meta.url),'utf8');
+  const probe=JSON.parse(await run('docker',[...args,'exec','-T','plugin-broker','node','--import','/app/node_modules/tsx/dist/loader.mjs',
+    '--input-type=module','-e',diagnostic,...hashes],{stdoutOnly:true,timeout:60000}));
+  if(probe.version!==1||probe.ready!==true||probe.hostSha256!==hashes[0]||probe.registrySha256!==hashes[1]||probe.configSha256!==hashes[2])
+    throw Error('Invalid isolated broker readiness readback');
+  // A concurrent writer invalidates the readback; never silently accept it.
+  for(let i=0;i<files.length;i++)if(digest(await fs.readFile(files[i]))!==hashes[i])throw Error('Broker binding changed during verification');
+  return probe;
+}
+
 const healthCodes = new Set(['RELAY_UNREADABLE','RELAY_NOT_POLLING','RELAY_STALE','HOST_UNREADABLE','HOST_STALE','PLUGIN_BROKER_UNREADABLE']);
 async function healthEvidence(config,run) {
   const id=(await run('docker',[...relayArgs(config),'ps','-q','relay'])).trim();
@@ -140,6 +160,17 @@ export async function perform(home,job,hooks) {
   const next=await eligibility(home,job.target,root,job.automatic);
   if(next.old.root!==job.previousRoot||next.old.pkg.version!==job.previousVersion)throw Error('Stale upgrade job');
   job.status='applying';job.root=root;job.startedAt=new Date().toISOString();await save();
+  const complete=async()=>{
+    job.status='completed';job.endedAt=new Date().toISOString();await save();
+    try { job.cleanup=await cleanupStaleBackups(home);await save(); }
+    catch(error) { job.cleanupError=error.message;await save().catch(()=>{}); }
+    return job;
+  };
+  const fail=async(error,recoveryHooks)=>{
+    let message=error.message;
+    if(job.target==='main'&&job.rollback&&/is unhealthy/.test(message))message+=await healthEvidence(config,run).catch(()=> '');
+    job.error=message;await save();return recover(home,job,recoveryHooks);
+  };
   try {
     if(job.target==='main') {
       job.packageManager=await packageManager(root,run);await save();
@@ -183,38 +214,44 @@ export async function perform(home,job,hooks) {
         job.brokerReadiness=await refreshBroker(config,run,brokerImage);
       }
     } else {
-      const old=next.old.record,r=await read(path.join(home,'registry.json'));
+      const old=next.old.record;
       const secrets=await read(path.join(home,'packages',job.target,'secrets.json')).catch(e=>{if(e.code==='ENOENT')return {};throw e;});
       const s=await snapshot(root),candidate={...old,source:root,revision:s.revision,manifest:s.manifest,deployment:s.deployment,sharedRevisions:s.sharedRevisions};
       for (const key of old.sharedEnabled || []) if (sharedIdentity(old, key).fingerprint !== sharedIdentity(candidate, key).fingerprint) throw Error('Shared worker changed; disable this client and coordinate an explicit shared worker upgrade before updating');
       await checkFolders(config,candidate,home);
       const stage={...candidate,compose:path.join(dir,'compose.json')};await atomic(stage.compose,await compose(config,stage,secrets,home));
       for(const [service,spec] of Object.entries(stage.deployment.services))await run('docker',[...pluginArgs(stage),spec.image?'pull':'build',service]);
-      const running=Boolean((await run('docker',[...pluginArgs(old),'ps','--status','running','-q'])).trim());
-      job.rollback={record:old,registry:structuredClone(r),compose:await read(old.compose),running};await save();
-      await run('docker',[...pluginArgs(old),'stop']);
-      const backup=path.join(dir,'backup');await fs.mkdir(backup,{mode:0o700});
-      const volumes=new Set(Object.values(old.deployment.services).flatMap(s=>Object.keys(s.volumes||{})));
-      for(const name of volumes)await backupVolume(run,old,name,backup);
-      await atomic(old.compose,await compose(config,candidate,secrets,home));
-      if(running)await run('docker',[...pluginArgs(candidate),'up','-d','--wait','--wait-timeout','90','--no-build']);
-      r.plugins[job.target]=candidate;
-      for(const alias of Object.keys(candidate.manifest.commands))r.commands[alias]=job.target;
-      await atomic(path.join(home,'registry.json'),r);
-      job.brokerReadiness=await refreshBroker(config,run);
-      job.runtimeVerified=running;
+      return await locked(home,async()=>{
+        try {
+          const current=await eligibility(home,job.target,root,job.automatic);
+          if(current.old.root!==job.previousRoot||current.old.pkg.version!==job.previousVersion||JSON.stringify(current.old.record)!==JSON.stringify(old)||JSON.stringify((await state(home)).config)!==JSON.stringify(config))throw Error('Stale upgrade job');
+          if(await brokerNeedsRefresh(config,run)&&!hooks.refreshPluginBroker)throw Error('Broker host binding changed; retry the plugin update after native work drains');
+          const r=await read(path.join(home,'registry.json'));
+          const running=Boolean((await run('docker',[...pluginArgs(old),'ps','--status','running','-q'])).trim());
+          job.rollback={record:old,registry:structuredClone(r),compose:await read(old.compose),running};await save();
+          await run('docker',[...pluginArgs(old),'stop']);
+          const backup=path.join(dir,'backup');await fs.mkdir(backup,{mode:0o700});
+          const volumes=new Set(Object.values(old.deployment.services).flatMap(s=>Object.keys(s.volumes||{})));
+          for(const name of volumes)await backupVolume(run,old,name,backup);
+          await atomic(old.compose,await compose(config,candidate,secrets,home));
+          if(running)await run('docker',[...pluginArgs(candidate),'up','-d','--wait','--wait-timeout','90','--no-build']);
+          r.plugins[job.target]=candidate;
+          for(const alias of Object.keys(candidate.manifest.commands))r.commands[alias]=job.target;
+          await atomic(path.join(home,'registry.json'),r);
+          job.brokerReadiness=hooks.refreshPluginBroker?await refreshBroker(config,run):await verifyBroker(config,run);
+          await publishBrokerPlugins(home,agent.controlDir);
+          job.runtimeVerified=running;
+          return await complete();
+        } catch(error) { return fail(error,{...hooks,registryLocked:true}); }
+      },{drainInvocations:true,signal:hooks.signal});
     }
-    job.status='completed';job.endedAt=new Date().toISOString();await save();
-    try { job.cleanup=await cleanupStaleBackups(home);await save(); }
-    catch(error) { job.cleanupError=error.message;await save().catch(()=>{}); }
-    return job;
-  }catch(error){
-    let message=error.message;
-    if(job.target==='main'&&job.rollback&&/is unhealthy/.test(message))message+=await healthEvidence(config,run).catch(()=> '');
-    job.error=message;await save();return recover(home,job,hooks);
-  }
+    return await complete();
+  }catch(error){return fail(error,hooks);}
+
 }
 export async function recover(home,job,hooks) {
+  if(job.target!=='main'&&job.rollback&&!hooks.registryLocked)
+    return locked(home,()=>recover(home,job,{...hooks,registryLocked:true}),{drainInvocations:true,signal:hooks.signal});
   const run=hooks.execute||execute,{config}=await state(home);
   try {
     if(job.rollback) {
@@ -239,7 +276,8 @@ export async function recover(home,job,hooks) {
         const b=job.rollback;
         await run('docker',[...pluginArgs(b.record),'stop']);
         await atomic(b.record.compose,b.compose);await atomic(path.join(home,'registry.json'),b.registry);
-        job.rollbackBrokerReadiness=await refreshBroker(config,run);
+        job.rollbackBrokerReadiness=hooks.refreshPluginBroker?await refreshBroker(config,run):await verifyBroker(config,run);
+        await publishBrokerPlugins(home,(await state(home)).agent.controlDir);
         if(b.running)await run('docker',[...pluginArgs(b.record),'up','-d','--wait','--wait-timeout','90','--no-build']);
       }
       job.status='rolled-back';

package/src/updates/supervisor.mjs

@@ -5,7 +5,7 @@ import { spawn } from 'node:child_process';
 import { atomic, locked } from '../plugins/manager.mjs';
 import { callDeliverySocket } from '../delivery-socket-client.mjs';
 import { state, read, jobs, jobPath, check, prepare, submit, missing, cleanupStaleBackups } from './control.mjs';
-import { perform, environment } from './runtime.mjs';
+import { perform, environment, brokerNeedsRefresh } from './runtime.mjs';
 
 const reservedProviderKeys=new Set(['HOME','LANG','LC_ALL','LOGNAME','PATH','SHELL','TERM','TMPDIR','USER','CODEX_HOME','NODE_OPTIONS']);
 const providerEnvironmentKey=value=>{
@@ -52,6 +52,11 @@ export async function withIdleUpgrade(control,job,signal,apply,{isIdle=idle,wait
     if(!signal.aborted)return await apply();
   } finally {await fs.rm(pause,{force:true});}
 }
+// Plugin replacement drains command leases at activation, not native turns.
+export async function withUpgrade(control,job,signal,apply,options) {
+  if(job.target==='main'||options?.requiresIdle)return withIdleUpgrade(control,job,signal,apply,options);
+  if(!signal.aborted)return apply();
+}
 export async function queueAutomatic(home,available) {
   const existing=await jobs(home);
   if(existing.some(job=>['queued','applying','recovery-required'].includes(job.status)))return null;
@@ -94,6 +99,15 @@ export async function supervise(deployment,signal,{discover=check}={}) {
     await waitForHostHeartbeat(agent.controlDir,child,()=>error);
   };
   const pause=path.join(agent.controlDir,'upgrade-pause.json');
+  const refreshRequirement=async job=>{
+    if(job.target==='main')return false;
+    try {return await brokerNeedsRefresh((await state(home)).config);}
+    catch(error) {
+      job.status=job.status==='applying'?'recovery-required':'failed';job.error=error.message;
+      await atomic(path.join(jobPath(home,job.id),'job.json'),job);
+      return null;
+    }
+  };
   const beat=setInterval(()=>{void atomic(path.join(directory,'supervisor.json'),{pid:process.pid,at:Date.now()}).catch(()=>{});},1000);
   try {
     await atomic(path.join(directory,'supervisor.json'),{pid:process.pid,at:Date.now()});
@@ -107,8 +121,11 @@ export async function supervise(deployment,signal,{discover=check}={}) {
     await sleep(1200);
     const interrupted=(await jobs(home)).find(j=>j.status==='applying');
     if(interrupted){
-      await atomic(pause,{id:interrupted.id});const result=await locked(home,()=>perform(home,interrupted,{stopHost,startHost}));await fs.rm(pause,{force:true});
-      if(result.status==='completed'&&interrupted.target==='main')return;
+      const refreshPluginBroker=await refreshRequirement(interrupted);
+      const result=refreshPluginBroker===null?undefined:interrupted.target==='main'
+        ? await (async()=>{await atomic(pause,{id:interrupted.id});try{return await locked(home,()=>perform(home,interrupted,{stopHost,startHost}));}finally{await fs.rm(pause,{force:true});}})()
+        : await withUpgrade(agent.controlDir,interrupted,signal,()=>perform(home,interrupted,{stopHost,startHost,signal,refreshPluginBroker}),{requiresIdle:refreshPluginBroker});
+      if(result?.status==='completed'&&interrupted.target==='main')return;
     }
     const isolated=host.isolation==='isolated'
     if(!isolated && !child)await startHost((await state(home)).config.packageRoot);
@@ -117,15 +134,19 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       if(!isolated && child?.exitCode!==null&&child?.exitCode!==undefined)throw Error('Host transport exited; supervisor service should restart');
       const pending=(await jobs(home)).find(j=>j.status==='queued');
       if(pending) {
-        const result=await withIdleUpgrade(agent.controlDir,pending,signal,async()=>{
+        const refreshPluginBroker=await refreshRequirement(pending);
+        if(refreshPluginBroker===null)continue;
+        const apply=async()=>{
           try {
-            return await locked(home,async()=>{const latest=await read(path.join(jobPath(home,pending.id),'job.json'));return perform(home,latest,{stopHost,startHost});});
+            const activate=async()=>{const latest=await read(path.join(jobPath(home,pending.id),'job.json'));return perform(home,latest,{stopHost,startHost,signal,refreshPluginBroker});};
+            return pending.target==='main'?await locked(home,activate):await activate();
           } catch(error) {
             // A pre-switch rejection is terminal. Applying jobs keep their journal for recovery.
             const latest=await read(path.join(jobPath(home,pending.id),'job.json'));
             if(latest.status==='queued'){latest.status='failed';latest.error=error.message;await atomic(path.join(jobPath(home,pending.id),'job.json'),latest);}else throw error;
           }
-        });
+        };
+        const result=await withUpgrade(agent.controlDir,pending,signal,apply,{requiresIdle:refreshPluginBroker});
         if(result?.status==='completed') {
           if(pending.target==='main')return;
           nextCheck=0;