@jc_stack/ez-agents 0.1.0-beta.51 → 0.1.0-beta.58

package/src/scheduler.ts

@@ -1,15 +1,16 @@
 import { validPreflight, installedPreflight, type Preflight, type PreflightReceipt } from './schedule-preflight.js'
 import { assertEffort, assertScheduledModel } from './model-policy.js'
 import { needsFailureReview } from './failure.js'
-import { mkdir, readFile, readdir, writeFile, rename, link, rm } from 'node:fs/promises'
+import { mkdir, readFile, readdir, writeFile, rename, link, rm, stat } from 'node:fs/promises'
 import { randomUUID, createHash } from 'node:crypto'
 import { join } from 'node:path'
-import { type Owner, sameOwner, validOwner, ownerId, ownerEpoch } from './control-state.js'
+import { ControlStore, type Owner, sameOwner, validOwner, ownerId, ownerEpoch } from './control-state.js'
 import { validApplicationOrigin } from './application-origin.js'
 import { ApplicationBindings } from './application-channel.js'
 import { assertId, ownsRun } from './identity.js'
 import { type AiPreset, type ExecutionChoice, isExecutionChoice, persistedPreset } from './ai.js'
 import { type Trigger, validateTrigger, nextOccurrence } from './schedule-time.js'
+import { quotaCategory, quotaScope } from './setup-fallback.js'
 import { RunStore, type RunRecord } from './runs.js'
 import { Scripts, assertScriptArgs, assertScriptId, type ScriptRunRef } from './scripts.js'
 
@@ -23,6 +24,9 @@ export type Schedule = {
   execution?: ExecutionChoice
   script?: { id: string; args: string[] }
   delivery?: { bindingId: string; scope: string }
+  // Occurrences queued or running when a staged edit replaced their revision;
+  // they finish with the instructions and settings they captured.
+  continuingRunIds?: string[]
 }
 export type ScheduledRunSummary = {
   status: 'completed' | 'failed' | 'cancelled'
@@ -49,9 +53,11 @@ const scriptRef = async (controlDir: string, s: Schedule, owner: Owner): Promise
 const runTexts = (s: Schedule, dueAt: string) => s.script ? [`[schedule ${s.id} due ${dueAt}] script ${s.script.id}`] : [`[schedule ${s.id} due ${dueAt}]`, s.text]
 export const scheduledRunId = (s: Schedule, due: number) => 'r_schedule_' + createHash('sha256')
   .update(JSON.stringify([s.id,s.revision,due])).digest('hex')
+export const occurrenceOf = (s: Schedule, r: RunRecord): boolean =>
+  r.scheduled?.id === s.id && (r.scheduled.revision === s.revision || Boolean(s.continuingRunIds?.includes(r.id)))
 export const holdsSchedule = (s: Schedule, r: RunRecord): boolean =>
-  r.scheduled?.id === s.id && r.scheduled.revision === s.revision &&
-  Boolean(r.interrupted || (s.when === 'unreviewed-failures' && r.status === 'failed'))
+  occurrenceOf(s, r) && Boolean(r.interrupted || (s.when === 'unreviewed-failures' && r.status === 'failed'))
+const inFlight = (r: RunRecord) => r.status === 'queued' || r.status === 'running'
 const atomic = async (file: string, value: unknown, exclusive = false) => {
   const tmp = `${file}.${randomUUID()}.tmp`
   try {
@@ -60,14 +66,42 @@ const atomic = async (file: string, value: unknown, exclusive = false) => {
     else await rename(tmp,file)
   } finally { await rm(tmp,{force:true}) }
 }
+const validContinuing = (ids: unknown) => ids === undefined ||
+  (Array.isArray(ids) && ids.length > 0 && ids.length <= 100 && ids.every(id => typeof id === 'string' && /^[a-zA-Z0-9_-]+$/.test(id)))
 export class Scheduler {
   private dir: string
   constructor(private controlDir: string, private checkPreflight = installedPreflight) { this.dir = join(controlDir,'schedules') }
   private async ensure() { await mkdir(this.dir,{recursive:true,mode:0o700}) }
+  // Definition writes and dispatch commits for one task exclude each other
+  // across the relay and CLI processes. Holders need milliseconds; an older
+  // lock was left by a crashed process.
+  private async lock(id: string, wait = true): Promise<(() => Promise<void>) | undefined> {
+    await this.ensure()
+    const file = join(this.dir,assertId(id)+'.lock'), token = randomUUID()
+    for (let attempt = 0; ; attempt++) {
+      try {
+        await writeFile(file,token,{mode:0o600,flag:'wx'})
+        // Release only this holder's lock, never one taken after it was broken as stale.
+        return async () => { if (await readFile(file,'utf8').catch(() => '') === token) await rm(file,{force:true}) }
+      }
+      catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error
+        if (Date.now() - await stat(file).then(lock => lock.mtimeMs, () => Date.now()) > 30000) { await rm(file,{force:true}); continue }
+        if (!wait) return undefined
+        if (attempt >= 200) throw new Error('Schedule is busy with another update or dispatch; retry')
+        await new Promise(resolve => setTimeout(resolve,25))
+      }
+    }
+  }
+  private async locked<T>(id: string, work: () => Promise<T>): Promise<T> {
+    const release = (await this.lock(id))!
+    try { return await work() } finally { await release() }
+  }
   async get(id: string): Promise<Schedule> {
     const s = JSON.parse(await readFile(join(this.dir,assertId(id)+'.json'),'utf8')) as Schedule
     if (s.version !== 1 || s.id !== id || !validScheduledOrigin({id:s.id,revision:s.revision,dueAt:new Date().toISOString(),pairedAt:s.owner?.pairedAt,originRunId:s.originRunId}) ||
       (s.when !== undefined && s.when !== 'unreviewed-failures') || (s.preflight !== undefined && !validPreflight(s.preflight)) || typeof s.enabled !== 'boolean' || !s.name || typeof s.text !== 'string' ||
+      !validContinuing(s.continuingRunIds) ||
       !validOwner(s.owner) || !validExecution(s) ||
       (s.delivery !== undefined && !validApplicationOrigin({...s.delivery, requestId: s.id})))
       throw new Error('Invalid schedule record')
@@ -105,7 +139,7 @@ export class Scheduler {
   async listTasksReadOnly(runs: RunRecord[]): Promise<ScheduledTaskView[]> {
     const active: ScheduledTaskView[] = []
     for (const s of await this.listReadOnly()) {
-      const current = runs.filter(r => r.scheduled?.id === s.id && r.scheduled.revision === s.revision && ownsRun(s.owner,r))
+      const current = runs.filter(r => occurrenceOf(s,r) && ownsRun(s.owner,r))
       const history = runs.filter(r => r.scheduled?.id === s.id && r.scheduled.pairedAt === s.owner.pairedAt && ownsRun(s.owner,r))
         .filter((r): r is RunRecord & { status: ScheduledRunSummary['status'] } =>
           r.status === 'completed' || r.status === 'failed' || r.status === 'cancelled')
@@ -128,6 +162,7 @@ export class Scheduler {
     await this.ensure(); assertId(input.id)
     if (input.when !== undefined && input.when !== 'unreviewed-failures') throw new Error('Unknown schedule condition')
     if(input.preflight !== undefined && !validPreflight(input.preflight)) throw Error('Invalid schedule preflight')
+    if (!validContinuing(input.continuingRunIds)) throw new Error('Invalid continuing occurrences')
     let s: Schedule
     if (input.script) {
       if (input.execution !== undefined || !input.name) throw new Error('Script schedules need a name and no AI selection')
@@ -181,21 +216,75 @@ export class Scheduler {
     try {return JSON.parse(await readFile(join(this.dir,`${s.id}.${s.revision}.cursor`),'utf8')).preflight}
     catch(e) {if((e as NodeJS.ErrnoException).code==='ENOENT')return undefined;throw e}
   }
+  // Replaces a definition only from the revision the caller read. An edit never
+  // invalidates an occurrence that is already queued or running: it applies to an
+  // idle task, or `stage` lets those occurrences finish on their captured settings.
+  // It keeps the paused/enabled state; only pause/resume toggle it.
+  async edit(input: Omit<Schedule,'version'|'revision'|'continuingRunIds'>, expectedRevision: string,
+    runs: () => Promise<RunRecord[]>, stage = false): Promise<Schedule> {
+    return this.locked(input.id, async () => {
+      const current = await this.get(input.id)
+      if (current.revision !== expectedRevision) throw new Error(`Schedule changed since revision ${expectedRevision}; read it again with show and reapply the edit`)
+      if (!sameOwner(current.owner,input.owner)) throw new Error('Schedule ownership mismatch')
+      const active = (await runs()).filter(r => occurrenceOf(current, r) && inFlight(r)).map(r => r.id)
+      if (active.length && !stage) throw new Error(`Schedule has queued or running occurrences (${active.join(', ')}); edit again when idle, cancel them, or pass --stage to apply the edit to future occurrences only`)
+      return this.save({...input,enabled:current.enabled,continuingRunIds:active.length ? active : undefined})
+    })
+  }
   async enable(id: string, enabled: boolean): Promise<Schedule> {
-    const s = await this.get(id)
-    // Pausing preserves the cursor. Resuming coalesces missed recurrences like restart.
-    await atomic(join(this.dir,assertId(id)+'.json'),{...s,enabled})
-    return {...s,enabled}
+    return this.locked(id, async () => {
+      const s = await this.get(id)
+      // Pausing preserves the cursor. Resuming coalesces missed recurrences like restart.
+      await atomic(join(this.dir,assertId(id)+'.json'),{...s,enabled})
+      return {...s,enabled}
+    })
   }
-  async remove(id: string) { await rm(join(this.dir,assertId(id)+'.json')) }
+  async remove(id: string) { await this.locked(id, () => rm(join(this.dir,assertId(id)+'.json'))) }
   async current(run: RunRecord, owner: Schedule['owner']): Promise<boolean> {
     if (!run.scheduled) return false
     try {
       const s = await this.get(run.scheduled.id)
-      return s.revision === run.scheduled.revision && sameOwner(s.owner, owner) &&
+      return occurrenceOf(s, run) && sameOwner(s.owner, owner) &&
         (!!s.delivery || ((s.owner.telegramLinkedAt ?? s.owner.pairedAt) === (owner.telegramLinkedAt ?? owner.pairedAt) && s.owner.telegramChatId === owner.telegramChatId))
     } catch { return false }
   }
+  // Explicit retry changes quota admission only; it never repeats another failure.
+  async retryQuota(owner: Owner, runs: RunStore, runId?: string): Promise<{ runIds: string[]; scopes: string[] }> {
+    const all = await runs.list()
+    const requested = runId ? [await runs.get(runId)] : all.filter(r => r.status === 'queued' && r.quotaWait && ownsRun(owner, r))
+    const ready: RunRecord[] = []
+    for (const run of requested) {
+      const quotaFailure = run?.status === 'failed' && run.failureReason === 'setups-unavailable' &&
+        Boolean(run.attempts?.length) && run.attempts!.every(a => quotaCategory(a.category))
+      const eligible = run && ownsRun(owner, run) && run.scheduled && !run.script && !run.interrupted &&
+        ((run.status === 'queued' && run.quotaWait) || quotaFailure) &&
+        await this.current(run, owner) && (await this.get(run.scheduled.id)).enabled && !await this.cancelled(run.id) &&
+        !all.some(other => other.id !== run.id && other.scheduled?.id === run.scheduled!.id && ['queued', 'running'].includes(other.status))
+      if (!eligible) {
+        if (runId) throw new Error('Run is not a current, uncancelled quota-blocked occurrence; inspect it before retrying')
+        continue
+      }
+      if (run.delivery) await new ApplicationBindings(this.controlDir).authorize(run)
+      ready.push(run)
+    }
+    const retried: string[] = [], cleared = new Set<string>()
+    // An edit that replaced the occurrence's revision meanwhile wins over the requeue;
+    // only a requeued occurrence clears its setups' quota backoff.
+    for (const run of ready) await this.locked(run.scheduled!.id, async () => {
+      if (!await this.current(run, owner) || (await runs.list()).some(other => other.id !== run.id && other.scheduled?.id === run.scheduled!.id && inFlight(other))) {
+        if (runId) throw new Error('Schedule changed or has another occurrence in flight; inspect the run before retrying')
+        return
+      }
+      const scopes = [run.execution!.preset, ...run.execution!.fallbacks ?? []].map(quotaScope)
+      await new ControlStore(this.controlDir, 1000).clearQuotaWait(owner, scopes)
+      for (const scope of scopes) cleared.add(scope)
+      await runs.patch(run.id, { status: 'queued', quotaWait: {}, pid: undefined, endedAt: undefined,
+        exitCode: undefined, failure: undefined, failureReason: undefined, failureReview: undefined })
+      retried.push(run.id)
+    })
+    return { runIds: retried, scopes: [...cleared] }
+  }
+
   async cancel(runId: string, run?: RunRecord | null) {
     assertId(runId); await this.ensure()
     const record = run ?? await new RunStore(this.controlDir).get(runId)
@@ -228,16 +317,18 @@ export class Scheduler {
     if(s.preflight) {
       checked=await this.preflight(s.id,s.revision)
       if(!checked.receipt.eligible) throw Error('Schedule preflight has no eligible work')
-      const current=await this.get(s.id)
-      if(!current.enabled || current.revision!==s.revision) throw Error('Schedule changed during preflight')
     }
-    const dueAt=new Date().toISOString()
-    const run=await runs.create({id:runId,ownerId:ownerId(owner),ownerEpoch:ownerEpoch(owner),
-      ...(s.delivery ? {delivery:s.delivery} : {chatId:s.owner.telegramChatId,telegramUserId:s.owner.telegramUserId,telegramEpoch:s.owner.telegramLinkedAt ?? s.owner.pairedAt}),
-      texts:runTexts(s,dueAt),...(s.script ? {script:await scriptRef(this.controlDir,s,owner)} : {execution:{...s.execution!,sessionId:randomUUID()}}),
-      scheduled:{id:s.id,revision:s.revision,dueAt,pairedAt:s.owner.pairedAt,...(s.originRunId ? {originRunId:s.originRunId} : {})}},true)
-    if(checked) await atomic(checked.cursor,{...checked.previous,preflight:checked.receipt})
-    return run
+    return this.locked(s.id, async () => {
+      const current=await this.get(s.id)
+      if(!current.enabled || current.revision!==s.revision) throw Error('Schedule changed before the trigger was queued')
+      const dueAt=new Date().toISOString()
+      const run=await runs.create({id:runId,ownerId:ownerId(owner),ownerEpoch:ownerEpoch(owner),
+        ...(s.delivery ? {delivery:s.delivery} : {chatId:s.owner.telegramChatId,telegramUserId:s.owner.telegramUserId,telegramEpoch:s.owner.telegramLinkedAt ?? s.owner.pairedAt}),
+        texts:runTexts(s,dueAt),...(s.script ? {script:await scriptRef(this.controlDir,s,owner)} : {execution:{...s.execution!,sessionId:randomUUID()}}),
+        scheduled:{id:s.id,revision:s.revision,dueAt,pairedAt:s.owner.pairedAt,...(s.originRunId ? {originRunId:s.originRunId} : {})}},true)
+      if(checked) await atomic(checked.cursor,{...checked.previous,preflight:checked.receipt})
+      return run
+    })
   }
   async recover(runs: RunStore) {
     for (const run of await runs.list()) {
@@ -275,17 +366,23 @@ export class Scheduler {
             const checked=await this.preflight(s.id,s.revision)
             preflight=checked.receipt
             if(!checked.admitted) {await atomic(cursor,{next:future,preflight});continue}
-            const current=await this.get(s.id);if(!current.enabled || current.revision!==s.revision)continue
           } catch (error) {console.error('Schedule preflight unavailable',s.id,error instanceof Error ? error.message : 'Unknown error');const prior=await readFile(cursor,'utf8').then(v=>JSON.parse(v)).catch(e=>{if(e.code==='ENOENT')return {};throw e});await atomic(cursor,{next:future,preflight:{...prior.preflight,state:'unavailable',observedAt:new Date().toISOString()}});continue}
         }
-        const dueAt = new Date(next).toISOString()
-        await runs.create({id:scheduledRunId(s,next),
-          ownerId:ownerId(owner),ownerEpoch:ownerEpoch(owner),
-          ...(s.delivery ? {delivery:s.delivery} : {chatId:s.owner.telegramChatId,telegramUserId:s.owner.telegramUserId,telegramEpoch:s.owner.telegramLinkedAt ?? s.owner.pairedAt}),
-          texts:runTexts(s,dueAt),...(s.script ? {script:await scriptRef(this.controlDir,s,owner)} : {execution:s.execution}),
-          scheduled:{id:s.id,revision:s.revision,dueAt,pairedAt:s.owner.pairedAt,...(s.originRunId?{originRunId:s.originRunId}:{})}},true)
-        // A restart between run creation and this cursor write sees the same occurrence ID.
-        await atomic(cursor,{next:future,...(preflight ? {preflight} : {})})
+        // An edit or another dispatcher holds this task; a later tick retries.
+        const release = await this.lock(s.id, false)
+        if (!release) continue
+        try {
+          const latest = await this.get(s.id)
+          if (latest.revision !== s.revision || !latest.enabled) continue
+          const dueAt = new Date(next).toISOString()
+          await runs.create({id:scheduledRunId(s,next),
+            ownerId:ownerId(owner),ownerEpoch:ownerEpoch(owner),
+            ...(s.delivery ? {delivery:s.delivery} : {chatId:s.owner.telegramChatId,telegramUserId:s.owner.telegramUserId,telegramEpoch:s.owner.telegramLinkedAt ?? s.owner.pairedAt}),
+            texts:runTexts(s,dueAt),...(s.script ? {script:await scriptRef(this.controlDir,s,owner)} : {execution:s.execution}),
+            scheduled:{id:s.id,revision:s.revision,dueAt,pairedAt:s.owner.pairedAt,...(s.originRunId?{originRunId:s.originRunId}:{})}},true)
+          // A restart between run creation and this cursor write sees the same occurrence ID.
+          await atomic(cursor,{next:future,...(preflight ? {preflight} : {})})
+        } finally { await release() }
       } catch (error) { console.error('Schedule dispatch failed',s.id,error instanceof Error ? error.message : 'Unknown error') }
     }
   }

package/src/setup-fallback.ts

@@ -1,12 +1,14 @@
-import { presetLabel, presetProvider, type AiPreset, type ExecutionChoice } from './ai.js'
+import { presetLabel, presetProvider, sameChoice, type AiPreset, type ExecutionChoice } from './ai.js'
+import type { AttemptTelemetry } from './run-telemetry.js'
 
-// Ordered task setups for one scheduled occurrence. Core advances only on typed
+// Ordered setups for one scheduled occurrence or captured chat/application
+// turn. Core advances only on typed
 // evidence: a missing CLI or a provider rejection before any model output or
 // tool item, or a usage limit the run ended on after work began (continued in a
 // fresh session that is told so). Anything else stops the chain.
-export type StartupCategory = 'cli-unavailable' | 'login-unavailable' | 'quota' | 'provider-rejected' | 'access-denied'
+export type StartupCategory = 'cli-unavailable' | 'login-unavailable' | 'quota' | 'credits-required' | 'provider-rejected' | 'access-denied'
 export type StartupRejection = { category: StartupCategory; resetAt?: string }
-export type AttemptCategory = StartupCategory | 'quota-after-work-began' | 'after-work-began' | 'uncertain' | 'same-quota-scope' | 'same-login-scope'
+export type AttemptCategory = StartupCategory | 'quota-after-work-began' | 'credits-required-after-work-began' | 'after-work-began' | 'uncertain' | 'same-quota-scope' | 'same-login-scope'
 export type SetupAttempt = {
   preset: AiPreset
   quotaScope: string
@@ -17,9 +19,11 @@ export type SetupAttempt = {
   sessionId?: string
   startedAt?: string
   endedAt?: string
+  // Present only when the owner enabled run telemetry for this run.
+  telemetry?: AttemptTelemetry
 }
 
-const categories: StartupCategory[] = ['cli-unavailable', 'login-unavailable', 'quota', 'provider-rejected', 'access-denied']
+const categories: StartupCategory[] = ['cli-unavailable', 'login-unavailable', 'quota', 'credits-required', 'provider-rejected', 'access-denied']
 
 // A failed attempt's category and reset time, and whether the occurrence
 // continues on the next setup. Once work began only a quota limit the run ended
@@ -27,17 +31,48 @@ const categories: StartupCategory[] = ['cli-unavailable', 'login-unavailable', '
 export const attemptFailure = ({ workBegan, rejection }: { workBegan: boolean; rejection?: StartupRejection }): { category: AttemptCategory; resetAt?: string; continues: boolean } => {
   const resetAt = rejection?.resetAt ? { resetAt: rejection.resetAt } : {}
   if (!workBegan) return rejection ? { category: rejection.category, ...resetAt, continues: rejection.category !== 'access-denied' } : { category: 'uncertain', continues: false }
-  return rejection?.category === 'quota' ? { category: 'quota-after-work-began', ...resetAt, continues: true } : { category: 'after-work-began', continues: false }
+  return rejection && ['quota', 'credits-required'].includes(rejection.category) ? { category: rejection.category === 'quota' ? 'quota-after-work-began' : 'credits-required-after-work-began', ...resetAt, continues: true } : { category: 'after-work-began', continues: false }
 }
 
 // Transport fact for a setup that continues an occurrence after an earlier setup
 // stopped at its usage limit once work had begun. Facts only, no guidance.
 export const continuationLine = (runId: string, attempts: SetupAttempt[]): string | undefined => {
-  const limited = attempts.filter(a => a.category === 'quota-after-work-began')
+  const limited = attempts.filter(a => ['quota-after-work-began', 'credits-required-after-work-began'].includes(a.category ?? ''))
   return limited.length ? `[run ${runId} continues: ${limited.map(a => `${presetLabel(a.preset)} began work in native session ${a.sessionId} and stopped at its usage limit${a.resetAt ? ` (resets ${a.resetAt})` : ''}`).join('; ')}. That work may be partly applied.]` : undefined
 }
 
-export const setupCandidates = (execution: ExecutionChoice): AiPreset[] => [execution.preset, ...(execution.fallbacks ?? [])]
+export type SetupHealth = Record<string, { lastSuccessAt?: string; resetAt?: string; retryAt?: string; quotaFailures?: number; manualRetry?: true }>
+export type QuotaWait = { retryAt?: string }
+export const quotaBlockedUntil = (health: SetupHealth[string] = {}): number => health.manualRetry ? Infinity :
+  Math.max(Date.parse(health.resetAt ?? '') || 0, Date.parse(health.retryAt ?? '') || 0)
+// Any available saved login may run; otherwise wait for the earliest one.
+export const quotaWait = (candidates: AiPreset[], health: SetupHealth, now = Date.now()): QuotaWait | undefined => {
+  const times = candidates.map(p => quotaBlockedUntil(health[quotaScope(p)]))
+  if (!times.length || times.some(at => at <= now)) return undefined
+  const next = Math.min(...times)
+  return Number.isFinite(next) ? { retryAt: new Date(next).toISOString() } : {}
+}
+export const quotaCategory = (category?: string): boolean =>
+  ['quota', 'quota-after-work-began', 'credits-required', 'credits-required-after-work-began', 'same-quota-scope'].includes(category ?? '')
+export const validSetupHealth = (value: unknown): value is SetupHealth => Boolean(value && typeof value === 'object' && !Array.isArray(value) &&
+  Object.entries(value).every(([scope, health]) => /^[a-z][a-zA-Z0-9@_./:-]{0,199}$/.test(scope) && health && typeof health === 'object' && !Array.isArray(health) &&
+    Object.entries(health).every(([key, at]) => key === 'quotaFailures' ? Number.isInteger(at) && Number(at) >= 0 && Number(at) <= 4 :
+      key === 'manualRetry' ? at === true : ['lastSuccessAt', 'resetAt', 'retryAt'].includes(key) && typeof at === 'string' &&
+      Number.isFinite(Date.parse(at)) && new Date(at).toISOString() === at)))
+
+// Rotate only the primary's contiguous same-model/effort login choices. Other
+// explicitly saved engines/models keep their fallback priority.
+export const setupCandidates = (execution: ExecutionChoice, health: SetupHealth = {}, now = Date.now()): AiPreset[] => {
+  const saved = [execution.preset, ...(execution.fallbacks ?? [])], primary = saved[0]
+  const end = saved.findIndex(p => p.cli !== primary.cli || p.provider !== primary.provider || p.model !== primary.model || p.effort !== primary.effort)
+  const group = saved.slice(0, end < 0 ? saved.length : end)
+  let best = 0, latest = -Infinity
+  for (let i = 0; i < group.length; i++) {
+    const state = health[quotaScope(group[i])], at = Date.parse(state?.lastSuccessAt ?? '')
+    if (!(quotaBlockedUntil(state) > now) && at > latest) { best = i; latest = at }
+  }
+  return [...group.slice(best), ...group.slice(0, best), ...saved.slice(group.length)]
+}
 
 // The owner provisions native profiles for separate provider accounts. Models
 // in one login share its quota scope; another named login may be tried once.
@@ -53,15 +88,18 @@ export const quotaScope = (preset: AiPreset): string => {
 
 // The next untried setup, skipping a login that already rejected authentication
 // or reached its quota in this occurrence. Each candidate is tried at most once.
-export const nextSetup = (candidates: AiPreset[], attempts: SetupAttempt[]): { skipped: SetupAttempt[]; preset?: AiPreset } => {
-  const exhausted = new Map(attempts.filter(a => ['quota', 'quota-after-work-began', 'login-unavailable'].includes(a.category!))
+export const nextSetup = (candidates: AiPreset[], attempts: SetupAttempt[], health: SetupHealth = {}, now = Date.now()): { skipped: SetupAttempt[]; preset?: AiPreset } => {
+  const exhausted = new Map(attempts.filter(a => quotaCategory(a.category) || a.category === 'login-unavailable')
     .map(a => [a.quotaScope, a.category === 'login-unavailable' ? 'same-login-scope' as const : 'same-quota-scope' as const]))
   const skipped: SetupAttempt[] = []
-  for (const preset of candidates.slice(attempts.length)) {
+  for (const preset of candidates) {
+    if (attempts.some(attempt => sameChoice(attempt.preset, preset))) continue
     const scope = quotaScope(preset)
-    if (!exhausted.has(scope)) return { skipped, preset }
-    const at = new Date().toISOString()
-    skipped.push({ preset, quotaScope: scope, outcome: 'skipped', category: exhausted.get(scope), startedAt: at, endedAt: at })
+    const resetAt = health[scope]?.resetAt
+    const category = exhausted.get(scope) ?? (quotaBlockedUntil(health[scope]) > now ? 'same-quota-scope' : undefined)
+    if (!category) return { skipped, preset }
+    const at = new Date(now).toISOString()
+    skipped.push({ preset, quotaScope: scope, outcome: 'skipped', category, ...(resetAt ? { resetAt } : {}), startedAt: at, endedAt: at })
   }
   return { skipped }
 }
@@ -75,7 +113,7 @@ const epochSeconds = (value: unknown): string | undefined =>
   typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? new Date(value * 1000).toISOString() : undefined
 
 const claudeErrors: Record<string, StartupCategory> = {
-  rate_limit: 'quota', billing_error: 'quota',
+  rate_limit: 'quota', billing_error: 'credits-required',
   authentication_failed: 'login-unavailable', oauth_org_not_allowed: 'access-denied', account_on_hold: 'access-denied',
   verification_required: 'access-denied', cloud_credential_error: 'access-denied',
   overloaded: 'provider-rejected', server_error: 'provider-rejected', model_not_found: 'provider-rejected', invalid_request: 'provider-rejected',
@@ -129,11 +167,11 @@ export const startupObserver = () => {
       else if (rejection?.category !== 'access-denied') rejection = undefined
       return
     }
-    if (event?.type === 'thread.started' || event?.type === 'result' ||
+    if (event?.type === 'thread.started' || event?.type === 'result' || event?.type === 'ez.telemetry' ||
         (event?.type === 'system' && event.subtype === 'init')) return
     if (event?.type === 'rate_limit_event') {
       const info = event.rate_limit_info
-      if (info?.status === 'rejected') reject({ category: 'quota', ...(epochSeconds(info.resetsAt) ? { resetAt: epochSeconds(info.resetsAt) } : {}) })
+      if (info?.status === 'rejected') reject({ category: info.errorCode === 'credits_required' ? 'credits-required' : 'quota', ...(epochSeconds(info.resetsAt) ? { resetAt: epochSeconds(info.resetsAt) } : {}) })
       return
     }
     // Native API retry/error metadata is not model output. Do not inspect error
@@ -143,7 +181,7 @@ export const startupObserver = () => {
       if (category) { reject({ category }); return }
     }
     const category = event?.type === 'assistant' && typeof event.error === 'string' ? claudeErrors[event.error] : undefined
-    if (category) reject({ category, ...(category === 'quota' && rejection?.resetAt ? { resetAt: rejection.resetAt } : {}) })
+    if (category) reject({ category: category === 'quota' && rejection?.category === 'credits-required' ? 'credits-required' : category, ...(category === 'quota' && rejection?.resetAt ? { resetAt: rejection.resetAt } : {}) })
     else work()
   }
   return {
@@ -154,6 +192,7 @@ export const startupObserver = () => {
       for (const text of lines) line(text)
     },
     result: () => { line(buffer); buffer = ''; return { workBegan, rejection } },
+    accepted: () => workBegan && !rejection,
     summary: () => startupLine({ workBegan, rejection }),
   }
 }

package/src/updates/runtime.mjs

@@ -19,13 +19,14 @@ export function execute(command,args,options={}) {
   return new Promise((resolve,reject)=>{
     const invocation=process.env.EZ_DOCKER_COMPOSE==='standalone'&&command==='docker'&&args[0]==='compose'
       ? {command:'docker-compose',args:args.slice(1)} : {command,args};
-    const child=spawn(invocation.command,invocation.args,{cwd:options.cwd,env:environment(),stdio:['ignore','pipe','pipe']});let output='',stdout='';
+    const child=spawn(invocation.command,invocation.args,{cwd:options.cwd,env:environment(),stdio:[options.input===undefined?'ignore':'pipe','pipe','pipe']});let output='',stdout='';
     const timer=setTimeout(()=>child.kill('SIGKILL'),options.timeout||600000);
     const file=options.outputFile?createWriteStream(options.outputFile,{flags:'wx',mode:0o600}):null;
     const written=file?new Promise((resolve,reject)=>{file.once('finish',resolve);file.once('error',reject);}):Promise.resolve();
     if(file)child.stdout.pipe(file);else child.stdout.on('data',b=>{output=(output+b).slice(-16000);if(options.stdoutOnly)stdout=(stdout+b).slice(-16000);});
     child.stderr.on('data',b=>{output=(output+b).slice(-16000);});
     void written.catch(()=>child.kill('SIGKILL'));
+    if(options.input!==undefined){child.stdin.on('error',error=>{child.kill('SIGKILL');reject(error);});child.stdin.end(options.input);}
     child.once('error',error=>{clearTimeout(timer);reject(error);});
     child.once('close',async code=>{clearTimeout(timer);try{await written;if(code!==0)throw Error(`${invocation.command} failed (${code}): ${output}`);resolve(options.stdoutOnly?stdout:output);}catch(error){reject(error);}});
   });

package/src/updates/supervisor.mjs

@@ -30,13 +30,14 @@ export async function waitForHostHeartbeat(controlDir,child,failure,{attempts=30
   }
   throw Error('Host transport heartbeat timeout');
 }
-export async function idle(control) {
+export async function idle(control,{pendingRequests=true,relayRequired=false}={}) {
   const host=await fs.readdir(path.join(control,'host-executor')).catch(e=>{if(e.code==='ENOENT')return [];throw e;});
-  if(host.some(f=>f.endsWith('.running.json')||f.endsWith('.request.json')))return false;
+  if(host.some(f=>f.endsWith('.running.json')||(pendingRequests&&f.endsWith('.request.json'))))return false;
   // Run records live in relay memory: the live socket status is the only
-  // cross-process view of active work. A down relay cannot own a run.
+  // cross-process view of active work. Reload additionally requires its
+  // admission barrier; an unavailable response cannot prove a safe shutdown.
   const status=await callDeliverySocket(path.join(control,'delivery.sock'),{op:'status'},2000).catch(()=>null);
-  return !status||status.running===0;
+  return status ? status.running===0 : !relayRequired;
 }
 // Pause admission only after work is idle, and release it if a turn raced the
 // check. A queued upgrade must never hold the owner's chat behind a long turn.
@@ -73,6 +74,26 @@ export async function queueAutomatic(home,available) {
   }
   return null;
 }
+// Docker archive copy keeps the binding private and readonly without changing
+// host ownership, which may differ from the unprivileged container identity.
+export function executorBindingArchive(binding,gid) {
+  if(!Number.isSafeInteger(gid)||gid<0||gid>0o7777777)throw Error('Invalid executor group');
+  const data=Buffer.from(JSON.stringify(binding)),header=Buffer.alloc(512);
+  const oct=(offset,width,value)=>header.write(value.toString(8).padStart(width-1,'0')+'\0',offset);
+  header.write('executor.json');oct(100,8,0o440);oct(108,8,0);oct(116,8,gid);
+  oct(124,12,data.length);oct(136,12,0);header.fill(32,148,156);
+  header.write('0',156);header.write('ustar\0',257);header.write('00',263);
+  header.write(header.reduce((sum,byte)=>sum+byte,0).toString(8).padStart(6,'0')+'\0 ',148);
+  return Buffer.concat([header,data,Buffer.alloc((512-data.length%512)%512+1024)]);
+}
+export function executorVolumes(deployment,worker) {
+  const names=[...new Set(worker.volumes.filter(m=>m.type==='volume').map(m=>m.source))];
+  return Object.fromEntries(names.map(name=>{
+    const actual=deployment.volumes?.[name]?.name;
+    if(typeof actual!=='string'||!actual)throw Error('Missing executor volume binding');
+    return [name,{name:actual,external:true}];
+  }));
+}
 export async function supervise(deployment,signal,{discover=check}={}) {
   const host=await read(path.join(deployment,'host-executor.json'));
   if(host.agents.length!==1||!host.agents[0].toolsHome)throw Error('Initialize a single deployment plugin registry before starting the supervisor');
@@ -86,11 +107,13 @@ export async function supervise(deployment,signal,{discover=check}={}) {
   await fs.writeFile(lock,JSON.stringify({pid:process.pid}),{flag:'wx',mode:0o600});
   let child,container;
   let retirements=[];
-  let reloadPending=false;
+  let reloadPending;
+  let reloadReady=false;
+  let admissionReady=false;
   const generations=new Map();
   const removeContainer=async id=>{
     const generation=generations.get(id);
-    if(generation){await execute('docker',['compose','--project-name',generation.project,'--file',generation.file,'down']);await fs.rm(generation.file,{force:true});await fs.rm(generation.configFile,{force:true});generations.delete(id);}
+    if(generation){await execute('docker',['compose','--project-name',generation.project,'--file',generation.file,'down']);await fs.rm(generation.file,{force:true});generations.delete(id);}
     else await execute('docker',['rm',id]);
   };
   const stopHost=async()=>{
@@ -121,20 +144,28 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       const generation=path.basename(root).replace(/[^a-zA-Z0-9_.-]/g,'-')+'-'+Date.now();
       const project='ez-executor-'+generation.toLowerCase();
       const file=path.join(directory,project+'.json');
-      const configFile=path.join(directory,project+'-binding.json');
-      await atomic(configFile,{...host,agents:host.agents.map(a=>({...a,binDir:'/app/bin'}))});
+      const binding={...host,agents:host.agents.map(a=>({...a,binDir:'/app/bin'}))};
       const worker={image:relay.image,init:true,restart:'no',user:`${relay.environment.EZ_RUNTIME_UID||1000}:${relay.environment.EZ_RUNTIME_GID||1000}`,
         cap_drop:['ALL'],security_opt:['no-new-privileges:true'],entrypoint:['node','--import','/app/node_modules/tsx/dist/loader.mjs','/app/src/host-executor.ts','/run/executor.json'],
         environment:{...relay.environment,EZ_CONTAINER_EXECUTOR:'1',EZ_EXECUTOR_TRANSPORT:'local'},
-        volumes:[...relay.volumes,{type:'bind',source:configFile,target:'/run/executor.json',read_only:true}],tmpfs:['/tmp'],
+        volumes:relay.volumes,tmpfs:['/tmp'],
         ...Object.fromEntries(['networks','network_mode','extra_hosts','dns','dns_search'].filter(key=>relay[key]!==undefined).map(key=>[key,relay[key]]))};
       if(worker.networks&&!Array.isArray(worker.networks))worker.networks=Object.fromEntries(Object.keys(worker.networks).map(name=>[name,{}]));
       const networks=worker.networks?Object.fromEntries(Object.entries(deployment.networks||{}).map(([name,binding])=>[name,{name:binding.name,external:true}])):undefined;
-      await atomic(file,{services:{executor:worker},...(networks?{networks}:{})});
-      await execute('docker',['compose','--project-name',project,'--file',file,'up','-d','--no-build','executor']);
-      container=(await execute('docker',['compose','--project-name',project,'--file',file,'ps','-q','executor'],{stdoutOnly:true})).trim();
-      if(!/^[a-f0-9]{12,64}$/.test(container))throw Error('Missing isolated executor');
-      generations.set(container,{project,file,configFile});
+      const volumes=executorVolumes(deployment,worker);
+      const archive=executorBindingArchive(binding,Number(worker.user.split(':')[1]));
+      await atomic(file,{services:{executor:worker},...(networks?{networks}:{}),...(Object.keys(volumes).length?{volumes}:{})});
+      try {
+        await execute('docker',['compose','--project-name',project,'--file',file,'create','--no-build','executor']);
+        container=(await execute('docker',['compose','--project-name',project,'--file',file,'ps','-aq','executor'],{stdoutOnly:true})).trim();
+        if(!/^[a-f0-9]{12,64}$/.test(container))throw Error('Missing isolated executor');
+      } catch(error) {
+        await execute('docker',['compose','--project-name',project,'--file',file,'down']);
+        await fs.rm(file,{force:true});container=undefined;throw error;
+      }
+      generations.set(container,{project,file});
+      await execute('docker',['cp','-',`${container}:/run`],{input:archive});
+      await execute('docker',['start',container]);
       for(let n=0;n<300;n++){
         const running=await execute('docker',['inspect','--format','{{.State.Running}}',container],{stdoutOnly:true});
         if(running.trim()!=='true')throw Error('Isolated executor exited during startup');
@@ -167,6 +198,7 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       (host.isolation!=='isolated'||(container&&old.ezRelease?.containerExecutor===1&&candidate.ezRelease?.containerExecutor===1));
   };
   const pause=path.join(agent.controlDir,'upgrade-pause.json');
+  const inheritedPause=await read(pause).catch(missing);
   const refreshRequirement=async job=>{
     if(job.target==='main')return false;
     try {return await brokerNeedsRefresh((await state(home)).config);}
@@ -198,7 +230,7 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       const result=refreshPluginBroker===null?undefined:interrupted.target==='main'
         ? await (async()=>{await atomic(pause,{id:interrupted.id});try{return await locked(home,()=>perform(home,interrupted,{stopHost,startHost,retireHost,preserveRuns:interrupted.preserveRuns===true}));}finally{await fs.rm(pause,{force:true});}})()
         : await withUpgrade(agent.controlDir,interrupted,signal,()=>perform(home,interrupted,{stopHost,startHost,retireHost,signal,refreshPluginBroker}),{requiresIdle:refreshPluginBroker});
-      if(['completed','rolled-back'].includes(result?.status)&&interrupted.target==='main')reloadPending=true;
+      if(['completed','rolled-back'].includes(result?.status)&&interrupted.target==='main')reloadPending=interrupted;
     }
     const isolated=host.isolation==='isolated'
     if(!isolated&&!child)await startHost((await state(home)).config.packageRoot);
@@ -206,7 +238,9 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       if(!isolated||container)return;
       const root=(await state(home)).config.packageRoot;
       if((await read(path.join(root,'package.json'))).ezRelease?.containerExecutor!==1)return;
-      await withIdleUpgrade(agent.controlDir,{id:'executor-migration'},signal,()=>startHost(root));
+      // Queued requests need this worker to drain. Existing native runs still
+      // block migration; the worker lock continues to enforce single ownership.
+      await withIdleUpgrade(agent.controlDir,{id:'executor-migration'},signal,()=>startHost(root),{isIdle:()=>idle(agent.controlDir,{pendingRequests:false})});
     };
     await initializeIsolated();
     let nextCheck=0;
@@ -214,7 +248,15 @@ export async function supervise(deployment,signal,{discover=check}={}) {
       if(!isolated && child?.exitCode!==null&&child?.exitCode!==undefined)throw Error('Host transport exited; supervisor service should restart');
       await initializeIsolated();
       if(container&&(await execute('docker',['inspect','--format','{{.State.Running}}',container],{stdoutOnly:true})).trim()!=='true')throw Error('Isolated executor exited; supervisor service should restart');
-      if(reloadPending&&await idle(agent.controlDir))return;
+      // Reopen only after the successor worker is ready. The relay and its
+      // unclaimed requests remain running throughout this supervisor restart.
+      if(!admissionReady&&!reloadPending&&(!isolated||container)){await fs.rm(pause,{force:true});admissionReady=true;}
+      if(reloadPending) {
+        // Hold new turns while existing generations finish. Queued requests
+        // cannot block a restart that leaves the relay and its ledger running.
+        await atomic(pause,{id:reloadPending.id});
+        if(await idle(agent.controlDir,{pendingRequests:false,relayRequired:true})){reloadReady=true;return;}
+      }
       // Reap retired containers only after their worker exits.
       for(const item of retirements.filter(r=>r.container)){
         if((await execute('docker',['inspect','--format','{{.State.Running}}',item.container],{stdoutOnly:true})).trim()==='false'){await removeContainer(item.container);retirements=retirements.filter(r=>r!==item);}
@@ -236,7 +278,7 @@ export async function supervise(deployment,signal,{discover=check}={}) {
         };
         const result=await withUpgrade(agent.controlDir,pending,signal,apply,{requiresIdle:refreshPluginBroker,preserveRuns:preserve});
         if(['completed','rolled-back'].includes(result?.status)) {
-          if(pending.target==='main')reloadPending=true;
+          if(pending.target==='main')reloadPending=pending;
           nextCheck=0;
         }
       }
@@ -256,7 +298,10 @@ export async function supervise(deployment,signal,{discover=check}={}) {
   }finally {
     clearInterval(beat);await stopHost();
     for(const item of retirements){if(item.container){await execute('docker',['stop','--time','10',item.container]);await removeContainer(item.container);}else if(item.child.exitCode===null)item.child.kill('SIGTERM');}await fs.rm(lock,{force:true});
-    if(!(await jobs(home)).some(j=>j.status==='applying'))await fs.rm(pause,{force:true});
+    if(!reloadReady) {
+      if(!admissionReady&&inheritedPause)await atomic(pause,inheritedPause);
+      else if(!(await jobs(home)).some(j=>j.status==='applying'))await fs.rm(pause,{force:true});
+    }
   }
 }
 if(process.argv[1]&&path.resolve(process.argv[1])===fileURLToPath(import.meta.url)) {