@jc_stack/ez-agents 0.1.0-beta.52 → 0.1.0-beta.58

package/src/index.ts

@@ -11,7 +11,7 @@ import { dispatchChannel } from './channel-backend.js'
 import { randomUUID } from 'node:crypto'
 import { mkdir, stat, writeFile } from 'node:fs/promises'
 import { Scheduler } from './scheduler.js'
-import { Scripts, DEFAULT_SCRIPT_TIMEOUT_SECONDS } from './scripts.js'
+import { Scripts } from './scripts.js'
 import { redactFailure } from './failure.js'
 import { ownedScheduledTasks, scheduledTaskDetailText, scheduledTasksText } from './scheduled-tasks.js'
 import { EventSources, eventRunId, batchReady, type SourceEvent } from './event-sources.js'
@@ -25,12 +25,13 @@ import { isOwner, ownsRun } from './identity.js'
 import type { Update } from 'grammy/types'
 import { InboxStore, type IncomingItem } from './inbox.js'
 import { loadConfig, type Config } from './config.js'
-import { ControlStore, telegramOwner, ownerId, ownerEpoch, type SessionState } from './control-state.js'
+import { ControlStore, telegramOwner, ownerId, ownerEpoch, type Owner, type SessionState } from './control-state.js'
 import { ApprovalStore } from './approval.js'
 import { CliUnavailableError, startExecutorJob, attachHostJob, terminateJob, opencodeDataHome, claudeConfigHome, usesFileTransport } from './executor.js'
-import { attemptFailure, continuationLine, nextSetup, quotaScope, setupCandidates, startupObserver, unavailableSummary, SetupsUnavailableError, type SetupAttempt } from './setup-fallback.js'
+import { attemptFailure, continuationLine, nextSetup, quotaScope, quotaWait, quotaCategory, setupCandidates, startupObserver, unavailableSummary, SetupsUnavailableError, type SetupAttempt } from './setup-fallback.js'
+import { telemetryObserver } from './run-telemetry.js'
 import { assertScheduledModel } from './model-policy.js'
-import { RunStore, saveRunHandoff, takeRunHandoff, type RunRecord } from './runs.js'
+import { RunStore, activeSetup, saveRunHandoff, takeRunHandoff, type RunRecord } from './runs.js'
 import { splitTelegramText } from './reply.js'
 import { markdownToTelegramHtml, escapeHtml } from './format.js'
 import { sanitizeFileName, stageChatAttachment, workspaceFile } from './files.js'
@@ -95,6 +96,8 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   let activeTypingTimer: ReturnType<typeof setInterval> | null = null
   let activeBackend = false
   const ownerStopped = new WeakSet<ChildProcess>()
+  // Runs the owner stopped; an ordered run never advances to its next setup.
+  const stoppedRuns = new Set<string>()
   // Host-transport executors outlive this relay; a graceful stop hands them off.
   const hostTransport = async () => !config.channelBackendUrl && await usesFileTransport(config.controlDir)
   const childRuns = new WeakMap<ChildProcess, string>()
@@ -111,7 +114,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     nextSendAt = Date.now() + 1000
   }
   let startLock: Promise<void> = Promise.resolve()
-  const withStartLock = (work: () => Promise<void>): Promise<void> => {
+  const withStartLock = <T>(work: () => Promise<T>): Promise<T> => {
     const next = startLock.then(work, work)
     startLock = next.then(
       () => undefined,
@@ -150,8 +153,9 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
         const child = background.get(id)
         if (child) { await scheduler.cancel(id); terminateJob(child); return }
         if (!activeChild || (await runs.running(false))?.id !== id) throw new Error('Application cancellation unavailable')
+        stoppedRuns.add(id)
         ownerStopped.add(activeChild)
-        terminateJob(activeChild)
+        void terminateJob(activeChild).then(stopped => { if (!stopped) stoppedRuns.delete(id) })
       }
     }),
   })
@@ -188,6 +192,17 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   const telegramSource = telegramEnabled ? new TelegramSource(config.controlDir, config.telegramBotToken.split(':')[0], sendChat,
     async(chatId,text,bytes,filename)=>[(await bot!.api.sendDocument(chatId,new InputFile(bytes,filename),{caption:text})).message_id]) : null
 
+  const setupAvailability = async (owner: Owner, runId?: string) => {
+    const health = await control.scheduledSetupHealth(owner)
+    // Only one waiting occurrence probes a login until structured work is accepted.
+    for (const active of await runs.list()) {
+      if (active.id === runId || active.status !== 'running' || !active.quotaWait) continue
+      const preset = activeSetup(active)
+      if (preset) health[quotaScope(preset)] = { ...health[quotaScope(preset)], manualRetry: true }
+    }
+    return health
+  }
+
   const startJob = async (run: RunRecord): Promise<void> => {
     await withStartLock(async () => {
       if (shuttingDown || activeBackend) return
@@ -253,235 +268,326 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
         await releaseExternal(run)
         return
       }
-      // Shared by a first launch and a fallback launch that cannot start.
-      const startFailed = async (error: unknown) => {
-        if (!run.scheduled && activeTypingTimer) {
-          clearInterval(activeTypingTimer)
-          activeTypingTimer = null
-        }
-        if (run.scheduled) background.delete(run.id)
-        const unavailable = error instanceof SetupsUnavailableError
-        await runs.patch(run.id, { status: 'failed', failureReason: unavailable ? 'setups-unavailable' : 'executor-start', failure: await failureEvidence(config.controlDir, safeError(error)), endedAt: new Date().toISOString() })
-        await releaseExternal(run)
-        console.error('run start failed', run.id, safeError(error))
-        if (bot && !run.application && !run.delivery && run.chatId !== undefined)
-          await sendChat(run.chatId, unavailable ? `Run ${run.id} could not start. ${safeError(error)}` : `Run ${run.id} failed to start. Check the local relay log.`)
-        setImmediate(() => {
-          void runs
-            .nextQueued()
-            .then((next) => next && startJob(next))
-            .catch(console.error)
-        })
+      if (run.scheduled && !run.script) {
+        const waiting = quotaWait(setupCandidates(run.execution!), await setupAvailability(owner, run.id))
+        if (waiting) { await runs.patch(run.id, { quotaWait: waiting }); return }
       }
-      try {
-        const launchStarted = performance.now()
-        const started = await runs.patch(run.id, { status: 'running', startedAt: new Date().toISOString() })
-        // A registered script occurrence: no model, prompt or native session.
-        const script = run.script && run.scheduled
-          ? { registration: await new Scripts(config.controlDir).owned(run.script.id, owner), ref: run.script, scheduled: run.scheduled }
-          : undefined
-        if (run.script && !script) throw new Error('Script runs require a scheduled occurrence')
-        if (!started.execution && !run.taskId && !script) throw new Error('Legacy queued work has no pinned AI. Resend the request after /new.')
-        const startsOwnSession = Boolean(run.external || run.taskId || run.scheduled)
-        const session = startsOwnSession
-          ? { sessionId: randomUUID(), hasStarted: false, nativeSessionId: undefined }
-          : await control.executionSession(started.execution!)
-        const selected = script ? initialPreset('codex') : run.taskId ? (started.execution?.preset.cli === 'codex' ? started.execution.preset : initialPreset('codex')) : started.execution!.preset
-        if (run.scheduled && !script) assertScheduledModel(selected.model)
-        // A scheduled occurrence may name ordered fallback setups. They share this
-        // run, workspace and concurrency slot; each is tried at most once.
-        const candidates = run.scheduled && !script ? setupCandidates(started.execution!) : [selected]
-        const ordered = candidates.length > 1
-        let attempts: SetupAttempt[] = []
-        const settle = (change: Partial<SetupAttempt>) => {
-          attempts = [...attempts.slice(0, -1), { ...attempts[attempts.length - 1], ...change, endedAt: new Date().toISOString() }]
+      await watchRun(run, texts, owner)
+    })
+  }
+
+  const cancelUnobservedHostRun = async (id: string): Promise<void> => {
+    const base = join(config.controlDir, 'host-executor', id)
+    const exists = (suffix: string) => stat(base + suffix).then(() => true, e => { if (e.code === 'ENOENT') return false; throw e })
+    if (await exists('.request.json') || await exists('.running.json')) await writeFile(base + '.cancel', '', { mode: 0o600 })
+  }
+
+  // First launches, adopted host sessions and fallback launches share one watcher.
+  const watchRun = async (run: RunRecord, texts: string[], owner: Owner, adopted = false): Promise<void> => {
+    // Shared by a first launch and a fallback launch that cannot start.
+    const startFailed = async (error: unknown) => {
+      if (!run.scheduled && activeTypingTimer) {
+        clearInterval(activeTypingTimer)
+        activeTypingTimer = null
+      }
+      if (run.scheduled) background.delete(run.id)
+      // A later ordered setup can fail after an earlier attempt held the slot.
+      else if (activeChild && childRuns.get(activeChild) === run.id) activeChild = null
+      stoppedRuns.delete(run.id)
+      const unavailable = error instanceof SetupsUnavailableError
+      if (unavailable && run.scheduled && run.execution) {
+        const owner = (await control.status()).owner
+        if (owner && ownsRun(owner, run) && await scheduler.current(run, owner) && !await scheduler.cancelled(run.id)) {
+          const waiting = quotaWait(setupCandidates(run.execution), await setupAvailability(owner, run.id))
+          if (waiting) {
+            await runs.patch(run.id, { status: 'queued', quotaWait: waiting, pid: undefined })
+            return
+          }
         }
-        // Advance only after an eligible failure (see attemptFailure) of a current,
-        // uncancelled occurrence. Setups in an exhausted scope are skipped and recorded.
-        const advance = async (): Promise<AiPreset | undefined> => {
-          let preset: AiPreset | undefined
-          const current = !shuttingDown && !await scheduler.cancelled(run.id) ? (await control.status()).owner : undefined
-          if (current && ownsRun(current, run) && await scheduler.current(run, current)) {
-            const next = nextSetup(candidates, attempts)
-            attempts = [...attempts, ...next.skipped]
-            if (!next.preset) {
-              await runs.patch(run.id, { attempts })
-              throw new SetupsUnavailableError(unavailableSummary(attempts))
-            }
-            preset = next.preset
+      }
+      await runs.patch(run.id, { status: 'failed', failureReason: unavailable ? 'setups-unavailable' : 'executor-start', failure: await failureEvidence(config.controlDir, safeError(error)), endedAt: new Date().toISOString() })
+      await releaseExternal(run)
+      console.error('run start failed', run.id, safeError(error))
+      if (bot && !run.application && !run.delivery && run.chatId !== undefined)
+        await sendChat(run.chatId, unavailable ? `Run ${run.id} could not start. ${safeError(error)}` : `Run ${run.id} failed to start. Check the local relay log.`)
+      setImmediate(() => {
+        void runs
+          .nextQueued()
+          .then((next) => next && startJob(next))
+          .catch(console.error)
+      })
+    }
+    try {
+      const launchStarted = performance.now()
+      const started = adopted ? run : await runs.patch(run.id, { status: 'running', startedAt: new Date().toISOString() })
+      // A registered script occurrence: no model, prompt or native session.
+      const script = run.script && run.scheduled
+        ? { registration: await new Scripts(config.controlDir).owned(run.script.id, owner), ref: run.script, scheduled: run.scheduled }
+        : undefined
+      if (run.script && !script) throw new Error('Script runs require a scheduled occurrence')
+      if (!started.execution && !run.taskId && !script) throw new Error('Legacy queued work has no pinned AI. Resend the request after /new.')
+      const startsOwnSession = Boolean(run.external || run.taskId || run.scheduled)
+      const session = startsOwnSession
+        ? { sessionId: randomUUID(), hasStarted: false, nativeSessionId: undefined }
+        : await control.executionSession(started.execution!)
+      const selected = script ? initialPreset('codex') : run.taskId ? (started.execution?.preset.cli === 'codex' ? started.execution.preset : initialPreset('codex')) : (adopted ? activeSetup(started) : started.execution!.preset)
+      if (!selected) throw new Error('Handed-off run has no authorized saved setup')
+      if (run.scheduled && !script) assertScheduledModel(selected.model)
+      // Owner-enabled telemetry is fixed for the run once it starts.
+      const telemetryOn = !script && !run.taskId && (adopted ? Boolean(run.telemetry) : Boolean((await control.status()).runTelemetry))
+      if (telemetryOn && !adopted) await runs.patch(run.id, { telemetry: true })
+      // A scheduled occurrence, or chat/application work with a captured chain,
+      // may name ordered fallback setups. They share this run, workspace and
+      // execution slot; each is tried at most once. Telemetry also records attempts.
+      const chat = !startsOwnSession
+      const ordered = Boolean(!script && !run.taskId && (run.scheduled || started.execution?.fallbacks?.length || telemetryOn))
+      const candidates = run.scheduled && !script ? setupCandidates(started.execution!, await control.scheduledSetupHealth(owner))
+        : ordered ? [started.execution!.preset, ...started.execution!.fallbacks ?? []] : [selected]
+      const runCancelled = async () => stoppedRuns.has(run.id) || Boolean(run.scheduled && await scheduler.cancelled(run.id))
+      // Authority is rechecked before every later setup.
+      const authorized = async (current: Owner) => ownsRun(current, run) && (run.scheduled ? await scheduler.current(run, current)
+        : !executionBlockReason(run, current) && (!(run.application || run.delivery) || await applicationChannel.bindings.authorize(run).then(() => true, () => false)))
+      let attempts: SetupAttempt[] = [...run.attempts ?? []]
+      // Keep audit history and partial-work facts; a wake starts a new attempt round.
+      const roundStart = adopted ? run.attemptRound ?? 0 : attempts.length
+      if (ordered && !adopted) await runs.patch(run.id, { attemptRound: roundStart })
+      const settle = (change: Partial<SetupAttempt>) => {
+        attempts = [...attempts.slice(0, -1), { ...attempts[attempts.length - 1], ...change, endedAt: new Date().toISOString() }]
+      }
+      // Advance only after an eligible failure (see attemptFailure) of a current,
+      // uncancelled occurrence. Setups in an exhausted scope are skipped and recorded.
+      // Owner-present work (chat, application, event sources) never refuses on
+      // recorded quota health alone: when every saved login is cooling down, it
+      // tries them in order instead.
+      let ignoreHealth = false
+      const advance = async (): Promise<AiPreset | undefined> => {
+        let preset: AiPreset | undefined
+        const current = !shuttingDown && !await runCancelled() ? (await control.status()).owner : undefined
+        if (current && await authorized(current)) {
+          const round = attempts.slice(roundStart)
+          let next = nextSetup(candidates, round, ignoreHealth ? {} : await setupAvailability(current, run.id))
+          if (!next.preset && !run.scheduled && !round.length && !ignoreHealth) { ignoreHealth = true; next = nextSetup(candidates, round) }
+          attempts = [...attempts, ...next.skipped]
+          if (!next.preset) {
+            await runs.patch(run.id, { attempts })
+            // Owner-present work whose setup already began work ends as that
+            // run's failure; it did start.
+            if (!run.scheduled && round.some(a => a.category?.endsWith('after-work-began'))) return undefined
+            throw new SetupsUnavailableError(unavailableSummary(attempts))
           }
-          await runs.patch(run.id, { attempts })
-          return preset
+          preset = next.preset
         }
-        // Each attempt is a fresh native session: a rejected Claude attempt already
-        // claimed its ID, and a session cannot resume under another login's home.
-        const launchSetup = async (preset: AiPreset): Promise<{ child: ChildProcess; cleanup: () => Promise<void>; preset: AiPreset }> => {
-          for (;;) {
-            const sessionId = attempts.length ? randomUUID() : session.nativeSessionId || session.sessionId
-            const continuation = continuationLine(started.id, attempts)
-            if (ordered) {
-              attempts = [...attempts, { preset, quotaScope: quotaScope(preset), outcome: 'running', sessionId, startedAt: new Date().toISOString() }]
+        await runs.patch(run.id, { attempts })
+        return preset
+      }
+      // A scheduled attempt is a fresh native session: a rejected Claude attempt
+      // already claimed its ID, and a session cannot resume under another login's
+      // home. An ordered chat attempt resumes its own login's conversation session.
+      type Launched = { child: ChildProcess; cleanup: () => Promise<void>; preset: AiPreset; resumed?: boolean }
+      const launchSetup = async (preset: AiPreset): Promise<Launched> => {
+        for (;;) {
+          const setup = preset
+          const engine = chat && ordered ? await control.engineSession(session.sessionId, setup) : undefined
+          const sessionId = engine ? engine.nativeSessionId || engine.sessionId : attempts.length ? randomUUID() : session.nativeSessionId || session.sessionId
+          const resumed = engine ? engine.hasStarted : session.hasStarted
+          const continuation = continuationLine(started.id, attempts)
+          if (ordered) {
+            attempts = [...attempts, { preset, quotaScope: quotaScope(preset), outcome: 'running', sessionId, startedAt: new Date().toISOString() }]
+            await runs.patch(run.id, { attempts })
+          }
+          try {
+            return { ...await launch(continuation ? [...texts, continuation] : texts, script ? {
+              workspace: config.workspace, timeoutMs: 0, repairEnabled: config.repairEnabled, runId: started.id,
+              controlDir: config.controlDir, binDir,
+              script: { ...script.ref, scheduleId: script.scheduled.id, scheduleRevision: script.scheduled.revision, dueAt: script.scheduled.dueAt },
+            } : {
+              workspace: config.workspace,
+              timeoutMs: config.executorTimeoutMs,
+              repairEnabled: config.repairEnabled,
+              runId: started.id,
+              controlDir: config.controlDir,
+              binDir,
+              cli: preset.cli,
+              model: preset.model,
+              effort: preset.effort,
+              provider: preset.provider,
+              authProfile: preset.authProfile,
+              codexAutoCompactTokens: config.codexAutoCompactTokens,
+              codexSandbox: !run.taskId && preset.cli === 'codex' ? config.codexSandbox : undefined,
+              sessionId,
+              isResume: resumed,
+              promptSuffix: runPromptSuffix(started),
+              taskRun: Boolean(run.taskId),
+              nativeSession: Boolean(run.scheduled),
+              structuredOutput: ordered,
+              telemetry: telemetryOn,
+              onSession: run.external || run.taskId ? undefined : async (id) => {
+                if (ordered) attempts = attempts.map(a => a.sessionId === sessionId ? { ...a, sessionId: id } : a)
+                await runs.patch(run.id, { nativeSessionId: id, ...(ordered ? { attempts } : {}) })
+                if (engine) await control.saveEngineSession(session.sessionId, setup, id)
+                else if (!run.scheduled) await control.saveNativeSession(session.sessionId, id)
+              },
+            }), preset, resumed }
+          } catch (error) {
+            if (ordered && !(error instanceof CliUnavailableError)) {
+              settle({ outcome: 'failed', category: 'uncertain' })
               await runs.patch(run.id, { attempts })
             }
-            try {
-              return { ...await launch(continuation ? [...texts, continuation] : texts, script ? {
-                workspace: config.workspace, timeoutMs: 0, repairEnabled: config.repairEnabled, runId: started.id,
-                controlDir: config.controlDir, binDir,
-                script: { ...script.ref, scheduleId: script.scheduled.id, scheduleRevision: script.scheduled.revision, dueAt: script.scheduled.dueAt },
-              } : {
-                workspace: config.workspace,
-                timeoutMs: config.executorTimeoutMs,
-                repairEnabled: config.repairEnabled,
-                runId: started.id,
-                controlDir: config.controlDir,
-                binDir,
-                cli: preset.cli,
-                model: preset.model,
-                effort: preset.effort,
-                provider: preset.provider,
-                authProfile: preset.authProfile,
-                codexAutoCompactTokens: config.codexAutoCompactTokens,
-                codexSandbox: !run.taskId && preset.cli === 'codex' ? config.codexSandbox : undefined,
-                sessionId,
-                isResume: session.hasStarted,
-                promptSuffix: runPromptSuffix(started),
-                taskRun: Boolean(run.taskId),
-                nativeSession: Boolean(run.scheduled),
-                onSession: run.external || run.taskId ? undefined : async (id) => { await runs.patch(run.id,{nativeSessionId:id}); if (!run.scheduled) await control.saveNativeSession(session.sessionId,id) },
-              }), preset }
-            } catch (error) {
-              if (ordered && !(error instanceof CliUnavailableError)) {
-                settle({ outcome: 'failed', category: 'uncertain' })
-                await runs.patch(run.id, { attempts })
-              }
-              if (!ordered || !(error instanceof CliUnavailableError)) throw error
-              settle({ outcome: 'failed', category: 'cli-unavailable' })
-              const next = await advance()
-              if (!next) throw error
-              preset = next
-            }
+            if (!ordered || !(error instanceof CliUnavailableError)) throw error
+            settle({ outcome: 'failed', category: 'cli-unavailable' })
+            const next = await advance()
+            if (!next) throw error
+            preset = next
           }
         }
-        const watch = async ({ child, cleanup, preset }: Awaited<ReturnType<typeof launchSetup>>): Promise<void> => {
-          const executionStarted = performance.now()
-          // Attach before disk writes: a fast child can close while PID persistence
-          // is pending, and Node drains its remaining pipes during process close.
-          let failureReason = 'executor-exit', errorTail = '', interrupted = false, outputTail = '', timedOut = false
-          // Script output is bounded diagnostics for run inspection, never chat.
-          if (script) child.stdout?.setEncoding('utf8').on('data', (chunk: string) => { outputTail = (outputTail + chunk).slice(-16384) })
-          // Structured startup events decide whether a failed setup may advance.
-          const startup = ordered ? startupObserver() : undefined
-          if (startup) child.stdout?.setEncoding('utf8').on('data', startup.write)
-          const scriptTimer = script ? setTimeout(() => { timedOut = true; terminateJob(child) }, script.registration.timeoutSeconds * 1000) : undefined
-          child.once('close', () => clearTimeout(scriptTimer))
-          child.stderr?.setEncoding('utf8').on('data', (chunk: string) => {
-            errorTail = (errorTail + chunk).slice(-16384)
-            if (script) outputTail = (outputTail + chunk).slice(-16384)
-            if (chunk.includes('Host CLI executor is offline')) failureReason = 'host-executor-offline'
-            if (chunk.includes('Host executor client interrupted by')) {
-              failureReason = 'host-executor-transport-interrupted'
-              interrupted = true
-            }
-            if (chunk.trim()) console.error('executor stderr', started.id, chunk.trim())
-          })
-          console.info('run timing', { run_id: run.id, phase: 'launch',
-            queue_ms: Math.max(0, Date.parse(started.startedAt!) - Date.parse(run.createdAt)),
-            startup_ms: Math.round(executionStarted - launchStarted), resumed: session.hasStarted })
-          if (run.scheduled) background.set(run.id,child)
-          else activeChild = child
-          childRuns.set(child, run.id)
-          const finished = new Promise<number | null>((resolve) => {
-            if (child.exitCode !== null || child.signalCode !== null) resolve(child.exitCode)
-            else child.once('close', resolve)
-          })
-          await runs.patch(started.id, { pid: child.pid })
-          console.info('run started', {
-            run_id: started.id,
-            pid: child.pid,
-            cli: script ? 'script' : preset.cli,
-            ...(script ? { script: script.ref.id, script_revision: script.ref.revision } : {}),
-            ...(ordered ? { setup: attempts.length } : {}),
-            session: session.sessionId,
-            isResume: session.hasStarted,
-          })
+      }
+      const watch = async ({ child, cleanup, preset, resumed }: Launched): Promise<void> => {
+        const executionStarted = performance.now()
+        // Attach before disk writes: a fast child can close while PID persistence
+        // is pending, and Node drains its remaining pipes during process close.
+        let failureReason = 'executor-exit', errorTail = '', interrupted = false, outputTail = '', timedOut = false
+        // Script output is bounded diagnostics for run inspection, never chat.
+        if (script) child.stdout?.setEncoding('utf8').on('data', (chunk: string) => { outputTail = (outputTail + chunk).slice(-16384) })
+        // Structured startup events decide whether a failed setup may advance.
+        const startup = ordered ? startupObserver() : undefined
+        let probing = Boolean(run.quotaWait)
+        // A resumed Codex session reports its session-wide total.
+        const telemetry = telemetryOn ? telemetryObserver({ cumulativeSession: Boolean(resumed) && ['codex', 'codex-gui'].includes(preset.cli) }) : undefined
+        if (telemetry) child.stdout?.setEncoding('utf8').on('data', (chunk: string) => telemetry.write(chunk))
+        if (startup) child.stdout?.setEncoding('utf8').on('data', (chunk: string) => {
+          startup.write(chunk)
+          if (probing && startup.accepted()) {
+            probing = false
+            void runs.patch(run.id, { quotaWait: undefined }).catch(console.error)
+          }
+        })
+        const scriptTimer = script ? setTimeout(() => { timedOut = true; terminateJob(child) }, Math.max(0,
+          script.registration.timeoutSeconds * 1000 - (adopted ? Date.now() - Date.parse(started.startedAt ?? started.createdAt) : 0))) : undefined
+        child.once('close', () => clearTimeout(scriptTimer))
+        child.stderr?.setEncoding('utf8').on('data', (chunk: string) => {
+          errorTail = (errorTail + chunk).slice(-16384)
+          if (script) outputTail = (outputTail + chunk).slice(-16384)
+          if (chunk.includes('Host CLI executor is offline')) failureReason = 'host-executor-offline'
+          if (chunk.includes('Host executor client interrupted by')) {
+            failureReason = 'host-executor-transport-interrupted'
+            interrupted = true
+          }
+          if (chunk.includes('outcome is unknown')) {
+            failureReason = 'handoff-outcome-unknown'
+            interrupted = true
+          }
+          if (chunk.trim()) console.error('executor stderr', started.id, chunk.trim())
+        })
+        console.info('run timing', { run_id: run.id, phase: 'launch',
+          queue_ms: Math.max(0, Date.parse(started.startedAt!) - Date.parse(run.createdAt)),
+          startup_ms: Math.round(executionStarted - launchStarted), resumed: session.hasStarted })
+        if (run.scheduled) background.set(run.id,child)
+        else activeChild = child
+        childRuns.set(child, run.id)
+        // A stop that arrived while this setup was launching reached only the
+        // previous attempt; stop the new one before it does any work.
+        if (await runCancelled()) { ownerStopped.add(child); void terminateJob(child) }
+        const finished = new Promise<number | null>((resolve) => {
+          if (child.exitCode !== null || child.signalCode !== null) resolve(child.exitCode)
+          else child.once('close', resolve)
+        })
+        await runs.patch(started.id, { pid: child.pid })
+        console.info('run started', {
+          run_id: started.id,
+          pid: child.pid,
+          cli: script ? 'script' : preset.cli,
+          ...(script ? { script: script.ref.id, script_revision: script.ref.revision } : {}),
+          ...(ordered ? { setup: attempts.length } : {}),
+          session: session.sessionId,
+          isResume: session.hasStarted,
+        })
 
-          if (!run.scheduled && activeTypingTimer) clearInterval(activeTypingTimer)
-          if (bot && run.chatId !== undefined && !run.application && !run.external && !run.taskId && !run.scheduled) activeTypingTimer = setInterval(() => {
-            if (performance.now() - executionStarted < 30000) void bot.api.sendChatAction(run.chatId!, 'typing').catch(() => {})
-          }, 4000)
+        if (!run.scheduled && activeTypingTimer) clearInterval(activeTypingTimer)
+        if (bot && run.chatId !== undefined && !run.application && !run.external && !run.taskId && !run.scheduled) activeTypingTimer = setInterval(() => {
+          if (performance.now() - executionStarted < 30000) void bot.api.sendChatAction(run.chatId!, 'typing').catch(() => {})
+        }, 4000)
 
-          const free = () => {
-            if (run.scheduled) background.delete(run.id)
-            else {
-              activeChild = null
-              if (activeTypingTimer) clearInterval(activeTypingTimer)
-              activeTypingTimer = null
-            }
+        const free = () => {
+          if (run.scheduled) background.delete(run.id)
+          else {
+            activeChild = null
+            if (activeTypingTimer) clearInterval(activeTypingTimer)
+            activeTypingTimer = null
           }
-          const finalize = async (code: number | null, cancelled: boolean, failed: boolean) => {
-            await runs.patch(started.id, { status: cancelled ? 'cancelled' : failed ? 'failed' : 'completed', endedAt: new Date().toISOString(), exitCode: code,
-              ...(script ? { output: redactFailure(safeError(outputTail)), ...(timedOut ? { timedOut } : {}) } : {}),
-              ...(failed ? { failureReason, interrupted, failure: await failureEvidence(config.controlDir, safeError(timedOut ? `Script exceeded its ${script!.registration.timeoutSeconds}s timeout and was terminated` : errorTail || `Executor exited with ${code === null ? 'a signal' : `code ${code}`}`)) } : {}) })
-            await releaseExternal(run)
-          }
-          const completion = finished.then((code) => {
-            console.info('run timing', { run_id: run.id, phase: 'execution',
-              execution_ms: Math.round(performance.now() - executionStarted), exit_code: code })
-            return (async () => {
-              let eligible = false
-              await withStartLock(async () => {
-                try {
-                  await cleanup()
-                  // The next relay records a handed-off run's result.
-                  if (handedOff.has(child)) return
-                  if (code === 0 && !run.external && !run.taskId && !run.scheduled) await control.markSessionStarted(session.sessionId)
-                  const cancelled = !timedOut && (ownerStopped.has(child) || Boolean(run.scheduled && await scheduler.cancelled(run.id)))
-                  if (timedOut) failureReason = 'script-timeout'
-                  const observed = startup?.result()
-                  const rejected = observed?.rejection && !observed.workBegan ? observed.rejection : undefined
-                  // A rejection before work is a failure even when the CLI exits 0.
-                  const failed = !cancelled && (code !== 0 || timedOut || Boolean(rejected))
-                  if (startup) {
-                    const { continues, ...failure } = failed ? attemptFailure(observed!) : { continues: false }
-                    settle({ outcome: cancelled ? 'cancelled' : failed ? 'failed' : 'completed', exitCode: code, ...failure })
-                    await runs.patch(run.id, { attempts })
-                    eligible = continues && !interrupted
-                  }
-                  if (!eligible) await finalize(code, cancelled, failed)
-                } catch (error) {
-                  eligible = false
-                  await runs.patch(started.id, { status: ownerStopped.has(child) ? 'cancelled' : 'failed', failureReason: 'session-finalization', failure: await failureEvidence(config.controlDir, safeError(error)), endedAt: new Date().toISOString() })
-                  console.error('Session completion failed', safeError(error))
-                } finally {
-                  // An advancing occurrence keeps its concurrency slot for the next setup.
-                  if (!eligible) free()
-                }
-              })
-              console.info('run ended', { run_id: started.id, code, ...(eligible ? { advancing: true } : {}) })
-              // Re-check cancellation and shutdown under the start lock, so a stop
-              // that already snapshotted children never misses the next setup.
-              if (eligible) return withStartLock(async () => {
-                try {
-                  const next = await advance()
-                  if (next) return await watch(await launchSetup(next))
-                  const cancelled = await scheduler.cancelled(run.id)
-                  await finalize(code, cancelled, !cancelled)
-                  free()
-                  setImmediate(() => { void runs.nextQueued().then(queued => { if (queued && !shuttingDown) return startJob(queued) }).catch(console.error) })
-                } catch (error) { await startFailed(error) }
-              })
-              const next = await runs.nextQueued()
-              if (next && !shuttingDown) await startJob(next)
-            })().catch((error) => console.error('Run completion failed', error.message))
-          })
-          completions.add(completion)
-          void completion.finally(() => completions.delete(completion))
         }
-        await watch(await launchSetup(selected))
-      } catch (error) {
-        await startFailed(error)
+        const finalize = async (code: number | null, cancelled: boolean, failed: boolean) => {
+          stoppedRuns.delete(run.id)
+          await runs.patch(started.id, { status: cancelled ? 'cancelled' : failed ? 'failed' : 'completed', quotaWait: undefined, endedAt: new Date().toISOString(), exitCode: code,
+            ...(script ? { output: redactFailure(safeError(outputTail)), ...(timedOut ? { timedOut } : {}) } : {}),
+            ...(failed ? { failureReason, interrupted, failure: await failureEvidence(config.controlDir, safeError(timedOut ? `Script exceeded its ${script!.registration.timeoutSeconds}s timeout and was terminated` : errorTail || `Executor exited with ${code === null ? 'a signal' : `code ${code}`}`)) } : {}) })
+          await releaseExternal(run)
+        }
+        const completion = finished.then((code) => {
+          console.info('run timing', { run_id: run.id, phase: 'execution',
+            execution_ms: Math.round(performance.now() - executionStarted), exit_code: code })
+          return (async () => {
+            let eligible = false
+            await withStartLock(async () => {
+              try {
+                await cleanup()
+                // The next relay records a handed-off run's result.
+                if (handedOff.has(child)) return
+                const observed = startup?.result()
+                if (chat && ordered) await control.settleEngineSession(session.sessionId, preset, code === 0 || Boolean(observed?.workBegan))
+                else if (code === 0 && chat) await control.markSessionStarted(session.sessionId)
+                const cancelled = !timedOut && (ownerStopped.has(child) || await runCancelled())
+                if (timedOut) failureReason = 'script-timeout'
+                const rejected = observed?.rejection && !observed.workBegan ? observed.rejection : undefined
+                // A rejection before work is a failure even when the CLI exits 0.
+                const failed = !cancelled && (code !== 0 || timedOut || Boolean(rejected) || quotaCategory(observed?.rejection?.category))
+                if (startup) {
+                  const { continues, ...failure } = failed ? attemptFailure(observed!) : { continues: false }
+                  settle({ outcome: cancelled ? 'cancelled' : failed ? 'failed' : 'completed', exitCode: code, ...failure, ...(telemetry ? { telemetry: telemetry.result() } : {}) })
+                  if (!cancelled) await control.recordSetupOutcome(owner, attempts.at(-1)!)
+                  await runs.patch(run.id, { attempts })
+                  eligible = continues && !interrupted
+                }
+                if (!eligible) await finalize(code, cancelled, failed)
+              } catch (error) {
+                eligible = false
+                await runs.patch(started.id, { status: ownerStopped.has(child) ? 'cancelled' : 'failed', failureReason: 'session-finalization', failure: await failureEvidence(config.controlDir, safeError(error)), endedAt: new Date().toISOString() })
+                console.error('Session completion failed', safeError(error))
+              } finally {
+                // An advancing occurrence keeps its concurrency slot for the next setup.
+                if (!eligible) free()
+              }
+            })
+            console.info('run ended', { run_id: started.id, code, ...(eligible ? { advancing: true } : {}) })
+            // Re-check cancellation and shutdown under the start lock, so a stop
+            // that already snapshotted children never misses the next setup.
+            if (eligible) return withStartLock(async () => {
+              try {
+                const next = await advance()
+                if (next) return await watch(await launchSetup(next))
+                const cancelled = await runCancelled()
+                await finalize(code, cancelled, !cancelled)
+                free()
+                setImmediate(() => { void runs.nextQueued().then(queued => { if (queued && !shuttingDown) return startJob(queued) }).catch(console.error) })
+              } catch (error) { await startFailed(error) }
+            })
+            const next = await runs.nextQueued()
+            if (next && !shuttingDown) await startJob(next)
+          })().catch((error) => console.error('Run completion failed', error.message))
+        })
+        completions.add(completion)
+        void completion.finally(() => completions.delete(completion))
       }
-    })
+      const first = ordered && !adopted ? await advance() : selected
+      if (!first) throw new Error('Scheduled execution authority revoked')
+      await watch(adopted ? { child: await attachHostJob(config.controlDir, run.id), cleanup: async () => {}, preset: selected } : await launchSetup(first))
+    } catch (error) {
+      if (adopted) {
+        await cancelUnobservedHostRun(run.id).catch(() => {})
+        await runs.patch(run.id, { status: 'failed', endedAt: new Date().toISOString(), interrupted: true, failureReason: 'handoff-outcome-unknown', failure: await failureEvidence(config.controlDir, `Handed-off run outcome is unknown: ${safeError(error)}`) })
+        return
+      }
+      await startFailed(error)
+    }
   }
 
   const unavailableSources = new Set<string>()
@@ -775,7 +881,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
   // the single delivery executor; the socket only admits work into it.
   let drainingHandoff=false
   let deliveryServer: { socketPath: string; stop: () => Promise<void> } | null = null
-  const handleDeliveryOp = createLedgerHandler(config.controlDir, {
+  const ledgerHandler = createLedgerHandler(config.controlDir, {
     wake: () => { void drainOutbox() },
     status: () => ({
       polling: Boolean(bot?.isRunning()),
@@ -784,6 +890,14 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
       version: packageVersion,
     }),
   })
+  // Read idle status only after admissions already inside startJob finish.
+  // The updater closes admission before asking for this barrier.
+  const handleDeliveryOp: typeof ledgerHandler = op => ['status', 'retryScheduleQuota'].includes(op.op)
+    ? withStartLock(() => ledgerHandler(op)).then(result => {
+      if (op.op === 'retryScheduleQuota') void drainSources().catch(console.error)
+      return result
+    })
+    : ledgerHandler(op)
 
   const checkOwner = async (ctx: Context): Promise<boolean> => {
     if (!ctx.from || ctx.from.is_bot || ctx.message?.sender_chat) return false
@@ -815,6 +929,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     { command: 'status', description: 'Work, queue and delivery health' },
     { command: 'stop', description: 'Stop active work; keep queued messages' },
     { command: 'cancel', description: 'Cancel pending messages; keep active work' },
+    { command: 'retry_quota', description: 'Retry quota-blocked work after a reset or added credits' },
     { command: 'retry', description: 'Retry the latest failed incoming batch' },
     { command: 'new', description: 'New conversation; keep workspace files' },
   ]
@@ -882,6 +997,7 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
       `Current: ${running ? (waitingForHost ? 'waiting for the workspace' : 'running') : 'idle'}`,
       `Scheduled limit: ${config.scheduledConcurrency ?? 6}`,
       `Background: ${scheduled ? `${scheduled} scheduled task${scheduled === 1 ? '' : 's'} running` : 'none'}`,
+      ...all.filter(r => r.status === 'queued' && r.quotaWait).map(r => `Waiting for quota: ${r.scheduled?.id ?? r.id}${r.quotaWait!.retryAt ? ` · next check ${r.quotaWait!.retryAt}` : ' · awaiting availability or /retry_quota'}`),
       `Queue: ${queued || incoming.pending ? `${queued} run${queued === 1 ? '' : 's'}; ${incoming.pending} incoming message${incoming.pending === 1 ? '' : 's'}` : 'empty'}`,
       ...(attention.length ? ['', 'Needs attention', ...attention] : []),
       '',
@@ -898,7 +1014,9 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     if (activeChild) children.unshift([undefined, activeChild])
     let requested = true
     for (const [id, child] of children) {
-      if (!await terminateJob(child)) { requested = false; continue }
+      const runId = childRuns.get(child)
+      if (runId) stoppedRuns.add(runId)
+      if (!await terminateJob(child)) { requested = false; if (runId) stoppedRuns.delete(runId); continue }
       ownerStopped.add(child)
       if (id) await scheduler.cancel(id)
     }
@@ -1024,6 +1142,16 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
         .map((c) => `/${c.command} — ${c.description}`).join('\n'))
       return
     }
+    if (typeof text === 'string' && /^\/retry[_-]quota(?:\s|$)/.test(text)) {
+      const id = text.split(/\s+/)[1]
+      try {
+        if (text.split(/\s+/).length > 2) throw new Error('Usage: /retry_quota [RUN_ID]')
+        const result = await withStartLock(async () => scheduler.retryQuota((await control.status()).owner!, runs, id))
+        void drainSources().catch(console.error)
+        await ctx.reply(result.runIds.length ? `Quota retry queued for ${result.runIds.length} run(s).` : 'No queued quota-blocked work. Use /retry_quota RUN_ID for an earlier quota failure.')
+      } catch (error) { await ctx.reply(safeError(error)) }
+      return
+    }
     if (text === '/retry') {
       const id = await inbox.retryLatest(ctx.from.id, ctx.chat.id, (await control.status()).owner)
       if (id) scheduleIntake()
@@ -1305,59 +1433,20 @@ export const createRelay = (config: Config, launch = startExecutorJob) => {
     // Admit every record before the first attach, so no trigger races a hold.
     const admitted = (await Promise.all(handed.map(run => runs.adopt(run).catch(() => null)))).filter((run): run is RunRecord => run !== null)
     for (const run of admitted) await withStartLock(async () => {
-      let child: ChildProcess
+      let owner: Owner
       try {
+        const current = (await control.status()).owner
+        if (!current || !ownsRun(current, run)) throw new Error('Handed-off run authority revoked')
+        owner = current
         // The foreground lane is serial; a second foreground handoff is corrupt.
         if (!run.scheduled && activeChild) throw new Error('A second handed-off foreground run was cancelled; its outcome is unknown')
-        child = await attachHostJob(config.controlDir, run.id)
       } catch (error) {
         // Never leave an unobserved executor: ask the host to stop it.
-        await mkdir(join(config.controlDir, 'host-executor'), { recursive: true, mode: 0o700 })
-          .then(() => writeFile(join(config.controlDir, 'host-executor', run.id + '.cancel'), '', { mode: 0o600 })).catch(() => {})
+        await cancelUnobservedHostRun(run.id).catch(() => {})
         await runs.patch(run.id, { status: 'failed', endedAt: new Date().toISOString(), interrupted: true, failureReason: 'handoff-outcome-unknown', failure: await failureEvidence(config.controlDir, safeError(error)) })
         return
       }
-      // Listen before any await so a fast exit cannot be missed.
-      const closed = new Promise<number | null>(resolve => child.once('close', resolve))
-      let errorTail = '', outputTail = '', timedOut = false
-      child.stdout?.setEncoding('utf8').on('data', (chunk: string) => { if (run.script) outputTail = (outputTail + chunk).slice(-16384) })
-      child.stderr?.setEncoding('utf8').on('data', (chunk: string) => {
-        errorTail = (errorTail + chunk).slice(-16384)
-        if (run.script) outputTail = (outputTail + chunk).slice(-16384)
-      })
-      if (run.scheduled) background.set(run.id, child)
-      else activeChild = child
-      childRuns.set(child, run.id)
-      await runs.patch(run.id, { pid: child.pid })
-      console.info('run adopted', { run_id: run.id, pid: child.pid })
-      // A script keeps its registered bound from its original start.
-      const limit = run.script ? await new Scripts(config.controlDir).get(run.script.id).then(script => script.timeoutSeconds, () => DEFAULT_SCRIPT_TIMEOUT_SECONDS) : 0
-      const timer = limit ? setTimeout(() => { timedOut = true; terminateJob(child) }, Math.max(0, Date.parse(run.startedAt ?? run.createdAt) + limit * 1000 - Date.now())) : undefined
-      const completion = closed.then(code => withStartLock(async () => {
-        clearTimeout(timer)
-        if (run.scheduled) background.delete(run.id)
-        else if (activeChild === child) activeChild = null
-        if (handedOff.has(child)) return
-        const cancelled = !timedOut && (ownerStopped.has(child) || Boolean(run.scheduled && await scheduler.cancelled(run.id)))
-        const failed = !cancelled && code !== 0
-        const status = cancelled ? 'cancelled' : failed ? 'failed' : 'completed'
-        const unknown = errorTail.includes('outcome is unknown'), transport = errorTail.includes('Host executor client interrupted by')
-        const failureReason = timedOut ? 'script-timeout' : unknown ? 'handoff-outcome-unknown' : transport ? 'host-executor-transport-interrupted'
-          : errorTail.includes('Host CLI executor is offline') ? 'host-executor-offline' : 'executor-exit'
-        if (code === 0 && run.execution && !run.external && !run.taskId && !run.scheduled) await control.markSessionStarted(run.execution.sessionId)
-        const at = new Date().toISOString(), last = run.attempts?.at(-1)
-        await runs.patch(run.id, { status, endedAt: at, exitCode: code,
-          ...(last?.outcome === 'running' ? { attempts: [...run.attempts!.slice(0, -1), { ...last, outcome: status, exitCode: code, endedAt: at }] } : {}),
-          ...(run.script ? { output: redactFailure(safeError(outputTail)), ...(timedOut ? { timedOut } : {}) } : {}),
-          ...(failed ? { failureReason, interrupted: unknown || transport, failure: await failureEvidence(config.controlDir, safeError(errorTail || `Executor exited with ${code === null ? 'a signal' : `code ${code}`}`)) } : {}) })
-        await releaseExternal(run)
-        console.info('run ended', { run_id: run.id, code, adopted: true })
-      })).then(async () => {
-        const next = await runs.nextQueued()
-        if (next && !shuttingDown) await startJob(next)
-      }).catch(error => console.error('Adopted run completion failed', safeError(error)))
-      completions.add(completion)
-      void completion.finally(() => completions.delete(completion))
+      await watchRun(run, run.texts, owner, true)
     }).catch(error => console.error('Run adoption failed', run.id, safeError(error)))
   }