@open-mercato/cezar 0.9.3 → 0.10.0-pr892.1298

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/README.md +165 -36
  2. package/dist/agent-config/account-identity.d.ts +44 -0
  3. package/dist/agent-config/account-identity.js +128 -0
  4. package/dist/agent-config/account-identity.js.map +1 -0
  5. package/dist/agent-config/catalog.d.ts +8 -0
  6. package/dist/agent-config/catalog.js +18 -0
  7. package/dist/agent-config/catalog.js.map +1 -1
  8. package/dist/agent-config/model-settings/claude.d.ts +2 -0
  9. package/dist/agent-config/model-settings/claude.js +11 -0
  10. package/dist/agent-config/model-settings/claude.js.map +1 -0
  11. package/dist/agent-config/model-settings/codex.d.ts +2 -0
  12. package/dist/agent-config/model-settings/codex.js +16 -0
  13. package/dist/agent-config/model-settings/codex.js.map +1 -0
  14. package/dist/agent-config/model-settings/opencode.d.ts +2 -0
  15. package/dist/agent-config/model-settings/opencode.js +8 -0
  16. package/dist/agent-config/model-settings/opencode.js.map +1 -0
  17. package/dist/agent-config/model-settings/pi.d.ts +12 -0
  18. package/dist/agent-config/model-settings/pi.js +18 -0
  19. package/dist/agent-config/model-settings/pi.js.map +1 -0
  20. package/dist/agent-config/model-settings/shared.d.ts +9 -0
  21. package/dist/agent-config/model-settings/shared.js +93 -0
  22. package/dist/agent-config/model-settings/shared.js.map +1 -0
  23. package/dist/agent-config/model-settings/types.d.ts +9 -0
  24. package/dist/agent-config/model-settings/types.js +2 -0
  25. package/dist/agent-config/model-settings/types.js.map +1 -0
  26. package/dist/agent-config/models.d.ts +9 -0
  27. package/dist/agent-config/models.js +32 -0
  28. package/dist/agent-config/models.js.map +1 -0
  29. package/dist/agent-config/service.js +4 -4
  30. package/dist/agent-config/service.js.map +1 -1
  31. package/dist/automations/types.d.ts +4 -0
  32. package/dist/automations/types.js +2 -1
  33. package/dist/automations/types.js.map +1 -1
  34. package/dist/config.d.ts +10 -1
  35. package/dist/config.js +48 -5
  36. package/dist/config.js.map +1 -1
  37. package/dist/contract/agent-config.d.ts +2 -0
  38. package/dist/contract/agent-profiles.d.ts +347 -0
  39. package/dist/contract/automations.d.ts +24 -0
  40. package/dist/contract/events.d.ts +59 -0
  41. package/dist/contract/github.d.ts +88 -0
  42. package/dist/contract/health.d.ts +5 -1
  43. package/dist/contract/index.d.ts +1 -0
  44. package/dist/contract/index.js +981 -544
  45. package/dist/contract/projects.d.ts +33 -5
  46. package/dist/contract/runs.d.ts +247 -0
  47. package/dist/contract/skills.d.ts +7 -0
  48. package/dist/contract/workflows.d.ts +6 -0
  49. package/dist/contract/workspace.d.ts +137 -2
  50. package/dist/core/agent-env.js +41 -16
  51. package/dist/core/agent-env.js.map +1 -1
  52. package/dist/core/agent-model-policy.d.ts +10 -0
  53. package/dist/core/agent-model-policy.js +36 -0
  54. package/dist/core/agent-model-policy.js.map +1 -0
  55. package/dist/core/agent-profiles.d.ts +62 -0
  56. package/dist/core/agent-profiles.js +90 -0
  57. package/dist/core/agent-profiles.js.map +1 -0
  58. package/dist/core/agent-runner.d.ts +37 -6
  59. package/dist/core/agent-runner.js +35 -3
  60. package/dist/core/agent-runner.js.map +1 -1
  61. package/dist/core/backend-detect.d.ts +5 -5
  62. package/dist/core/backend-detect.js +44 -7
  63. package/dist/core/backend-detect.js.map +1 -1
  64. package/dist/core/claude-cli-runner.js +10 -5
  65. package/dist/core/claude-cli-runner.js.map +1 -1
  66. package/dist/core/codex-app-server-runner.js +13 -3
  67. package/dist/core/codex-app-server-runner.js.map +1 -1
  68. package/dist/core/codex-app-server-transport.js +7 -2
  69. package/dist/core/codex-app-server-transport.js.map +1 -1
  70. package/dist/core/model-identity.d.ts +23 -11
  71. package/dist/core/model-identity.js +41 -24
  72. package/dist/core/model-identity.js.map +1 -1
  73. package/dist/core/model-presets.d.ts +8 -2
  74. package/dist/core/model-presets.js +51 -8
  75. package/dist/core/model-presets.js.map +1 -1
  76. package/dist/core/opencode-model-catalog.d.ts +32 -0
  77. package/dist/core/opencode-model-catalog.js +153 -0
  78. package/dist/core/opencode-model-catalog.js.map +1 -0
  79. package/dist/core/opencode-server-runner.d.ts +2 -0
  80. package/dist/core/opencode-server-runner.js +42 -11
  81. package/dist/core/opencode-server-runner.js.map +1 -1
  82. package/dist/core/pi-runner.d.ts +24 -0
  83. package/dist/core/pi-runner.js +349 -0
  84. package/dist/core/pi-runner.js.map +1 -0
  85. package/dist/core/pi-ui-mapper.d.ts +34 -0
  86. package/dist/core/pi-ui-mapper.js +270 -0
  87. package/dist/core/pi-ui-mapper.js.map +1 -0
  88. package/dist/core/provider-auth.d.ts +95 -3
  89. package/dist/core/provider-auth.js +234 -15
  90. package/dist/core/provider-auth.js.map +1 -1
  91. package/dist/core/provider-availability.d.ts +1 -1
  92. package/dist/core/provider-availability.js +2 -1
  93. package/dist/core/provider-availability.js.map +1 -1
  94. package/dist/core/runner-factory.js +3 -0
  95. package/dist/core/runner-factory.js.map +1 -1
  96. package/dist/core/shell-env.d.ts +28 -0
  97. package/dist/core/shell-env.js +56 -0
  98. package/dist/core/shell-env.js.map +1 -0
  99. package/dist/core/ui-events.d.ts +1 -1
  100. package/dist/core/usage-limit.d.ts +42 -0
  101. package/dist/core/usage-limit.js +209 -0
  102. package/dist/core/usage-limit.js.map +1 -0
  103. package/dist/index.js +3 -2
  104. package/dist/index.js.map +1 -1
  105. package/dist/paths.d.ts +53 -5
  106. package/dist/paths.js +63 -9
  107. package/dist/paths.js.map +1 -1
  108. package/dist/planner.js +6 -0
  109. package/dist/planner.js.map +1 -1
  110. package/dist/release/snapshot.d.ts +13 -2
  111. package/dist/release/snapshot.js +25 -1
  112. package/dist/release/snapshot.js.map +1 -1
  113. package/dist/runs/agent-tmpdir.d.ts +41 -0
  114. package/dist/runs/agent-tmpdir.js +183 -0
  115. package/dist/runs/agent-tmpdir.js.map +1 -0
  116. package/dist/runs/auto-name.js +6 -1
  117. package/dist/runs/auto-name.js.map +1 -1
  118. package/dist/runs/event-history.d.ts +50 -0
  119. package/dist/runs/event-history.js +613 -0
  120. package/dist/runs/event-history.js.map +1 -0
  121. package/dist/runs/run-index.d.ts +19 -0
  122. package/dist/runs/run-index.js +43 -0
  123. package/dist/runs/run-index.js.map +1 -0
  124. package/dist/runs/store.d.ts +78 -19
  125. package/dist/runs/store.js +175 -43
  126. package/dist/runs/store.js.map +1 -1
  127. package/dist/server/forge/github.d.ts +169 -0
  128. package/dist/server/forge/github.js +511 -0
  129. package/dist/server/forge/github.js.map +1 -1
  130. package/dist/server/forge/index.d.ts +9 -0
  131. package/dist/server/forge/index.js +14 -0
  132. package/dist/server/forge/index.js.map +1 -1
  133. package/dist/server/github.d.ts +2 -2
  134. package/dist/server/github.js +1 -1
  135. package/dist/server/github.js.map +1 -1
  136. package/dist/server/open-in-app.d.ts +2 -1
  137. package/dist/server/open-in-app.js +3 -1
  138. package/dist/server/open-in-app.js.map +1 -1
  139. package/dist/server/open-in-terminal.d.ts +46 -1
  140. package/dist/server/open-in-terminal.js +72 -13
  141. package/dist/server/open-in-terminal.js.map +1 -1
  142. package/dist/server/provider-action-gate.js +3 -1
  143. package/dist/server/provider-action-gate.js.map +1 -1
  144. package/dist/server/server.d.ts +2199 -428
  145. package/dist/server/server.js +1171 -139
  146. package/dist/server/server.js.map +1 -1
  147. package/dist/server-install/engine.js +5 -18
  148. package/dist/server-install/engine.js.map +1 -1
  149. package/dist/server-install/steps.js +6 -2
  150. package/dist/server-install/steps.js.map +1 -1
  151. package/dist/skills.js +4 -0
  152. package/dist/skills.js.map +1 -1
  153. package/dist/workflows/run.d.ts +202 -2
  154. package/dist/workflows/run.js +782 -115
  155. package/dist/workflows/run.js.map +1 -1
  156. package/dist/workflows/types.d.ts +3 -0
  157. package/dist/workflows/types.js +11 -2
  158. package/dist/workflows/types.js.map +1 -1
  159. package/dist/workspace/agent-accounts.d.ts +153 -0
  160. package/dist/workspace/agent-accounts.js +304 -0
  161. package/dist/workspace/agent-accounts.js.map +1 -0
  162. package/dist/workspace/agent-profiles.d.ts +78 -0
  163. package/dist/workspace/agent-profiles.js +115 -0
  164. package/dist/workspace/agent-profiles.js.map +1 -0
  165. package/dist/workspace/config.d.ts +19 -1
  166. package/dist/workspace/config.js +56 -2
  167. package/dist/workspace/config.js.map +1 -1
  168. package/dist/workspace/projects-cli.js +54 -4
  169. package/dist/workspace/projects-cli.js.map +1 -1
  170. package/dist/workspace/projects.d.ts +19 -1
  171. package/dist/workspace/projects.js +34 -1
  172. package/dist/workspace/projects.js.map +1 -1
  173. package/dist/workspace/semaphore.d.ts +39 -0
  174. package/dist/workspace/semaphore.js +31 -0
  175. package/dist/workspace/semaphore.js.map +1 -1
  176. package/package.json +3 -3
  177. package/scripts/mock-claude.mjs +20 -0
  178. package/scripts/mock-pi-rpc.mjs +83 -0
  179. package/web/dist/assets/{alert-dialog-V9cigyAO.js → alert-dialog-Byp42_IG.js} +1 -1
  180. package/web/dist/assets/arrow-down-CGDbDYKD.js +1 -0
  181. package/web/dist/assets/arrow-left-ESCjr1Hq.js +1 -0
  182. package/web/dist/assets/centered-state-LfAW-igm.js +43 -0
  183. package/web/dist/assets/chevron-right-Du-P1IUd.js +1 -0
  184. package/web/dist/assets/{collapsible-BlF-7CeC.js → collapsible-CBiiqDJM.js} +1 -1
  185. package/web/dist/assets/{commit-list-D2mKElUx.js → commit-list-6PPve7PE.js} +1 -1
  186. package/web/dist/assets/{compare-variants-CFBY16Rc.js → compare-variants-BDnp6mXJ.js} +1 -1
  187. package/web/dist/assets/{diff-DE6u5dTe.js → diff-_jK0OWpF.js} +2 -2
  188. package/web/dist/assets/{diff-stat-B1j6L7C6.js → diff-stat-CIrTu4Ym.js} +1 -1
  189. package/web/dist/assets/{diff-view-Gb2kk3Le.js → diff-view-COA_5IyY.js} +1 -1
  190. package/web/dist/assets/dropdown-menu-BofYpFFw.js +1 -0
  191. package/web/dist/assets/editable-title-_J0zFDl-.js +1 -0
  192. package/web/dist/assets/{ellipsis-vertical-BbxtXb6s.js → ellipsis-vertical-BeMR-l62.js} +1 -1
  193. package/web/dist/assets/{file-j5O4NryU.js → file-DY996Pee.js} +1 -1
  194. package/web/dist/assets/{folder-w9Lwj9x9.js → folder-NO3Ec-TY.js} +1 -1
  195. package/web/dist/assets/git-toolbar-BqJeUWcD.js +1 -0
  196. package/web/dist/assets/github-C96HPV9E.js +1 -0
  197. package/web/dist/assets/{image-preview-CcFdkZXW.js → image-preview-D9cCDfKQ.js} +1 -1
  198. package/web/dist/assets/index-Bx-grqa9.css +2 -0
  199. package/web/dist/assets/index-DoVvGDfW.js +7 -0
  200. package/web/dist/assets/{markdown-q9lm7kQ2.js → markdown-DDzk11zj.js} +1 -1
  201. package/web/dist/assets/new-task-form-BNLLgC6P.js +1 -0
  202. package/web/dist/assets/pill-CVM1EmV4.js +1 -0
  203. package/web/dist/assets/{project-router-DudOmkHf.js → project-router-LRCtP48W.js} +1 -1
  204. package/web/dist/assets/prompt-templates-DFOvYgtW.js +15 -0
  205. package/web/dist/assets/{refresh-cw-_C2EGJK5.js → refresh-cw-D1ys6QY2.js} +1 -1
  206. package/web/dist/assets/repo-git-CzUnsgWY.js +1 -0
  207. package/web/dist/assets/{run-diff-BLoKEnTx.js → run-diff-BlnqwfSS.js} +2 -2
  208. package/web/dist/assets/run-header-Bn7cN0Z1.js +1 -0
  209. package/web/dist/assets/{search-x-BgxaJgro.js → search-x-D6FSpxId.js} +1 -1
  210. package/web/dist/assets/{skill-empty-hint-D5MOmycv.js → skill-empty-hint-iwpCcqgI.js} +1 -1
  211. package/web/dist/assets/skills-CezbOaol.js +1 -0
  212. package/web/dist/assets/{sparkles-ANSka5bH.js → sparkles-MrQrgCI9.js} +1 -1
  213. package/web/dist/assets/{square-terminal-qojq20cH.js → square-terminal-TTH64lO7.js} +1 -1
  214. package/web/dist/assets/tab-link-FYksz037.js +1 -0
  215. package/web/dist/assets/task-changes-ClWlnzBL.js +1 -0
  216. package/web/dist/assets/{task-commits-yXmDrCoE.js → task-commits-Crbrl86b.js} +1 -1
  217. package/web/dist/assets/{task-files-BDRmCCPN.js → task-files-uSCWyD0-.js} +2 -2
  218. package/web/dist/assets/task-thread-DPmamBA1.js +9 -0
  219. package/web/dist/assets/{textarea-DVO7pg8Y.js → textarea-B9Ahj4E4.js} +1 -1
  220. package/web/dist/assets/thread-loading-BH_ypQoy.js +1 -0
  221. package/web/dist/assets/{trash-2-DKcnmg6e.js → trash-2-D8S3V3fu.js} +1 -1
  222. package/web/dist/assets/{triangle-alert-CPUEVSZ3.js → triangle-alert-bB7p9n5E.js} +1 -1
  223. package/web/dist/assets/{upload-D5Jrq7QK.js → upload-Bd5oRLPV.js} +1 -1
  224. package/web/dist/assets/{use-desktop-CXae6uyk.js → use-desktop-BNvbRBeZ.js} +1 -1
  225. package/web/dist/assets/{use-submit-shortcut-RKJLKdsW.js → use-submit-shortcut-B-xhu4Bf.js} +1 -1
  226. package/web/dist/assets/utils-BhpbVMwN.js +64 -0
  227. package/web/dist/assets/{workflows-CGyTdyds.js → workflows-RmfbM9Vy.js} +3 -3
  228. package/web/dist/assets/{zoomable-image-hcqwUIlQ.js → zoomable-image-BCW6ZBV0.js} +1 -1
  229. package/web/dist/index.html +24 -23
  230. package/web/dist/assets/arrow-down-DflHdNlN.js +0 -1
  231. package/web/dist/assets/arrow-left-IIL4uR-C.js +0 -1
  232. package/web/dist/assets/centered-state-BkAJVLJE.js +0 -43
  233. package/web/dist/assets/chevron-right-CxAoD1eD.js +0 -1
  234. package/web/dist/assets/circle-check-CdwcIbkx.js +0 -1
  235. package/web/dist/assets/circle-x-Cnc43xJ2.js +0 -1
  236. package/web/dist/assets/dropdown-menu-Ns-rY6sm.js +0 -1
  237. package/web/dist/assets/editable-title-BGuiDxFT.js +0 -1
  238. package/web/dist/assets/git-pull-request-DlSop3Ch.js +0 -1
  239. package/web/dist/assets/git-toolbar-BdJ8XSxI.js +0 -1
  240. package/web/dist/assets/github-CCLn2OAU.js +0 -1
  241. package/web/dist/assets/index-BmHHLbwc.js +0 -6
  242. package/web/dist/assets/index-u8OAnq0r.css +0 -2
  243. package/web/dist/assets/new-task-form-BuYxAWN7.js +0 -1
  244. package/web/dist/assets/pill-t97MvPW_.js +0 -1
  245. package/web/dist/assets/prompt-templates-C5iO3Lda.js +0 -15
  246. package/web/dist/assets/repo-git-DqQzj33d.js +0 -1
  247. package/web/dist/assets/run-header-Dho4eurS.js +0 -1
  248. package/web/dist/assets/skills-B5g064CC.js +0 -1
  249. package/web/dist/assets/tab-link-CIcwFJtC.js +0 -1
  250. package/web/dist/assets/task-changes-yRfsEsnh.js +0 -1
  251. package/web/dist/assets/task-thread-C9AnIz4Z.js +0 -9
  252. package/web/dist/assets/thread-loading-DJDl701Z.js +0 -1
  253. package/web/dist/assets/utils-Wtkd5E6j.js +0 -64
@@ -4,27 +4,35 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync
4
4
  import { dirname, join } from 'node:path';
5
5
  import { parseAskMarkerResult, stripAskMarker, } from '../core/ask.js';
6
6
  import { onUsage, registerRunProcess, unregisterRunProcess } from '../core/process-usage.js';
7
+ import { parseUsageLimit } from '../core/usage-limit.js';
7
8
  import { createRunner } from '../core/runner-factory.js';
8
9
  import { modelConflictsWithRunner } from '../core/model-presets.js';
10
+ import { AGENT_MODELS_LOCKED_ERROR, agentModelsLocked } from '../core/agent-model-policy.js';
9
11
  import { ModelIdentityError, formatModelIdentity, normalizeModelForBackend, } from '../core/model-identity.js';
10
12
  import { HANDOFF_ONLY_INSTRUCTIONS, HANDOFF_INSTRUCTIONS, appendHandoffHeartbeat, followupsEnabled, handoffPath, seedHandoffFile, } from '../handoff.js';
11
13
  import { todosPath } from '../todos.js';
12
14
  import { discoverSkills } from '../skills.js';
13
15
  import { materializeSkillDir } from '../skills-remote.js';
14
16
  import { seedAgentConfigLocalLayer } from '../agent-config/seed.js';
17
+ import { readAgentModelProvider } from '../agent-config/models.js';
15
18
  import { loadConfig, resolveWorktreeRetention } from '../config.js';
16
19
  import { autosaveCommit, createWorktree, resolveBaseRef, worktreeDiff, worktreeShortstat } from '../git-worktree.js';
17
20
  import { getHeadCommit, getRepoInfo } from '../server/git.js';
18
21
  import { loadWorkflows } from './load.js';
19
22
  import { reclaimWorktrees, rematerializeReclaimedWorktree } from '../runs/retention.js';
23
+ import { AgentTempDirError, agentTmpEnv, removeAgentTmpDir, sweepAgentTmpDirs, } from '../runs/agent-tmpdir.js';
20
24
  import { extractTaskRefs, refineTaskRefs, titleRefNumber } from '../runs/task-refs.js';
21
25
  import { parseTaskMarkers, stripTaskMarkers } from '../runs/task-markers.js';
22
26
  import { autoNamingActive, generateRunName, liveTitleUpdatesEnabled, postValidateTitle } from '../runs/auto-name.js';
23
27
  import { reviewGateEnabled } from '../runs/review-gate.js';
28
+ import { resolveProfileEnvForRoot } from '../workspace/agent-profiles.js';
24
29
  import { WorkspaceSemaphore } from '../workspace/semaphore.js';
25
30
  import { UiEventSink } from '../runs/ui-event-sink.js';
26
31
  import { chainStepNote, DEFAULT_ALLOWED_TOOLS, stepKind } from './types.js';
27
32
  const CHECK_OUTPUT_CAP = 20_000;
33
+ async function configuredModelProvider(backend, repoRoot) {
34
+ return readAgentModelProvider(backend, repoRoot).catch(() => undefined);
35
+ }
28
36
  /** An interactive session that hears nothing from the user closes itself. */
29
37
  export const IDLE_TIMEOUT_MS = 15 * 60_000;
30
38
  /**
@@ -113,6 +121,95 @@ const REPOSITORY_ROOT_LOCK_DISABLED_NOTE = 'repository-root lock disabled by CEZ
113
121
  const MAX_AUTO_CONTINUES = 40;
114
122
  const AUTONOMOUS_NUDGE = 'Continue working autonomously until the task is fully complete. Do not ask me for confirmation or clarification — make reasonable assumptions and proceed. When everything is done, end the session with your done signal.';
115
123
  const MONITORING_WAKE_NUDGE = 'Re-check the downstream work you were monitoring. Continue toward the task goal; emit CEZ:MONITORING again only if it is still pending.';
124
+ /**
125
+ * Auto-resume after a provider usage limit (spec 2026-08-03-auto-resume-after-usage-limit).
126
+ *
127
+ * The wait is the provider's own reset instant plus this grace: resuming AT the boundary races the
128
+ * provider's clock (and its rounding), and one failed resume costs the whole window over again.
129
+ * Thirty seconds is cheap next to five hours and long enough to be past any sane skew.
130
+ */
131
+ export const AUTO_RESUME_GRACE_MS = 30_000;
132
+ /**
133
+ * Consecutive automatic resumes allowed without a human turn. A resume can only fire after a real
134
+ * reset instant, so this is not a throttle — it is the backstop for the pathological case (a
135
+ * provider that answers "limit reached, retry now" in a loop), and it is deliberately generous
136
+ * enough to sit through a couple of days of five-hour windows.
137
+ */
138
+ export const MAX_AUTO_RESUMES = 12;
139
+ /**
140
+ * How long a missed deadline stays worth acting on. The promise is "we pick this up when the
141
+ * window reopens" — kept across a restart or an overnight close, which is the case the feature
142
+ * exists for. A day later it is no longer that promise: the user has moved on, and a task
143
+ * springing back to life is a surprise rather than a service. Such a deadline is retired with a
144
+ * note instead of fired, so the only tasks a sweep can revive are ones someone is still waiting on.
145
+ */
146
+ export const AUTO_RESUME_MISSED_WINDOW_MS = 24 * 60 * 60_000;
147
+ /**
148
+ * How often the queue checks that it is not wedged.
149
+ *
150
+ * A hold is the only thing in the engine that can make an idle queue CORRECT, so it is also the
151
+ * only thing that can make a wedged one look correct. This tick is the way out: cheap (a few
152
+ * in-memory checks), unref'd, and it only ever acts when idling has no justification left.
153
+ */
154
+ export const QUEUE_WATCHDOG_MS = 60_000;
155
+ /** Shared empty holds for the common "nothing is held" pump — avoids allocating per sweep. */
156
+ const NO_HOLDS = { deadline: new Set(), inFlight: new Set() };
157
+ /**
158
+ * May this run start, given what its account is holding?
159
+ *
160
+ * The two kinds of hold bind different work, and getting that wrong has produced a bug in each
161
+ * direction (spec 2026-08-03-auto-resume-after-usage-limit):
162
+ *
163
+ * - a `deadline` hold means the window is KNOWN shut until an instant, so it blocks everything
164
+ * on that account — resumes included. Exempting them let four resumes fire at once and
165
+ * re-limit one after another, which is the stampede wearing a different hat.
166
+ * - an `inFlight` hold means a resume is testing the window right now and nothing is proven, so
167
+ * it blocks fresh work but not other resumes. Blocking those deadlocked a live workspace.
168
+ */
169
+ function accountHeldFor(run, holds, fallbackRunner) {
170
+ const key = runAccountKey(run, fallbackRunner);
171
+ if (holds.deadline.has(key))
172
+ return true;
173
+ return holds.inFlight.has(key) && !resumeInFlight(run);
174
+ }
175
+ /**
176
+ * Which agent ACCOUNT a run's work runs on — the thing a provider usage limit actually closes
177
+ * (spec 2026-08-03-auto-resume-after-usage-limit).
178
+ *
179
+ * Backend plus agent account, because those are the two axes a limit is scoped to: a Claude
180
+ * limit must never stall a Codex task, and a second Claude login is a second budget. A record
181
+ * that names no runner has not started yet and will take the configured default, which is what
182
+ * `fallbackRunner` carries; a run that HAS started always carries its resolved runner (execute
183
+ * persists it), and only started runs can be holding.
184
+ */
185
+ export function runAccountKey(run, fallbackRunner) {
186
+ return `${run.runner ?? fallbackRunner}:${run.agentProfile ?? 'default'}`;
187
+ }
188
+ /**
189
+ * Is this run an automatic resume that has not completed a turn yet?
190
+ *
191
+ * Such a run is the work the reopened window is FOR, so the hold must never apply to it — not
192
+ * its own, and not another resume's. Two resumes that hold each other is a deadlock the queue
193
+ * cannot recover from: both sit `queued` with a counter and no deadline, each waiting for the
194
+ * other to prove a window neither will ever get to test. That is the shape a live run produced
195
+ * — two scheduled tasks fired, both went `queued`, and nothing in the workspace moved again.
196
+ *
197
+ * The hold exists to stop NEW work walking into a closed window. A resume is not new work.
198
+ */
199
+ function resumeInFlight(run) {
200
+ return (run.autoResumeAttempts !== undefined && (run.status === 'queued' || run.status === 'running'));
201
+ }
202
+ const AUTO_RESUME_PROMPT = 'The provider usage limit that interrupted this task has reset. Read the handoff file (CEZ_HANDOFF_FILE) to recover context, then continue the task from where you left off.';
203
+ /**
204
+ * The wake instant as a human reads it — local, to the SECOND, with the zone named. The
205
+ * transcript line is what someone scanning a stalled task actually reads, and "18:41" is not
206
+ * enough to tell a wait that is nearly over from one that just started; the machine-readable ISO
207
+ * copy lives on `RunRecord.autoResumeAt`. Server-side formatting is honest here because cezar is
208
+ * local-first: the process and the browser reading it are the same machine.
209
+ */
210
+ function formatWakeInstant(at) {
211
+ return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'long' }).format(at);
212
+ }
116
213
  /**
117
214
  * The effective "extra" system prompt for a run (spec §protocol v2, R2 2.3):
118
215
  * the per-run override (`POST /api/runs` `systemPrompt`) REPLACES the
@@ -137,6 +234,17 @@ export function composeSystemPrompt(...parts) {
137
234
  .filter((p) => Boolean(p))
138
235
  .join('\n\n---\n\n');
139
236
  }
237
+ /**
238
+ * The directories a spawned agent may reach outside its worktree: the run-state
239
+ * folder that holds its handoff file, plus its own temp directory when this run
240
+ * got one (#785). Handing an agent a `TMPDIR` its file tools are not allowed to
241
+ * write would trade one silent failure for another, so the two travel together;
242
+ * under `CEZ_AGENT_TMPDIR=0` there is no per-run directory and the list is
243
+ * exactly what it always was.
244
+ */
245
+ export function agentDirectories(runsDir, env) {
246
+ return env.TMPDIR ? [runsDir, env.TMPDIR] : [runsDir];
247
+ }
140
248
  /**
141
249
  * Materialized pasted attachment: the on-disk name/serving-URL pair the
142
250
  * transcript already used, plus the absolute path that lets the agent
@@ -239,6 +347,12 @@ export class RunManager {
239
347
  /** Messages that landed in the dequeue → session-open gap (#472), flushed as
240
348
  * ordinary follow-up turns the moment the session opens. In-memory only. */
241
349
  deferredMessages = new Map();
350
+ /** Armed usage-limit resumes, keyed by run id (spec
351
+ * 2026-08-03-auto-resume-after-usage-limit). The DEADLINE itself lives on the record
352
+ * (`autoResumeAt`) — this map holds only the process-local timer, so a restart rebuilds it
353
+ * from the record rather than losing the wait. Runs here are `failed` and therefore NOT in
354
+ * `active`, which is why the timer cannot live on an `ActiveRun` like the monitoring one. */
355
+ autoResumeTimers = new Map();
242
356
  pumping = false;
243
357
  /** A pump that arrived while one was in flight — replayed by `pump()`'s own
244
358
  * loop so a slot freed mid-sweep is never a lost wakeup. */
@@ -258,6 +372,13 @@ export class RunManager {
258
372
  /** Unsubscribe handle for the constructor's `onUsage` subscription — released
259
373
  * by dispose() so a torn-down manager stops receiving sampler ticks. */
260
374
  offUsage;
375
+ /** The stalled-queue watchdog (see `rescueStalledQueue`). */
376
+ queueWatchdog;
377
+ /** Set by the watchdog for exactly one sweep: ignore the usage-limit hold and make progress. */
378
+ forceNextPump = false;
379
+ /** Runs the watchdog started despite the hold. The spawn-time gate (`requeueWhileHeld`) would
380
+ * otherwise hand them straight back and the rescue would undo itself in a millisecond. */
381
+ forceStarted = new Set();
261
382
  /** The workspace-wide parallel-cap semaphore + cached resource config
262
383
  * (spec 2026-07-20, step 2.5). Boot constructs ONE and every manager shares
263
384
  * it; the private fallback keeps single-manager callers and tests working. */
@@ -274,10 +395,13 @@ export class RunManager {
274
395
  busySlots: () => this.busySlots(),
275
396
  pump: () => this.pump(),
276
397
  oldestQueuedAt: () => this.oldestQueuedAt(),
398
+ accountHolds: () => this.accountHolds(),
277
399
  });
278
400
  // Memory guard (#memory-guard): the shared process-tree sampler already ticks ~every 2 s for
279
401
  // the runs table; piggyback on it to enforce the per-task memory ceiling.
280
402
  this.offUsage = onUsage((snapshot) => void this.enforceMemoryLimit(snapshot));
403
+ this.queueWatchdog = setInterval(() => void this.rescueStalledQueue(), QUEUE_WATCHDOG_MS);
404
+ this.queueWatchdog.unref?.();
281
405
  }
282
406
  /**
283
407
  * Release everything this manager owns without touching run records
@@ -292,6 +416,7 @@ export class RunManager {
292
416
  dispose() {
293
417
  this.offUsage();
294
418
  this.offSemaphore();
419
+ clearInterval(this.queueWatchdog);
295
420
  for (const [runId, state] of this.active) {
296
421
  this.clearIdleTimer(state);
297
422
  this.clearMonitoringWakeTimer(state, runId);
@@ -299,6 +424,9 @@ export class RunManager {
299
424
  state.releaseRepoRoot?.();
300
425
  state.releaseRepoRoot = undefined;
301
426
  }
427
+ for (const timer of this.autoResumeTimers.values())
428
+ clearTimeout(timer);
429
+ this.autoResumeTimers.clear();
302
430
  this.active.clear();
303
431
  this.waiting.clear();
304
432
  this.starting.clear();
@@ -364,21 +492,64 @@ export class RunManager {
364
492
  * key would let a value inherited from *this* process through — a nested
365
493
  * cezar (an agent running `cez serve`/`cez run`/the test suite) would then
366
494
  * write follow-ups into the parent's inbox despite the opt-out. Empty is the
367
- * established "absent" spelling — consumers guard with `if (todosFile)`. */
495
+ * established "absent" spelling — consumers guard with `if (todosFile)`.
496
+ *
497
+ * `TMPDIR`/`TEMP`/`TMP` (#785) point at this run's own scratch directory
498
+ * instead of the machine-wide one every agent used to share. Created and
499
+ * write-probed here, on the last common path before a spawn, so an unusable
500
+ * temp directory throws `AgentTempDirError` at the caller rather than
501
+ * turning into empty command output inside a running agent. */
368
502
  agentEnv(runId, generateFollowups = true) {
369
503
  return {
370
504
  CEZ_HANDOFF_FILE: handoffPath(this.dataDir, runId),
371
505
  CEZ_TASK_ID: runId,
372
506
  CEZ_TODOS_FILE: generateFollowups ? todosPath(this.dataDir) : '',
507
+ ...agentTmpEnv(this.dataDir, runId),
508
+ };
509
+ }
510
+ /**
511
+ * `agentEnv` plus the agent-account variable for the profile this STEP runs under (spec
512
+ * 2026-07-29-agent-profiles), and the id it resolved to so the caller can record it.
513
+ *
514
+ * Resolved per step, not per run, because a workflow can mix backends: an override naming a
515
+ * Claude account says nothing about which Codex account a codex step should use. Resolution
516
+ * order, most specific first:
517
+ *
518
+ * 1. the step's ALREADY-RECORDED `profileId` — a resume or Continue must reattach to the
519
+ * account that created the session, whatever the project has since been switched to;
520
+ * 2. the run's composer override, but only for steps on the run's own runner;
521
+ * 3. the project's stored selection, and failing that the discovered default.
522
+ *
523
+ * Read fresh every time. `~/.cezar/config.json` is shared by every cezar process on this
524
+ * machine, so a cached snapshot is a staleness bug, and one small JSON read is free next to
525
+ * spawning a CLI. Never throws: an unreadable home degrades to the default profile, which is
526
+ * exactly the behaviour that predates profiles.
527
+ */
528
+ async agentEnvForStep(runId, backend, options = {}) {
529
+ const run = this.store.getRun(runId);
530
+ const profileId = options.recordedProfileId
531
+ ?? (backend === (run?.runner ?? 'claude') ? run?.agentProfile : undefined);
532
+ const resolved = await resolveProfileEnvForRoot(this.repoRoot, backend, profileId);
533
+ return {
534
+ env: { ...this.agentEnv(runId, options.generateFollowups), ...resolved.env },
535
+ profileId: resolved.profile.id,
373
536
  };
374
537
  }
375
538
  startRun(workflow, input, group) {
539
+ // Sanitize at the manager boundary so CLI runs, workflows, variants, and
540
+ // direct callers cannot bypass the HTTP policy.
541
+ const effectiveInput = agentModelsLocked(this.repoRoot)
542
+ ? { ...input, model: undefined }
543
+ : input;
376
544
  const run = this.store.createRun({
377
545
  title: makeRunTitle(input.task, workflow) + (group ? ` (${group.variant})` : ''),
378
546
  workflow: workflow.name,
379
547
  task: input.task,
380
- model: input.model,
548
+ model: effectiveInput.model,
381
549
  runner: input.runner,
550
+ // The composer's per-task account (spec 2026-07-29-agent-profiles). Persisted at creation
551
+ // so a queued run picks it up at dequeue and every later resume reads the same answer.
552
+ agentProfile: input.agentProfile,
382
553
  // The global inbox is the ceiling on the per-run flag (#471). Enforced here rather than
383
554
  // at the HTTP route because `cezar run`, the inbox's own "▶ Run" and variants all reach
384
555
  // startRun directly — a route-level gate would leave those writing todos.json.
@@ -425,7 +596,7 @@ export class RunManager {
425
596
  // above shows instantly; the namer's short title replaces it when (and if)
426
597
  // the model answers. Never awaited, never fails the run.
427
598
  void this.autoNameRun(run.id, skillHint, input.task);
428
- this.pendingJobs.set(run.id, { workflow, input });
599
+ this.pendingJobs.set(run.id, { workflow, input: effectiveInput });
429
600
  this.queue.push(run.id);
430
601
  void this.pump();
431
602
  return run;
@@ -494,6 +665,7 @@ export class RunManager {
494
665
  */
495
666
  async pump() {
496
667
  this.reconcileMonitoringWakeTimers();
668
+ this.reconcileAutoResumes();
497
669
  // A pump requested while one is in flight can't just be dropped: the
498
670
  // in-flight pass may already have read capacity (it awaits `getRepoInfo`
499
671
  // before the first check), so a slot freed in that window would be lost
@@ -518,10 +690,46 @@ export class RunManager {
518
690
  const capacity = () => this.semaphore.busy() < maxParallel &&
519
691
  this.busySlots() < projectMax &&
520
692
  (repo !== null || this.busySlots() < 1);
693
+ // The usage-limit hold (spec 2026-08-03-auto-resume-after-usage-limit).
694
+ //
695
+ // A limit closes an ACCOUNT, not a run — so starting the next queued task walks it into
696
+ // the same wall. Measured before this gate existed: eight tasks under `maxParallel: 2`
697
+ // all failed within 517 ms, each spawning a CLI (and, outside worktree-opt-out mode, a
698
+ // worktree and a branch) only to be marked `scheduled`. The cap was respected at every
699
+ // instant and was no brake at all, because a doomed run lives ~200 ms.
700
+ //
701
+ // So: while any run on an account is waiting out a limit, nothing new starts on THAT
702
+ // account. Other accounts (a second login, a different backend) keep running — the hold
703
+ // is keyed, not global. The set is derived from the durable records rather than tracked
704
+ // separately, which is what makes it survive a restart, expire on its own, and lift the
705
+ // instant a user cancels a resume.
706
+ // The watchdog's one-shot override — read and cleared here, so a forced sweep never
707
+ // leaks into the next ordinary one.
708
+ const forced = this.forceNextPump;
709
+ this.forceNextPump = false;
710
+ const holds = this.queue.length > 0 && !forced ? this.semaphore.accountHolds() : NO_HOLDS;
711
+ const anyHold = holds.deadline.size > 0 || holds.inFlight.size > 0;
712
+ // Only pay for the config read when something is actually held: a queued record may name
713
+ // no runner, and then the account it would use is the configured default.
714
+ const defaultRunner = anyHold ? (await loadConfig(this.repoRoot)).defaultRunner : undefined;
521
715
  while (this.queue.length > 0 && capacity()) {
522
- const runId = this.queue.shift();
716
+ // FIFO among the runs that CAN start; a held one keeps its place in the queue rather
717
+ // than being dequeued and re-queued (which would churn its position and its record).
718
+ const next = !anyHold
719
+ ? 0
720
+ : this.queue.findIndex((id) => {
721
+ const queued = this.store.getRun(id);
722
+ return !queued || !accountHeldFor(queued, holds, defaultRunner ?? 'claude');
723
+ });
724
+ if (next === -1)
725
+ break; // everything queued is waiting on a held account
726
+ const runId = this.queue.splice(next, 1)[0];
523
727
  if (!runId)
524
728
  break;
729
+ // A forced sweep has to reach the spawn: the gate inside `execute` asks the same
730
+ // question and would send this run straight back to the queue.
731
+ if (forced)
732
+ this.forceStarted.add(runId);
525
733
  const job = this.pendingJobs.get(runId);
526
734
  const continuation = this.pendingContinuations.get(runId);
527
735
  this.pendingJobs.delete(runId);
@@ -572,6 +780,87 @@ export class RunManager {
572
780
  this.pumping = false;
573
781
  }
574
782
  }
783
+ /**
784
+ * Make one `queued` RECORD executable again — the engine half a queued run needs but does not
785
+ * persist (`pendingJobs` / `pendingContinuations` are process-local, the record is not).
786
+ *
787
+ * Two callers, one path: boot recovery re-adopts everything the previous process was holding,
788
+ * and the queue watchdog re-adopts anything the running process has somehow lost. A queued
789
+ * record with no work item behind it is invisible to `pump()` and would sit there for good,
790
+ * which is the worst failure this engine has — the task is neither running nor failed, just
791
+ * silently never going to happen.
792
+ *
793
+ * A continuation is reconstructed first: its executable details are gone, but the pending
794
+ * `continue-N` step and the session before it are durable, which is enough. Otherwise the
795
+ * workflow is revived from the record. A run that can be neither is failed loudly rather than
796
+ * left in the queue as a ghost.
797
+ */
798
+ async reviveQueuedRun(run, reason) {
799
+ const queuedContinuation = [...run.steps]
800
+ .reverse()
801
+ .find((step) => step.status === 'pending' && step.id.startsWith('continue-'));
802
+ const sessionStep = queuedContinuation
803
+ ? [...run.steps].reverse().find((step) => step.id !== queuedContinuation.id && step.sessionId)
804
+ : undefined;
805
+ if (queuedContinuation && sessionStep?.sessionId) {
806
+ const backend = run.runner ?? 'claude';
807
+ const sessionBackend = sessionStep.backend ?? backend;
808
+ this.pendingContinuations.set(run.id, {
809
+ stepId: queuedContinuation.id,
810
+ sessionId: sessionBackend === backend ? sessionStep.sessionId : undefined,
811
+ backend,
812
+ prompt: RESTART_CONTINUATION_PROMPT,
813
+ images: [],
814
+ });
815
+ this.queue.push(run.id);
816
+ this.store.appendEvent(run.id, {
817
+ type: 'lifecycle',
818
+ message: `${reason} — interrupted continuation re-queued`,
819
+ });
820
+ return;
821
+ }
822
+ const workflow = await this.reviveWorkflow(run);
823
+ if (!workflow) {
824
+ this.store.updateRun(run.id, {
825
+ status: 'failed',
826
+ error: 'interrupted — workflow definition not recoverable after a restart',
827
+ finishedAt: new Date().toISOString(),
828
+ });
829
+ this.store.appendEvent(run.id, {
830
+ type: 'lifecycle',
831
+ message: `${reason} — workflow definition not recoverable, task failed`,
832
+ });
833
+ return;
834
+ }
835
+ // Re-apply the inbox ceiling (#471). `execute()` gates again at spawn time, so the agent is
836
+ // safe either way — but a run queued while the inbox was on and recovered after it was
837
+ // switched off would otherwise keep echoing `generateFollowups: true` on a run that
838
+ // demonstrably produced none. Normalize the record, the way startRun does.
839
+ const generateFollowups = followupsEnabled() ? run.generateFollowups : false;
840
+ if (generateFollowups !== run.generateFollowups) {
841
+ this.store.updateRun(run.id, { generateFollowups });
842
+ }
843
+ this.pendingJobs.set(run.id, {
844
+ workflow,
845
+ // Folded through the same helper `pump()` uses (#472) so a restart carries the stack.
846
+ // Idempotent: hydration always composes from `run.task` + the stack, never from an
847
+ // already-folded `input.task`, so re-hydrating at dequeue yields the same string.
848
+ input: this.hydrateQueuedInput(run.id, {
849
+ task: run.task,
850
+ model: run.model,
851
+ runner: run.runner,
852
+ generateFollowups,
853
+ // Re-thread autonomy (#489): the rebuilt input feeds `execute`, whose mid-run auto-nudge
854
+ // reads `input.autonomous`. Without this a recovered autonomous run would run
855
+ // non-autonomously and later wrongly park at `review`.
856
+ autonomous: run.autonomous,
857
+ // Preserve an explicit worktree opt-out across a queued restart.
858
+ worktree: run.worktree,
859
+ }),
860
+ });
861
+ this.queue.push(run.id);
862
+ this.store.appendEvent(run.id, { type: 'lifecycle', message: `${reason} — task re-queued` });
863
+ }
575
864
  /**
576
865
  * Startup recovery (#367) — re-adopt runs that were live when the previous
577
866
  * cezar process exited (requires the store opened with `keepLive`):
@@ -588,83 +877,13 @@ export class RunManager {
588
877
  .listRuns()
589
878
  .filter((r) => ['queued', 'waiting', 'running'].includes(r.status))
590
879
  .sort((a, b) => a.createdAt.localeCompare(b.createdAt));
880
+ // A crash never reaches `dropActive`, so its temp directory (#785) outlived the run.
881
+ // Startup is the one moment we know which runs are still live, so sweep every other
882
+ // per-run directory here — bounded to `<dataDir>/tmp`, never a sibling.
883
+ sweepAgentTmpDirs(this.dataDir, live.map((r) => r.id));
591
884
  for (const run of live) {
592
885
  if (run.status === 'queued') {
593
- // A second restart can find a continuation that the first restart put
594
- // behind a capacity gate. Its executable details are process-local,
595
- // but the queued continue step and preceding provider session are
596
- // durable, so reconstruct that job before considering workflow revival.
597
- const queuedContinuation = [...run.steps]
598
- .reverse()
599
- .find((step) => step.status === 'pending' && step.id.startsWith('continue-'));
600
- const sessionStep = queuedContinuation
601
- ? [...run.steps]
602
- .reverse()
603
- .find((step) => step.id !== queuedContinuation.id && step.sessionId)
604
- : undefined;
605
- if (queuedContinuation && sessionStep?.sessionId) {
606
- const backend = run.runner ?? 'claude';
607
- const sessionBackend = sessionStep.backend ?? backend;
608
- this.pendingContinuations.set(run.id, {
609
- stepId: queuedContinuation.id,
610
- sessionId: sessionBackend === backend ? sessionStep.sessionId : undefined,
611
- backend,
612
- prompt: RESTART_CONTINUATION_PROMPT,
613
- images: [],
614
- });
615
- this.queue.push(run.id);
616
- this.store.appendEvent(run.id, {
617
- type: 'lifecycle',
618
- message: 'cezar restarted — interrupted continuation re-queued',
619
- });
620
- continue;
621
- }
622
- const workflow = await this.reviveWorkflow(run);
623
- if (workflow) {
624
- // Re-apply the inbox ceiling (#471). `execute()` gates again at spawn time, so the
625
- // agent is safe either way — but a run queued while the inbox was on and recovered
626
- // after it was switched off would otherwise keep echoing `generateFollowups: true`
627
- // on a run that demonstrably produced none. Normalize the record, the way startRun
628
- // does, so the stored answer matches what actually happens.
629
- const generateFollowups = followupsEnabled() ? run.generateFollowups : false;
630
- if (generateFollowups !== run.generateFollowups) {
631
- this.store.updateRun(run.id, { generateFollowups });
632
- }
633
- this.pendingJobs.set(run.id, {
634
- workflow,
635
- // Folded through the same helper `pump()` uses (#472) so a restart
636
- // carries the stack. Idempotent: hydration always composes from
637
- // `run.task` + the stack, never from an already-folded `input.task`,
638
- // so re-hydrating at dequeue yields the same string, not a doubled one.
639
- input: this.hydrateQueuedInput(run.id, {
640
- task: run.task,
641
- model: run.model,
642
- runner: run.runner,
643
- generateFollowups,
644
- // Re-thread autonomy (#489): the rebuilt input feeds `execute`,
645
- // whose mid-run auto-nudge reads `input.autonomous`. Without this a
646
- // recovered queued autonomous run would run non-autonomously (no
647
- // auto-nudge) and later wrongly park at `review`.
648
- autonomous: run.autonomous,
649
- // Preserve an explicit worktree opt-out across a queued restart.
650
- // Missing on older records means the default isolated mode.
651
- worktree: run.worktree,
652
- }),
653
- });
654
- this.queue.push(run.id);
655
- this.store.appendEvent(run.id, { type: 'lifecycle', message: 'cezar restarted — task re-queued' });
656
- }
657
- else {
658
- this.store.updateRun(run.id, {
659
- status: 'failed',
660
- error: 'interrupted — workflow definition not recoverable after a restart',
661
- finishedAt: new Date().toISOString(),
662
- });
663
- this.store.appendEvent(run.id, {
664
- type: 'lifecycle',
665
- message: 'cezar restarted — workflow definition not recoverable, task failed',
666
- });
667
- }
886
+ await this.reviveQueuedRun(run, 'cezar restarted');
668
887
  continue;
669
888
  }
670
889
  if (run.status === 'waiting') {
@@ -704,6 +923,11 @@ export class RunManager {
704
923
  : `cezar restarted — could not resume the interrupted task (${resumed.error ?? 'unknown'})`,
705
924
  });
706
925
  }
926
+ // Re-arm usage-limit resumes (spec 2026-08-03-auto-resume-after-usage-limit): the wait is
927
+ // routinely longer than a cezar session, so the deadline is durable and the timer is rebuilt
928
+ // from it. `pump()` reconciles again on every sweep, so this is the fast path, not the only
929
+ // one — see `reconcileAutoResumes`.
930
+ this.reconcileAutoResumes();
707
931
  void this.pump();
708
932
  }
709
933
  /** The persisted definition when it looks sane, else the catalog by name. */
@@ -730,9 +954,20 @@ export class RunManager {
730
954
  this.active.delete(runId);
731
955
  this.memoryPausing.delete(runId);
732
956
  this.lastNamerKey.delete(runId);
957
+ this.forceStarted.delete(runId);
733
958
  // The run's slot is gone from busySlots() as of the deletes above — hand it
734
959
  // to the workspace's oldest queued run, in ANY project. Every terminal path
735
960
  // funnels through here, so this one call covers them all.
961
+ // Same reasoning as retention below — every terminal path funnels through here, so the
962
+ // usage-limit question ("did this run stop because the account is out of window, and when
963
+ // does that window reopen?") is asked once, in one place, off the record the failing path
964
+ // has already written. Nothing to do for any other outcome.
965
+ //
966
+ // BEFORE releasing the slot, and that order is the whole point: `releaseSlot` pumps every
967
+ // manager, and a pump reads the hold off the records. Publishing the schedule afterwards
968
+ // left a window — measured as exactly one extra task — where the queue saw a free slot and
969
+ // an account that looked healthy, and started work that was already doomed.
970
+ this.scheduleAutoResumeIfLimited(runId);
736
971
  this.releaseSlot();
737
972
  // A run leaving the active registry is a terminal transition (done/review/
738
973
  // failed/cancelled) — the one moment the finished-worktree count can grow.
@@ -740,6 +975,338 @@ export class RunManager {
740
975
  // terminal path. Fire-and-forget: retention must never delay or throw into
741
976
  // the lifecycle.
742
977
  void this.enforceRetention();
978
+ // The run's temp directory (#785) goes on the same terminal transition, and
979
+ // unconditionally — it is scratch, not an artifact, so unlike a worktree
980
+ // there is no keep-count to respect and nothing left to recover from it. A
981
+ // Continue (or an auto-resume) re-creates it through `agentEnv`.
982
+ removeAgentTmpDir(this.dataDir, runId);
983
+ }
984
+ // ---- usage-limit auto-resume (spec 2026-08-03-auto-resume-after-usage-limit) --------------
985
+ /**
986
+ * A run just failed: if the provider said "usage limit, back at T", promise to resume it at
987
+ * `T + AUTO_RESUME_GRACE_MS` instead of leaving the task dead until someone notices.
988
+ *
989
+ * Every refusal below is silent-but-honest — the run stays `failed` with its Continue button,
990
+ * which is exactly the pre-feature behavior — except the safety cap, which says so on the
991
+ * transcript, because a run that stops resuming itself needs to explain why.
992
+ */
993
+ scheduleAutoResumeIfLimited(runId) {
994
+ if (this.autoResumeTimers.has(runId))
995
+ return; // already promised
996
+ const run = this.store.getRun(runId);
997
+ if (!run || run.status !== 'failed')
998
+ return;
999
+ // Archiving IS resigning from a task. Reviving one because a window happened to reopen would
1000
+ // be the feature working against the clearest signal the user can give it.
1001
+ if (run.archived)
1002
+ return;
1003
+ const limit = parseUsageLimit(run.error);
1004
+ if (!limit)
1005
+ return;
1006
+ if (!this.semaphore.autoResumeOnUsageLimit())
1007
+ return;
1008
+ // No session to resume = nothing this feature can do; `continueRun` would refuse anyway.
1009
+ if (!run.steps.some((step) => step.sessionId))
1010
+ return;
1011
+ const attempts = run.autoResumeAttempts ?? 0;
1012
+ if (attempts >= MAX_AUTO_RESUMES) {
1013
+ this.store.appendEvent(runId, {
1014
+ type: 'note',
1015
+ message: `automatic resume cap reached (${MAX_AUTO_RESUMES}) — continue this task manually`,
1016
+ });
1017
+ return;
1018
+ }
1019
+ const wakeAt = new Date(limit.resetAt.getTime() + AUTO_RESUME_GRACE_MS);
1020
+ this.armAutoResume(runId, wakeAt.getTime());
1021
+ this.store.appendEvent(runId, {
1022
+ type: 'lifecycle',
1023
+ message: `usage limit reached — resuming automatically at ${formatWakeInstant(wakeAt)}`,
1024
+ });
1025
+ }
1026
+ /** Publish the deadline on the record (the cockpit's only source) and arm the timer for it. */
1027
+ armAutoResume(runId, deadline) {
1028
+ this.store.updateRun(runId, { autoResumeAt: new Date(deadline).toISOString() });
1029
+ const timer = setTimeout(() => this.fireAutoResume(runId), Math.max(0, deadline - Date.now()));
1030
+ timer.unref?.();
1031
+ this.autoResumeTimers.set(runId, timer);
1032
+ }
1033
+ /**
1034
+ * The window has reopened. Re-check the record synchronously — hours may have passed, and the
1035
+ * user may have continued, deleted or cancelled the run in them — then hand the resume to the
1036
+ * ordinary queued-continuation path so it obeys both concurrency caps like any other work.
1037
+ */
1038
+ fireAutoResume(runId) {
1039
+ this.autoResumeTimers.delete(runId);
1040
+ const run = this.store.getRun(runId);
1041
+ if (!run || run.status !== 'failed' || !run.autoResumeAt)
1042
+ return;
1043
+ // Belt and braces against the one gap `reconcileAutoResumes` cannot close: the setting going
1044
+ // off in the window between the last pump and this tick.
1045
+ if (!this.semaphore.autoResumeOnUsageLimit()) {
1046
+ this.clearAutoResume(runId);
1047
+ return;
1048
+ }
1049
+ const attempts = (run.autoResumeAttempts ?? 0) + 1;
1050
+ // `continueRun` retires the pending resume (timer + record fields) on the way in — this is a
1051
+ // resume, not a user turn, so the counter is put back straight after.
1052
+ const resumed = this.continueRun(runId, { text: AUTO_RESUME_PROMPT }, true);
1053
+ if (!resumed.ok) {
1054
+ // Refusals happen before `continueRun` retires anything, so the deadline is still on the
1055
+ // record — and a deadline in the past is a promise the cockpit keeps displaying and the
1056
+ // engine will never keep. Retire it here instead, and say why.
1057
+ this.clearAutoResume(runId);
1058
+ this.store.appendEvent(runId, {
1059
+ type: 'note',
1060
+ message: `automatic resume could not start — ${resumed.error ?? 'unknown'}`,
1061
+ });
1062
+ return;
1063
+ }
1064
+ this.store.updateRun(runId, { autoResumeAttempts: attempts });
1065
+ this.store.appendEvent(runId, {
1066
+ type: 'lifecycle',
1067
+ message: `usage limit reset — resuming automatically (${attempts}/${MAX_AUTO_RESUMES})`,
1068
+ });
1069
+ // A deferred continuation only ENQUEUES itself; the queue moves when something pumps it, and
1070
+ // `recover()` — the other deferring caller — pumps once after its whole bulk sweep. A timer
1071
+ // firing on its own has no such follow-up, so without this the resumed run sits at `queued`
1072
+ // until some unrelated run happens to finish. This is the pump for it.
1073
+ void this.pump();
1074
+ }
1075
+ /**
1076
+ * Make the armed timers agree with the records and the current setting. Runs on every `pump()`
1077
+ * — which is where a settings change lands (a config PUT refreshes the shared semaphore, which
1078
+ * pumps every manager) — and once from `recover()`.
1079
+ *
1080
+ * It is a RECONCILE rather than a one-shot restore because the deadline is durable state and
1081
+ * the timer is not: a restart, a rebuilt project context, a manager disposed mid-wait, or a
1082
+ * refusal all leave a record promising a resume that no timer is holding. Rebuilding from the
1083
+ * record covers every one of those at once — the alternative is a hint counting down to a time
1084
+ * that has already passed, which is exactly the failure this method exists to make impossible.
1085
+ *
1086
+ * Cheap: an in-memory scan, and arming is skipped for every run already held.
1087
+ */
1088
+ reconcileAutoResumes() {
1089
+ if (!this.semaphore.autoResumeOnUsageLimit()) {
1090
+ // Sweep the RECORDS, not the timer map. A record promising a resume that no timer is
1091
+ // holding is the exact population this method exists for, and it is also the one the
1092
+ // setting can be switched off in front of: cezar restarted while it was off, the config
1093
+ // was hand-edited, or the project context was disposed mid-wait. Retiring only the armed
1094
+ // timers leaves such a record with a live `autoResumeAt`, which `accountHolds()` reads as
1095
+ // a deadline hold — so nothing new starts on that account, `rescueStalledQueue` treats the
1096
+ // phantom appointment as a legitimate reason to sit still, and the cockpit shows a
1097
+ // `scheduled` row for a resume that will never come. `clearAutoResume` covers the armed
1098
+ // ones too, so this one loop is the whole cancellation.
1099
+ const pending = new Set([
1100
+ ...this.autoResumeTimers.keys(),
1101
+ ...this.store.listRuns().filter((run) => run.autoResumeAt !== undefined).map((run) => run.id),
1102
+ ]);
1103
+ for (const runId of pending) {
1104
+ this.clearAutoResume(runId);
1105
+ this.store.appendEvent(runId, {
1106
+ type: 'note',
1107
+ message: 'automatic resume cancelled — auto-resume is switched off',
1108
+ });
1109
+ }
1110
+ return;
1111
+ }
1112
+ for (const run of this.store.listRuns()) {
1113
+ if (run.status !== 'failed' || !run.autoResumeAt)
1114
+ continue;
1115
+ if (this.autoResumeTimers.has(run.id))
1116
+ continue;
1117
+ const deadline = Date.parse(run.autoResumeAt);
1118
+ // A deadline that is unreadable, belongs to a run that has spent its cap, or belongs to a
1119
+ // task the user has archived is retired rather than re-armed: it can only mislead. One
1120
+ // that has just passed arms at zero — the window is open, which is the point.
1121
+ if (run.archived
1122
+ || !Number.isFinite(deadline)
1123
+ || (run.autoResumeAttempts ?? 0) >= MAX_AUTO_RESUMES) {
1124
+ this.store.updateRun(run.id, { autoResumeAt: undefined });
1125
+ continue;
1126
+ }
1127
+ // …and one missed by more than a day is retired loudly: reviving a task from another era
1128
+ // is a surprise, not a service, and this is what keeps a sweep from resurrecting every
1129
+ // limit-stopped task a user has long since walked away from.
1130
+ if (Date.now() - deadline > AUTO_RESUME_MISSED_WINDOW_MS) {
1131
+ this.store.updateRun(run.id, { autoResumeAt: undefined });
1132
+ this.store.appendEvent(run.id, {
1133
+ type: 'note',
1134
+ message: 'automatic resume expired — its window reopened over a day ago; continue this task manually',
1135
+ });
1136
+ continue;
1137
+ }
1138
+ this.armAutoResume(run.id, deadline);
1139
+ }
1140
+ }
1141
+ /**
1142
+ * Hand a run that has not spawned anything back to the queue, when the account it would run on
1143
+ * went into a usage-limit hold (spec 2026-08-03-auto-resume-after-usage-limit).
1144
+ *
1145
+ * The dequeue-time gate in `pump()` cannot be the only one: a run can sit between dequeue and
1146
+ * spawn for a long time — an in-place run waiting for the exclusive repo-root lease is the
1147
+ * measured case — and the account can close in that gap. This is the last honest moment to
1148
+ * refuse, because everything after it costs a real agent turn.
1149
+ *
1150
+ * "Untouched" is the contract: the run has created no session and no worktree, so it goes back
1151
+ * as plain `queued` with its `startedAt` cleared, and `pump()` will pick it up when the window
1152
+ * reopens. Returns true when the caller must abandon the run.
1153
+ */
1154
+ requeueWhileHeld(runId, workflow, input, runner, state) {
1155
+ const run = this.store.getRun(runId);
1156
+ if (!run || run.status === 'cancelled' || state?.cancelled)
1157
+ return false;
1158
+ // The watchdog sent this one through. Checked, never consumed: the spawn path asks this
1159
+ // question TWICE — here at the top of `execute`, and again after the exclusive repo-root
1160
+ // lease is granted — so a one-shot flag would clear at the first gate and let the second one
1161
+ // hand an in-place run straight back, re-wedging the queue the rescue had just freed.
1162
+ // `dropActive` retires the entry on every terminal path, so the set still cleans itself up.
1163
+ if (this.forceStarted.has(runId))
1164
+ return false;
1165
+ if (!accountHeldFor({ ...run, runner }, this.semaphore.accountHolds(), runner))
1166
+ return false;
1167
+ state?.releaseRepoRoot?.();
1168
+ if (state)
1169
+ state.releaseRepoRoot = undefined;
1170
+ this.pendingJobs.set(runId, { workflow, input });
1171
+ this.queue.push(runId);
1172
+ this.store.updateRun(runId, { status: 'queued', startedAt: undefined, currentStepId: undefined });
1173
+ this.store.appendEvent(runId, {
1174
+ type: 'note',
1175
+ message: 'held in the queue — this agent account is waiting out a usage limit',
1176
+ });
1177
+ this.dropActive(runId);
1178
+ return true;
1179
+ }
1180
+ /**
1181
+ * The failsafe: a queue must never be able to wedge.
1182
+ *
1183
+ * Everything else in this file makes an idle queue CORRECT under some condition — a slot cap, a
1184
+ * repo-root lease, and now a usage-limit hold. That is also what makes a wedged queue look
1185
+ * correct, and the hold has already produced one in the field: two resumes fired together, each
1186
+ * holding the account the other was waiting on, and the whole workspace stopped with every task
1187
+ * `queued`. That specific bug is fixed and tested, but "the queue stopped and nothing will ever
1188
+ * restart it" is too expensive a failure mode to leave resting on any single fix being right.
1189
+ *
1190
+ * The test is deliberately about JUSTIFICATION rather than about any particular bug: idling is
1191
+ * legitimate while work is running (here or in another project), or while a real appointment is
1192
+ * still ahead — a scheduled resume that will fire and pump on its own. Anything else is a
1193
+ * queue with work in it, nothing running anywhere, and no event coming to wake it. That gets one
1194
+ * forced sweep, which starts work under the ordinary caps and lets the account's real state
1195
+ * re-assert itself: if the window truly is shut, that task meets the limit and re-establishes an
1196
+ * honest hold, with a real deadline behind it this time.
1197
+ *
1198
+ * Public so a test can drive the wedge directly instead of waiting out the interval.
1199
+ */
1200
+ async rescueStalledQueue(now = Date.now()) {
1201
+ // First, the worst shape: a record that says `queued` while the engine holds no job, no
1202
+ // continuation and no queue entry for it. `pump()` cannot see such a run — it iterates the
1203
+ // queue, and this one is not in it — so nothing will ever start it. Re-adopt it through the
1204
+ // same path boot recovery uses.
1205
+ for (const run of this.store.listRuns()) {
1206
+ if (run.status !== 'queued')
1207
+ continue;
1208
+ if (this.active.has(run.id) || this.starting.has(run.id))
1209
+ continue;
1210
+ if (this.pendingJobs.has(run.id) || this.pendingContinuations.has(run.id))
1211
+ continue;
1212
+ if (this.queue.includes(run.id))
1213
+ continue;
1214
+ console.warn(`[cez] queue watchdog: re-adopting queued run ${run.id} the engine had lost`);
1215
+ await this.reviveQueuedRun(run, 'queue watchdog');
1216
+ }
1217
+ if (this.queue.length === 0)
1218
+ return;
1219
+ if (this.busySlots() > 0 || this.starting.size > 0)
1220
+ return;
1221
+ if (this.semaphore.busy() > 0)
1222
+ return;
1223
+ // A future deadline is a real reason to sit still: that timer will fire and pump.
1224
+ for (const run of this.store.listRuns()) {
1225
+ if (run.status !== 'failed' || !run.autoResumeAt)
1226
+ continue;
1227
+ const deadline = Date.parse(run.autoResumeAt);
1228
+ if (Number.isFinite(deadline) && deadline > now)
1229
+ return;
1230
+ }
1231
+ if (this.semaphore.accountHolds().inFlight.size === 0) {
1232
+ // Not the hold, then — some other wakeup went missing. An ordinary pump is the whole fix,
1233
+ // and it is idempotent, so this stays quiet.
1234
+ void this.pump();
1235
+ return;
1236
+ }
1237
+ console.warn('[cez] queue watchdog: work is queued, nothing is running, and the usage-limit hold has no'
1238
+ + ' deadline behind it — starting the next task anyway');
1239
+ this.forceNextPump = true;
1240
+ void this.pump();
1241
+ }
1242
+ /**
1243
+ * The accounts this project is currently holding: one key per run parked on a usage-limit
1244
+ * resume that has not come due yet (spec 2026-08-03-auto-resume-after-usage-limit).
1245
+ *
1246
+ * Published to the shared semaphore so the hold spans PROJECTS — one Claude account can be
1247
+ * driving tasks in three repos, and a limit closes it for all of them. Derived from the
1248
+ * records on every ask rather than tracked as state: a deadline that passes, a resume that
1249
+ * fires, a cancel, an archive and a delete all lift the hold with no bookkeeping.
1250
+ *
1251
+ * Deliberately excludes a deadline that has already passed — that run is about to resume, and
1252
+ * holding the queue for it would only stall the very work the window reopened for.
1253
+ */
1254
+ accountHolds(now = Date.now()) {
1255
+ const deadline = new Set();
1256
+ const inFlight = new Set();
1257
+ for (const run of this.store.listRuns()) {
1258
+ // A holding run always carries the runner it actually ran on, so the fallback is unused
1259
+ // here — it is spelled out rather than `!` so a future record shape degrades, not throws.
1260
+ const key = () => runAccountKey(run, run.runner ?? 'claude');
1261
+ if (run.status === 'failed' && run.autoResumeAt) {
1262
+ const at = Date.parse(run.autoResumeAt);
1263
+ if (Number.isFinite(at) && at > now)
1264
+ deadline.add(key());
1265
+ }
1266
+ else if (resumeInFlight(run)) {
1267
+ inFlight.add(key());
1268
+ }
1269
+ }
1270
+ return { deadline, inFlight };
1271
+ }
1272
+ /**
1273
+ * The PER-TASK off switch (`DELETE /api/v1/runs/:id/auto-resume`, and the archive route):
1274
+ * stop resuming THIS task, without touching the workspace setting or any other task.
1275
+ *
1276
+ * Idempotent — a run with nothing pending answers the same way, because "this task will not
1277
+ * resume itself" is equally true either way. Returns false only when the run does not exist,
1278
+ * which is the route's 404.
1279
+ */
1280
+ cancelAutoResume(runId) {
1281
+ const run = this.store.getRun(runId);
1282
+ if (!run)
1283
+ return false;
1284
+ const pending = run.autoResumeAt !== undefined || this.autoResumeTimers.has(runId);
1285
+ this.clearAutoResume(runId);
1286
+ if (pending) {
1287
+ this.store.appendEvent(runId, {
1288
+ type: 'note',
1289
+ message: 'automatic resume cancelled for this task',
1290
+ });
1291
+ // This run may have been the last thing holding its account's queue — nothing else will
1292
+ // notice, since the hold is derived and its release is not an event.
1293
+ void this.pump();
1294
+ }
1295
+ return true;
1296
+ }
1297
+ /** Retire a pending resume — timer, deadline and counter. The counter goes too because every
1298
+ * caller is a fresh epoch: a human Continue, or a resume that re-stamps its own count. */
1299
+ clearAutoResume(runId) {
1300
+ const timer = this.autoResumeTimers.get(runId);
1301
+ if (timer)
1302
+ clearTimeout(timer);
1303
+ this.autoResumeTimers.delete(runId);
1304
+ const run = this.store.getRun(runId);
1305
+ if (!run)
1306
+ return;
1307
+ if (run.autoResumeAt !== undefined || run.autoResumeAttempts !== undefined) {
1308
+ this.store.updateRun(runId, { autoResumeAt: undefined, autoResumeAttempts: undefined });
1309
+ }
743
1310
  }
744
1311
  /** Reclaim finished worktrees beyond the keep-limit (#483) — directory only,
745
1312
  * `cez/<id8>` branch kept. Best-effort; a failure never affects run
@@ -1201,6 +1768,9 @@ export class RunManager {
1201
1768
  /** Restart recovery may discover several interrupted tasks at once. Those
1202
1769
  * continuations are queued; an explicit user Continue remains immediate. */
1203
1770
  deferForCapacity = false) {
1771
+ if (agentModelsLocked(this.repoRoot) && opts.model?.trim()) {
1772
+ return { ok: false, error: AGENT_MODELS_LOCKED_ERROR };
1773
+ }
1204
1774
  if (this.active.has(runId))
1205
1775
  return { ok: false, error: 'run is still active' };
1206
1776
  const run = this.store.getRun(runId);
@@ -1251,6 +1821,11 @@ export class RunManager {
1251
1821
  : {}),
1252
1822
  });
1253
1823
  }
1824
+ // Everything that could refuse this continuation has now passed, so a pending usage-limit
1825
+ // resume is superseded either way: this IS that resume (it re-stamps its own counter), or a
1826
+ // human got there first — and then the counter starts over, because the cap only exists to
1827
+ // bound UNATTENDED resumes.
1828
+ this.clearAutoResume(runId);
1254
1829
  const continuations = run.steps.filter((s) => s.id.startsWith('continue-')).length;
1255
1830
  const stepId = `continue-${continuations + 1}`;
1256
1831
  this.store.addStep(runId, { id: stepId, name: 'Continue', kind: 'agent' });
@@ -1338,6 +1913,11 @@ export class RunManager {
1338
1913
  this.armAutosave(state);
1339
1914
  if (record)
1340
1915
  seedHandoffFile(this.dataDir, record); // idempotent — normally already there
1916
+ // Registry snapshot for `/skill` expansion. `execute` loads this for the workflow's own
1917
+ // sessions; a continuation builds its OWN ActiveRun, and without this the resumed session
1918
+ // expanded against an empty registry and leaked `/om-...` verbatim to the backend, which
1919
+ // answered "Unknown skill" (#811). Best-effort — discovery must never break Continue.
1920
+ state.skills = await discoverSkills(this.repoRoot).catch(() => []);
1341
1921
  this.store.updateRun(runId, {
1342
1922
  status: 'running',
1343
1923
  error: undefined,
@@ -1353,17 +1933,6 @@ export class RunManager {
1353
1933
  backend,
1354
1934
  });
1355
1935
  this.store.appendEvent(runId, { type: 'step-start', stepId, name: 'Continue', kind: 'agent', iteration: 1 });
1356
- // Registry snapshot for `/skill` expansion. `execute` loads this for the workflow's own
1357
- // sessions; a continuation builds its OWN ActiveRun, and without this the resumed session
1358
- // expanded against an empty registry and leaked `/om-...` verbatim to the backend, which
1359
- // answered "Unknown skill" (#811). Best-effort — discovery must never break Continue.
1360
- //
1361
- // Loaded AFTER the run is marked `running`, not before: `continueRun` returns the moment it
1362
- // schedules this, so every await ahead of that write is a window in which the run still
1363
- // reads with its previous TERMINAL status. A directory scan there is long enough to be
1364
- // observed. Nothing between here and `startSession` reads `state.skills`, and no message can
1365
- // be delivered into a session that does not exist yet, so the invariant is unaffected.
1366
- state.skills = await discoverSkills(this.repoRoot).catch(() => []);
1367
1936
  // Attachments pasted into the follow-up composer, on the same terms as a live-session
1368
1937
  // message (#357): persisted to the run's own image store so the thread renders the bubble's
1369
1938
  // images rather than a bare count, and handed to the agent BOTH as base64 blocks (so it can
@@ -1384,6 +1953,7 @@ export class RunManager {
1384
1953
  });
1385
1954
  let stepCost = 0;
1386
1955
  let turnText = '';
1956
+ let sessionError;
1387
1957
  const sink = this.makeUiSink(runId, stepId);
1388
1958
  const onEvent = (event) => {
1389
1959
  if (event.type === 'image') {
@@ -1400,6 +1970,13 @@ export class RunManager {
1400
1970
  return;
1401
1971
  }
1402
1972
  this.store.appendEvent(runId, { ...event, stepId });
1973
+ if (event.type === 'error') {
1974
+ sessionError ??= event.message;
1975
+ state.session?.interrupt();
1976
+ return;
1977
+ }
1978
+ if (sessionError)
1979
+ return;
1403
1980
  if (event.type === 'session') {
1404
1981
  this.store.updateStep(runId, stepId, { sessionId: event.sessionId, backend });
1405
1982
  }
@@ -1478,21 +2055,51 @@ export class RunManager {
1478
2055
  this.releaseSlot();
1479
2056
  }
1480
2057
  }
2058
+ // A turn that completed is the ONLY evidence the provider's window actually reopened, so
2059
+ // it is what retires the consecutive-resume counter — which in turn releases the account
2060
+ // hold for every other task queued behind it (spec
2061
+ // 2026-08-03-auto-resume-after-usage-limit). `settleSuccess` does the same for a run that
2062
+ // finishes outright; this covers the far more common "parked for the user" ending.
2063
+ if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
2064
+ this.store.updateRun(runId, { autoResumeAttempts: undefined });
2065
+ }
1481
2066
  appendHandoffHeartbeat(this.dataDir, runId, `turn complete — status=${monitoring ? 'monitoring' : sessionOpen ? 'waiting' : 'running'}`);
1482
2067
  }
1483
2068
  };
1484
2069
  // Backend + model come off the record: the run's current backend by default, or the
1485
2070
  // follow-up override that `continueRun` persisted before scheduling (#401).
1486
2071
  const continueBackend = backend;
2072
+ /** Settle this turn as a failure before anything is spawned — the shape both
2073
+ * pre-spawn gates below need (model identity, #405; temp directory, #785). */
2074
+ const failBeforeSpawn = (message) => {
2075
+ const failedAt = new Date().toISOString();
2076
+ sink.sessionEnded('error', message);
2077
+ this.store.updateStep(runId, stepId, {
2078
+ status: 'failed',
2079
+ error: message,
2080
+ finishedAt: failedAt,
2081
+ });
2082
+ this.store.updateRun(runId, {
2083
+ status: 'failed',
2084
+ error: `continue failed: ${message}`,
2085
+ finishedAt: failedAt,
2086
+ currentStepId: undefined,
2087
+ });
2088
+ this.store.appendEvent(runId, {
2089
+ type: 'lifecycle',
2090
+ message: `continue failed — ${message}`,
2091
+ });
2092
+ this.dropActive(runId);
2093
+ };
1487
2094
  // Apply the SAME canonical-identity gate the first spawn applies (#405, review M1).
1488
2095
  // A follow-up may switch both runner and model (#401), so without this the record keeps
1489
2096
  // asserting the identity the run STARTED with while a different model serves the turn —
1490
- // the exact defect this PR exists to remove — and the raw record string reaches the CLI
2097
+ // the exact defect that PR existed to remove — and the raw record string reaches the CLI
1491
2098
  // in the un-normalised wire form the first step already converted away (`anthropic/opus`
1492
2099
  // instead of `opus`). Fail loud here too rather than let the backend pick a default.
1493
2100
  let continueModel;
1494
2101
  try {
1495
- const normalized = normalizeModelForBackend(continueBackend, record?.model);
2102
+ const normalized = normalizeModelForBackend(continueBackend, agentModelsLocked(this.repoRoot) ? undefined : record?.model, { configuredProvider: await configuredModelProvider(continueBackend, state.cwd) });
1496
2103
  continueModel = normalized?.backendModel;
1497
2104
  this.store.updateRun(runId, {
1498
2105
  modelIdentity: normalized ? formatModelIdentity(normalized.identity) : undefined,
@@ -1501,26 +2108,32 @@ export class RunManager {
1501
2108
  catch (err) {
1502
2109
  if (!(err instanceof ModelIdentityError))
1503
2110
  throw err;
1504
- const failedAt = new Date().toISOString();
1505
- sink.sessionEnded('error', err.message);
1506
- this.store.updateStep(runId, stepId, {
1507
- status: 'failed',
1508
- error: err.message,
1509
- finishedAt: failedAt,
1510
- });
1511
- this.store.updateRun(runId, {
1512
- status: 'failed',
1513
- error: `continue failed: ${err.message}`,
1514
- finishedAt: failedAt,
1515
- currentStepId: undefined,
1516
- });
1517
- this.store.appendEvent(runId, {
1518
- type: 'lifecycle',
1519
- message: `continue failed — ${err.message}`,
2111
+ failBeforeSpawn(err.message);
2112
+ return;
2113
+ }
2114
+ // Resuming reattaches to a session that lives inside ONE account's config dir, so the
2115
+ // continuation must run under the account that created it — not whatever the project has
2116
+ // been switched to since. The owning step is the one carrying this session id.
2117
+ const resumedProfileId = sessionId === undefined
2118
+ ? undefined
2119
+ : record?.steps.find((s) => s.sessionId === sessionId)?.profileId;
2120
+ // The temp-directory preflight (#785) rides along with the account resolution: a resumed
2121
+ // turn hits the same broken `/tmp` a fresh one would, and an agent whose shell silently
2122
+ // returns nothing is worse than a turn that refuses to start and says why.
2123
+ let continueProfile;
2124
+ try {
2125
+ continueProfile = await this.agentEnvForStep(runId, continueBackend, {
2126
+ generateFollowups,
2127
+ recordedProfileId: resumedProfileId,
1520
2128
  });
1521
- this.dropActive(runId);
2129
+ }
2130
+ catch (err) {
2131
+ if (!(err instanceof AgentTempDirError))
2132
+ throw err;
2133
+ failBeforeSpawn(err.message);
1522
2134
  return;
1523
2135
  }
2136
+ this.store.updateStep(runId, stepId, { profileId: continueProfile.profileId });
1524
2137
  const runner = createRunner(continueBackend);
1525
2138
  state.currentStepId = stepId;
1526
2139
  this.beginUsageInvocation(runId, state, stepId);
@@ -1540,8 +2153,8 @@ export class RunManager {
1540
2153
  ...(openingImages.length ? { images: openingImages } : {}),
1541
2154
  cwd: state.cwd,
1542
2155
  allowedTools: DEFAULT_ALLOWED_TOOLS,
1543
- additionalDirectories: [join(this.dataDir, 'runs')],
1544
- env: this.agentEnv(runId, generateFollowups),
2156
+ additionalDirectories: agentDirectories(join(this.dataDir, 'runs'), continueProfile.env),
2157
+ env: continueProfile.env,
1545
2158
  model: continueModel,
1546
2159
  sessionId,
1547
2160
  resume: sessionId !== undefined,
@@ -1556,6 +2169,8 @@ export class RunManager {
1556
2169
  const finishedAt = () => new Date().toISOString();
1557
2170
  try {
1558
2171
  await session.result;
2172
+ if (sessionError)
2173
+ throw new Error(sessionError);
1559
2174
  sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
1560
2175
  if (state.cancelled) {
1561
2176
  this.store.updateStep(runId, stepId, { status: 'cancelled', finishedAt: finishedAt() });
@@ -1608,6 +2223,12 @@ export class RunManager {
1608
2223
  // the config default. Per-step `runner` can still override it below.
1609
2224
  const config = await loadConfig(this.repoRoot);
1610
2225
  const taskBackend = input.runner ?? config.defaultRunner;
2226
+ // The account may have gone into a usage-limit hold since this run was dequeued — the queue
2227
+ // gate cannot be the only one, because dequeue is not the moment of no return. Nothing has
2228
+ // happened yet here, so the run goes back to the queue untouched (spec
2229
+ // 2026-08-03-auto-resume-after-usage-limit).
2230
+ if (this.requeueWhileHeld(runId, workflow, input, taskBackend))
2231
+ return;
1611
2232
  // Extra system prompt (R2 2.3): POST override > config default; echoed on
1612
2233
  // the record so the UI/API can show what the run actually used.
1613
2234
  const extraSystemPrompt = resolveExtraSystemPrompt(input.systemPrompt, config.systemPrompt);
@@ -1617,7 +2238,7 @@ export class RunManager {
1617
2238
  // can still override below); the authoritative fail-loud gate is at spawn.
1618
2239
  let modelIdentity;
1619
2240
  try {
1620
- const normalized = normalizeModelForBackend(taskBackend, input.model);
2241
+ const normalized = normalizeModelForBackend(taskBackend, agentModelsLocked(this.repoRoot) ? undefined : input.model, { configuredProvider: await configuredModelProvider(taskBackend, this.repoRoot) });
1621
2242
  modelIdentity = normalized ? formatModelIdentity(normalized.identity) : undefined;
1622
2243
  }
1623
2244
  catch {
@@ -1728,6 +2349,15 @@ export class RunManager {
1728
2349
  // run through the usual path.
1729
2350
  await this.acquireRepoRoot(runId, state);
1730
2351
  }
2352
+ // THE window that matters for an in-place run. Waiting for the exclusive tree can take
2353
+ // minutes, and a run parked on that lease holds no slot (#347) — so the queue keeps
2354
+ // advancing behind it and the dequeue-time gate is long past. Measured with five in-place
2355
+ // tasks and `maxParallel: 2`: four of them started. Re-ask here, where the very next thing
2356
+ // is a spawn, and hand the run back to the queue if the account closed meanwhile. This
2357
+ // check also covers the explicit lock-bypass path, where the account may close while the
2358
+ // run is preparing its first step.
2359
+ if (this.requeueWhileHeld(runId, workflow, input, taskBackend, state))
2360
+ return;
1731
2361
  }
1732
2362
  // Handoff journal (spec 007) — seeded after the worktree exists so the
1733
2363
  // header can name the branch. Idempotent: an existing file stays as-is.
@@ -1917,6 +2547,7 @@ export class RunManager {
1917
2547
  const startTokens = stepRecord?.tokensUsed ?? 0;
1918
2548
  let stepCost = stepRecord?.costUsd ?? 0;
1919
2549
  let turnText = '';
2550
+ let sessionError;
1920
2551
  const sink = this.makeUiSink(runId, step.id);
1921
2552
  const onEvent = (event) => {
1922
2553
  if (event.type === 'image') {
@@ -1933,6 +2564,13 @@ export class RunManager {
1933
2564
  return;
1934
2565
  }
1935
2566
  emit({ ...event, stepId: step.id });
2567
+ if (event.type === 'error') {
2568
+ sessionError ??= event.message;
2569
+ state.session?.interrupt();
2570
+ return;
2571
+ }
2572
+ if (sessionError)
2573
+ return;
1936
2574
  if (event.type === 'session') {
1937
2575
  // Codex/OpenCode mint their own session id — persist it so resume works.
1938
2576
  this.store.updateStep(runId, step.id, { sessionId: event.sessionId, backend });
@@ -2001,6 +2639,10 @@ export class RunManager {
2001
2639
  this.armIdleTimer(runId, state);
2002
2640
  this.releaseSlot(); // the freed slot can start a queued run right away — in any project
2003
2641
  }
2642
+ // The window is proven open — see the twin in `runContinuation`.
2643
+ if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
2644
+ this.store.updateRun(runId, { autoResumeAttempts: undefined });
2645
+ }
2004
2646
  // Cez's own heartbeat — the handoff stays current even when the
2005
2647
  // agent forgets to write (spec 007).
2006
2648
  appendHandoffHeartbeat(this.dataDir, runId, `turn complete — status=${monitoring ? 'monitoring' : waiting ? 'waiting' : 'running'}`);
@@ -2013,7 +2655,7 @@ export class RunManager {
2013
2655
  // instead of letting the backend silently substitute its default.
2014
2656
  let backendModel;
2015
2657
  try {
2016
- const normalized = normalizeModelForBackend(stepBackend, step.model ?? input.model);
2658
+ const normalized = normalizeModelForBackend(stepBackend, agentModelsLocked(this.repoRoot) ? undefined : step.model ?? input.model, { configuredProvider: await configuredModelProvider(stepBackend, state.cwd) });
2017
2659
  backendModel = normalized?.backendModel;
2018
2660
  // Persist the identity of what ACTUALLY runs (#405, review M1). The run-start echo
2019
2661
  // (line ~993) is best-effort from `taskBackend`/`input.model`; a per-step `runner`/`model`
@@ -2028,6 +2670,23 @@ export class RunManager {
2028
2670
  return err.message;
2029
2671
  throw err;
2030
2672
  }
2673
+ // Which agent account this step spawns under, and — recorded on the step before the spawn —
2674
+ // which one its session belongs to. `sessionId` and `profileId` are a pair: a resume that
2675
+ // reads the wrong account's config dir finds no session and silently starts a fresh one.
2676
+ // Resolved together with the temp-directory preflight (#785): the step fails with a named,
2677
+ // actionable error instead of spawning a backend whose shell would return empty output.
2678
+ let stepProfile;
2679
+ try {
2680
+ stepProfile = await this.agentEnvForStep(runId, stepBackend, {
2681
+ generateFollowups: followupsEnabled() && input.generateFollowups !== false,
2682
+ });
2683
+ }
2684
+ catch (err) {
2685
+ if (err instanceof AgentTempDirError)
2686
+ return err.message;
2687
+ throw err;
2688
+ }
2689
+ this.store.updateStep(runId, step.id, { profileId: stepProfile.profileId });
2031
2690
  const runner = createRunner(stepBackend);
2032
2691
  let session;
2033
2692
  state.currentStepId = step.id;
@@ -2045,8 +2704,8 @@ export class RunManager {
2045
2704
  allowedTools: step.allowedTools ?? DEFAULT_ALLOWED_TOOLS,
2046
2705
  bashAllowlist: step.bashAllowlist,
2047
2706
  // The handoff file lives outside the worktree — grant access.
2048
- additionalDirectories: [join(this.dataDir, 'runs')],
2049
- env: this.agentEnv(runId, followupsEnabled() && input.generateFollowups !== false),
2707
+ additionalDirectories: agentDirectories(join(this.dataDir, 'runs'), stepProfile.env),
2708
+ env: stepProfile.env,
2050
2709
  model: backendModel,
2051
2710
  sessionId,
2052
2711
  // Interactive sessions have no wall clock — the idle timer rules.
@@ -2069,6 +2728,10 @@ export class RunManager {
2069
2728
  registerRunProcess(runId, session.pid);
2070
2729
  try {
2071
2730
  const result = await session.result;
2731
+ if (sessionError) {
2732
+ sink.sessionEnded('error', sessionError);
2733
+ return sessionError;
2734
+ }
2072
2735
  // v2 counterpart of v1's `done` (spec: the mappers leave session-close
2073
2736
  // events to the RunManager — only it knows how the session settled).
2074
2737
  sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
@@ -2371,6 +3034,10 @@ export class RunManager {
2371
3034
  status: review ? 'review' : 'done',
2372
3035
  finishedAt: new Date().toISOString(),
2373
3036
  currentStepId: undefined,
3037
+ // A run that got all the way to a settled turn is not in a limit loop, so the resume
3038
+ // counter starts over — otherwise a task that legitimately met the limit once a week would
3039
+ // creep toward the cap forever and stop resuming for no reason anyone could see.
3040
+ autoResumeAttempts: undefined,
2374
3041
  });
2375
3042
  this.store.appendEvent(runId, {
2376
3043
  type: 'lifecycle',