@open-mercato/cezar 0.9.3 → 0.10.0-pr892.1298
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -36
- package/dist/agent-config/account-identity.d.ts +44 -0
- package/dist/agent-config/account-identity.js +128 -0
- package/dist/agent-config/account-identity.js.map +1 -0
- package/dist/agent-config/catalog.d.ts +8 -0
- package/dist/agent-config/catalog.js +18 -0
- package/dist/agent-config/catalog.js.map +1 -1
- package/dist/agent-config/model-settings/claude.d.ts +2 -0
- package/dist/agent-config/model-settings/claude.js +11 -0
- package/dist/agent-config/model-settings/claude.js.map +1 -0
- package/dist/agent-config/model-settings/codex.d.ts +2 -0
- package/dist/agent-config/model-settings/codex.js +16 -0
- package/dist/agent-config/model-settings/codex.js.map +1 -0
- package/dist/agent-config/model-settings/opencode.d.ts +2 -0
- package/dist/agent-config/model-settings/opencode.js +8 -0
- package/dist/agent-config/model-settings/opencode.js.map +1 -0
- package/dist/agent-config/model-settings/pi.d.ts +12 -0
- package/dist/agent-config/model-settings/pi.js +18 -0
- package/dist/agent-config/model-settings/pi.js.map +1 -0
- package/dist/agent-config/model-settings/shared.d.ts +9 -0
- package/dist/agent-config/model-settings/shared.js +93 -0
- package/dist/agent-config/model-settings/shared.js.map +1 -0
- package/dist/agent-config/model-settings/types.d.ts +9 -0
- package/dist/agent-config/model-settings/types.js +2 -0
- package/dist/agent-config/model-settings/types.js.map +1 -0
- package/dist/agent-config/models.d.ts +9 -0
- package/dist/agent-config/models.js +32 -0
- package/dist/agent-config/models.js.map +1 -0
- package/dist/agent-config/service.js +4 -4
- package/dist/agent-config/service.js.map +1 -1
- package/dist/automations/types.d.ts +4 -0
- package/dist/automations/types.js +2 -1
- package/dist/automations/types.js.map +1 -1
- package/dist/config.d.ts +10 -1
- package/dist/config.js +48 -5
- package/dist/config.js.map +1 -1
- package/dist/contract/agent-config.d.ts +2 -0
- package/dist/contract/agent-profiles.d.ts +347 -0
- package/dist/contract/automations.d.ts +24 -0
- package/dist/contract/events.d.ts +59 -0
- package/dist/contract/github.d.ts +88 -0
- package/dist/contract/health.d.ts +5 -1
- package/dist/contract/index.d.ts +1 -0
- package/dist/contract/index.js +981 -544
- package/dist/contract/projects.d.ts +33 -5
- package/dist/contract/runs.d.ts +247 -0
- package/dist/contract/skills.d.ts +7 -0
- package/dist/contract/workflows.d.ts +6 -0
- package/dist/contract/workspace.d.ts +137 -2
- package/dist/core/agent-env.js +41 -16
- package/dist/core/agent-env.js.map +1 -1
- package/dist/core/agent-model-policy.d.ts +10 -0
- package/dist/core/agent-model-policy.js +36 -0
- package/dist/core/agent-model-policy.js.map +1 -0
- package/dist/core/agent-profiles.d.ts +62 -0
- package/dist/core/agent-profiles.js +90 -0
- package/dist/core/agent-profiles.js.map +1 -0
- package/dist/core/agent-runner.d.ts +37 -6
- package/dist/core/agent-runner.js +35 -3
- package/dist/core/agent-runner.js.map +1 -1
- package/dist/core/backend-detect.d.ts +5 -5
- package/dist/core/backend-detect.js +44 -7
- package/dist/core/backend-detect.js.map +1 -1
- package/dist/core/claude-cli-runner.js +10 -5
- package/dist/core/claude-cli-runner.js.map +1 -1
- package/dist/core/codex-app-server-runner.js +13 -3
- package/dist/core/codex-app-server-runner.js.map +1 -1
- package/dist/core/codex-app-server-transport.js +7 -2
- package/dist/core/codex-app-server-transport.js.map +1 -1
- package/dist/core/model-identity.d.ts +23 -11
- package/dist/core/model-identity.js +41 -24
- package/dist/core/model-identity.js.map +1 -1
- package/dist/core/model-presets.d.ts +8 -2
- package/dist/core/model-presets.js +51 -8
- package/dist/core/model-presets.js.map +1 -1
- package/dist/core/opencode-model-catalog.d.ts +32 -0
- package/dist/core/opencode-model-catalog.js +153 -0
- package/dist/core/opencode-model-catalog.js.map +1 -0
- package/dist/core/opencode-server-runner.d.ts +2 -0
- package/dist/core/opencode-server-runner.js +42 -11
- package/dist/core/opencode-server-runner.js.map +1 -1
- package/dist/core/pi-runner.d.ts +24 -0
- package/dist/core/pi-runner.js +349 -0
- package/dist/core/pi-runner.js.map +1 -0
- package/dist/core/pi-ui-mapper.d.ts +34 -0
- package/dist/core/pi-ui-mapper.js +270 -0
- package/dist/core/pi-ui-mapper.js.map +1 -0
- package/dist/core/provider-auth.d.ts +95 -3
- package/dist/core/provider-auth.js +234 -15
- package/dist/core/provider-auth.js.map +1 -1
- package/dist/core/provider-availability.d.ts +1 -1
- package/dist/core/provider-availability.js +2 -1
- package/dist/core/provider-availability.js.map +1 -1
- package/dist/core/runner-factory.js +3 -0
- package/dist/core/runner-factory.js.map +1 -1
- package/dist/core/shell-env.d.ts +28 -0
- package/dist/core/shell-env.js +56 -0
- package/dist/core/shell-env.js.map +1 -0
- package/dist/core/ui-events.d.ts +1 -1
- package/dist/core/usage-limit.d.ts +42 -0
- package/dist/core/usage-limit.js +209 -0
- package/dist/core/usage-limit.js.map +1 -0
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/paths.d.ts +53 -5
- package/dist/paths.js +63 -9
- package/dist/paths.js.map +1 -1
- package/dist/planner.js +6 -0
- package/dist/planner.js.map +1 -1
- package/dist/release/snapshot.d.ts +13 -2
- package/dist/release/snapshot.js +25 -1
- package/dist/release/snapshot.js.map +1 -1
- package/dist/runs/agent-tmpdir.d.ts +41 -0
- package/dist/runs/agent-tmpdir.js +183 -0
- package/dist/runs/agent-tmpdir.js.map +1 -0
- package/dist/runs/auto-name.js +6 -1
- package/dist/runs/auto-name.js.map +1 -1
- package/dist/runs/event-history.d.ts +50 -0
- package/dist/runs/event-history.js +613 -0
- package/dist/runs/event-history.js.map +1 -0
- package/dist/runs/run-index.d.ts +19 -0
- package/dist/runs/run-index.js +43 -0
- package/dist/runs/run-index.js.map +1 -0
- package/dist/runs/store.d.ts +78 -19
- package/dist/runs/store.js +175 -43
- package/dist/runs/store.js.map +1 -1
- package/dist/server/forge/github.d.ts +169 -0
- package/dist/server/forge/github.js +511 -0
- package/dist/server/forge/github.js.map +1 -1
- package/dist/server/forge/index.d.ts +9 -0
- package/dist/server/forge/index.js +14 -0
- package/dist/server/forge/index.js.map +1 -1
- package/dist/server/github.d.ts +2 -2
- package/dist/server/github.js +1 -1
- package/dist/server/github.js.map +1 -1
- package/dist/server/open-in-app.d.ts +2 -1
- package/dist/server/open-in-app.js +3 -1
- package/dist/server/open-in-app.js.map +1 -1
- package/dist/server/open-in-terminal.d.ts +46 -1
- package/dist/server/open-in-terminal.js +72 -13
- package/dist/server/open-in-terminal.js.map +1 -1
- package/dist/server/provider-action-gate.js +3 -1
- package/dist/server/provider-action-gate.js.map +1 -1
- package/dist/server/server.d.ts +2199 -428
- package/dist/server/server.js +1171 -139
- package/dist/server/server.js.map +1 -1
- package/dist/server-install/engine.js +5 -18
- package/dist/server-install/engine.js.map +1 -1
- package/dist/server-install/steps.js +6 -2
- package/dist/server-install/steps.js.map +1 -1
- package/dist/skills.js +4 -0
- package/dist/skills.js.map +1 -1
- package/dist/workflows/run.d.ts +202 -2
- package/dist/workflows/run.js +782 -115
- package/dist/workflows/run.js.map +1 -1
- package/dist/workflows/types.d.ts +3 -0
- package/dist/workflows/types.js +11 -2
- package/dist/workflows/types.js.map +1 -1
- package/dist/workspace/agent-accounts.d.ts +153 -0
- package/dist/workspace/agent-accounts.js +304 -0
- package/dist/workspace/agent-accounts.js.map +1 -0
- package/dist/workspace/agent-profiles.d.ts +78 -0
- package/dist/workspace/agent-profiles.js +115 -0
- package/dist/workspace/agent-profiles.js.map +1 -0
- package/dist/workspace/config.d.ts +19 -1
- package/dist/workspace/config.js +56 -2
- package/dist/workspace/config.js.map +1 -1
- package/dist/workspace/projects-cli.js +54 -4
- package/dist/workspace/projects-cli.js.map +1 -1
- package/dist/workspace/projects.d.ts +19 -1
- package/dist/workspace/projects.js +34 -1
- package/dist/workspace/projects.js.map +1 -1
- package/dist/workspace/semaphore.d.ts +39 -0
- package/dist/workspace/semaphore.js +31 -0
- package/dist/workspace/semaphore.js.map +1 -1
- package/package.json +3 -3
- package/scripts/mock-claude.mjs +20 -0
- package/scripts/mock-pi-rpc.mjs +83 -0
- package/web/dist/assets/{alert-dialog-V9cigyAO.js → alert-dialog-Byp42_IG.js} +1 -1
- package/web/dist/assets/arrow-down-CGDbDYKD.js +1 -0
- package/web/dist/assets/arrow-left-ESCjr1Hq.js +1 -0
- package/web/dist/assets/centered-state-LfAW-igm.js +43 -0
- package/web/dist/assets/chevron-right-Du-P1IUd.js +1 -0
- package/web/dist/assets/{collapsible-BlF-7CeC.js → collapsible-CBiiqDJM.js} +1 -1
- package/web/dist/assets/{commit-list-D2mKElUx.js → commit-list-6PPve7PE.js} +1 -1
- package/web/dist/assets/{compare-variants-CFBY16Rc.js → compare-variants-BDnp6mXJ.js} +1 -1
- package/web/dist/assets/{diff-DE6u5dTe.js → diff-_jK0OWpF.js} +2 -2
- package/web/dist/assets/{diff-stat-B1j6L7C6.js → diff-stat-CIrTu4Ym.js} +1 -1
- package/web/dist/assets/{diff-view-Gb2kk3Le.js → diff-view-COA_5IyY.js} +1 -1
- package/web/dist/assets/dropdown-menu-BofYpFFw.js +1 -0
- package/web/dist/assets/editable-title-_J0zFDl-.js +1 -0
- package/web/dist/assets/{ellipsis-vertical-BbxtXb6s.js → ellipsis-vertical-BeMR-l62.js} +1 -1
- package/web/dist/assets/{file-j5O4NryU.js → file-DY996Pee.js} +1 -1
- package/web/dist/assets/{folder-w9Lwj9x9.js → folder-NO3Ec-TY.js} +1 -1
- package/web/dist/assets/git-toolbar-BqJeUWcD.js +1 -0
- package/web/dist/assets/github-C96HPV9E.js +1 -0
- package/web/dist/assets/{image-preview-CcFdkZXW.js → image-preview-D9cCDfKQ.js} +1 -1
- package/web/dist/assets/index-Bx-grqa9.css +2 -0
- package/web/dist/assets/index-DoVvGDfW.js +7 -0
- package/web/dist/assets/{markdown-q9lm7kQ2.js → markdown-DDzk11zj.js} +1 -1
- package/web/dist/assets/new-task-form-BNLLgC6P.js +1 -0
- package/web/dist/assets/pill-CVM1EmV4.js +1 -0
- package/web/dist/assets/{project-router-DudOmkHf.js → project-router-LRCtP48W.js} +1 -1
- package/web/dist/assets/prompt-templates-DFOvYgtW.js +15 -0
- package/web/dist/assets/{refresh-cw-_C2EGJK5.js → refresh-cw-D1ys6QY2.js} +1 -1
- package/web/dist/assets/repo-git-CzUnsgWY.js +1 -0
- package/web/dist/assets/{run-diff-BLoKEnTx.js → run-diff-BlnqwfSS.js} +2 -2
- package/web/dist/assets/run-header-Bn7cN0Z1.js +1 -0
- package/web/dist/assets/{search-x-BgxaJgro.js → search-x-D6FSpxId.js} +1 -1
- package/web/dist/assets/{skill-empty-hint-D5MOmycv.js → skill-empty-hint-iwpCcqgI.js} +1 -1
- package/web/dist/assets/skills-CezbOaol.js +1 -0
- package/web/dist/assets/{sparkles-ANSka5bH.js → sparkles-MrQrgCI9.js} +1 -1
- package/web/dist/assets/{square-terminal-qojq20cH.js → square-terminal-TTH64lO7.js} +1 -1
- package/web/dist/assets/tab-link-FYksz037.js +1 -0
- package/web/dist/assets/task-changes-ClWlnzBL.js +1 -0
- package/web/dist/assets/{task-commits-yXmDrCoE.js → task-commits-Crbrl86b.js} +1 -1
- package/web/dist/assets/{task-files-BDRmCCPN.js → task-files-uSCWyD0-.js} +2 -2
- package/web/dist/assets/task-thread-DPmamBA1.js +9 -0
- package/web/dist/assets/{textarea-DVO7pg8Y.js → textarea-B9Ahj4E4.js} +1 -1
- package/web/dist/assets/thread-loading-BH_ypQoy.js +1 -0
- package/web/dist/assets/{trash-2-DKcnmg6e.js → trash-2-D8S3V3fu.js} +1 -1
- package/web/dist/assets/{triangle-alert-CPUEVSZ3.js → triangle-alert-bB7p9n5E.js} +1 -1
- package/web/dist/assets/{upload-D5Jrq7QK.js → upload-Bd5oRLPV.js} +1 -1
- package/web/dist/assets/{use-desktop-CXae6uyk.js → use-desktop-BNvbRBeZ.js} +1 -1
- package/web/dist/assets/{use-submit-shortcut-RKJLKdsW.js → use-submit-shortcut-B-xhu4Bf.js} +1 -1
- package/web/dist/assets/utils-BhpbVMwN.js +64 -0
- package/web/dist/assets/{workflows-CGyTdyds.js → workflows-RmfbM9Vy.js} +3 -3
- package/web/dist/assets/{zoomable-image-hcqwUIlQ.js → zoomable-image-BCW6ZBV0.js} +1 -1
- package/web/dist/index.html +24 -23
- package/web/dist/assets/arrow-down-DflHdNlN.js +0 -1
- package/web/dist/assets/arrow-left-IIL4uR-C.js +0 -1
- package/web/dist/assets/centered-state-BkAJVLJE.js +0 -43
- package/web/dist/assets/chevron-right-CxAoD1eD.js +0 -1
- package/web/dist/assets/circle-check-CdwcIbkx.js +0 -1
- package/web/dist/assets/circle-x-Cnc43xJ2.js +0 -1
- package/web/dist/assets/dropdown-menu-Ns-rY6sm.js +0 -1
- package/web/dist/assets/editable-title-BGuiDxFT.js +0 -1
- package/web/dist/assets/git-pull-request-DlSop3Ch.js +0 -1
- package/web/dist/assets/git-toolbar-BdJ8XSxI.js +0 -1
- package/web/dist/assets/github-CCLn2OAU.js +0 -1
- package/web/dist/assets/index-BmHHLbwc.js +0 -6
- package/web/dist/assets/index-u8OAnq0r.css +0 -2
- package/web/dist/assets/new-task-form-BuYxAWN7.js +0 -1
- package/web/dist/assets/pill-t97MvPW_.js +0 -1
- package/web/dist/assets/prompt-templates-C5iO3Lda.js +0 -15
- package/web/dist/assets/repo-git-DqQzj33d.js +0 -1
- package/web/dist/assets/run-header-Dho4eurS.js +0 -1
- package/web/dist/assets/skills-B5g064CC.js +0 -1
- package/web/dist/assets/tab-link-CIcwFJtC.js +0 -1
- package/web/dist/assets/task-changes-yRfsEsnh.js +0 -1
- package/web/dist/assets/task-thread-C9AnIz4Z.js +0 -9
- package/web/dist/assets/thread-loading-DJDl701Z.js +0 -1
- package/web/dist/assets/utils-Wtkd5E6j.js +0 -64
package/dist/workflows/run.js
CHANGED
|
@@ -4,27 +4,35 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync
|
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { parseAskMarkerResult, stripAskMarker, } from '../core/ask.js';
|
|
6
6
|
import { onUsage, registerRunProcess, unregisterRunProcess } from '../core/process-usage.js';
|
|
7
|
+
import { parseUsageLimit } from '../core/usage-limit.js';
|
|
7
8
|
import { createRunner } from '../core/runner-factory.js';
|
|
8
9
|
import { modelConflictsWithRunner } from '../core/model-presets.js';
|
|
10
|
+
import { AGENT_MODELS_LOCKED_ERROR, agentModelsLocked } from '../core/agent-model-policy.js';
|
|
9
11
|
import { ModelIdentityError, formatModelIdentity, normalizeModelForBackend, } from '../core/model-identity.js';
|
|
10
12
|
import { HANDOFF_ONLY_INSTRUCTIONS, HANDOFF_INSTRUCTIONS, appendHandoffHeartbeat, followupsEnabled, handoffPath, seedHandoffFile, } from '../handoff.js';
|
|
11
13
|
import { todosPath } from '../todos.js';
|
|
12
14
|
import { discoverSkills } from '../skills.js';
|
|
13
15
|
import { materializeSkillDir } from '../skills-remote.js';
|
|
14
16
|
import { seedAgentConfigLocalLayer } from '../agent-config/seed.js';
|
|
17
|
+
import { readAgentModelProvider } from '../agent-config/models.js';
|
|
15
18
|
import { loadConfig, resolveWorktreeRetention } from '../config.js';
|
|
16
19
|
import { autosaveCommit, createWorktree, resolveBaseRef, worktreeDiff, worktreeShortstat } from '../git-worktree.js';
|
|
17
20
|
import { getHeadCommit, getRepoInfo } from '../server/git.js';
|
|
18
21
|
import { loadWorkflows } from './load.js';
|
|
19
22
|
import { reclaimWorktrees, rematerializeReclaimedWorktree } from '../runs/retention.js';
|
|
23
|
+
import { AgentTempDirError, agentTmpEnv, removeAgentTmpDir, sweepAgentTmpDirs, } from '../runs/agent-tmpdir.js';
|
|
20
24
|
import { extractTaskRefs, refineTaskRefs, titleRefNumber } from '../runs/task-refs.js';
|
|
21
25
|
import { parseTaskMarkers, stripTaskMarkers } from '../runs/task-markers.js';
|
|
22
26
|
import { autoNamingActive, generateRunName, liveTitleUpdatesEnabled, postValidateTitle } from '../runs/auto-name.js';
|
|
23
27
|
import { reviewGateEnabled } from '../runs/review-gate.js';
|
|
28
|
+
import { resolveProfileEnvForRoot } from '../workspace/agent-profiles.js';
|
|
24
29
|
import { WorkspaceSemaphore } from '../workspace/semaphore.js';
|
|
25
30
|
import { UiEventSink } from '../runs/ui-event-sink.js';
|
|
26
31
|
import { chainStepNote, DEFAULT_ALLOWED_TOOLS, stepKind } from './types.js';
|
|
27
32
|
const CHECK_OUTPUT_CAP = 20_000;
|
|
33
|
+
async function configuredModelProvider(backend, repoRoot) {
|
|
34
|
+
return readAgentModelProvider(backend, repoRoot).catch(() => undefined);
|
|
35
|
+
}
|
|
28
36
|
/** An interactive session that hears nothing from the user closes itself. */
|
|
29
37
|
export const IDLE_TIMEOUT_MS = 15 * 60_000;
|
|
30
38
|
/**
|
|
@@ -113,6 +121,95 @@ const REPOSITORY_ROOT_LOCK_DISABLED_NOTE = 'repository-root lock disabled by CEZ
|
|
|
113
121
|
const MAX_AUTO_CONTINUES = 40;
|
|
114
122
|
const AUTONOMOUS_NUDGE = 'Continue working autonomously until the task is fully complete. Do not ask me for confirmation or clarification — make reasonable assumptions and proceed. When everything is done, end the session with your done signal.';
|
|
115
123
|
const MONITORING_WAKE_NUDGE = 'Re-check the downstream work you were monitoring. Continue toward the task goal; emit CEZ:MONITORING again only if it is still pending.';
|
|
124
|
+
/**
|
|
125
|
+
* Auto-resume after a provider usage limit (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
126
|
+
*
|
|
127
|
+
* The wait is the provider's own reset instant plus this grace: resuming AT the boundary races the
|
|
128
|
+
* provider's clock (and its rounding), and one failed resume costs the whole window over again.
|
|
129
|
+
* Thirty seconds is cheap next to five hours and long enough to be past any sane skew.
|
|
130
|
+
*/
|
|
131
|
+
export const AUTO_RESUME_GRACE_MS = 30_000;
|
|
132
|
+
/**
|
|
133
|
+
* Consecutive automatic resumes allowed without a human turn. A resume can only fire after a real
|
|
134
|
+
* reset instant, so this is not a throttle — it is the backstop for the pathological case (a
|
|
135
|
+
* provider that answers "limit reached, retry now" in a loop), and it is deliberately generous
|
|
136
|
+
* enough to sit through a couple of days of five-hour windows.
|
|
137
|
+
*/
|
|
138
|
+
export const MAX_AUTO_RESUMES = 12;
|
|
139
|
+
/**
|
|
140
|
+
* How long a missed deadline stays worth acting on. The promise is "we pick this up when the
|
|
141
|
+
* window reopens" — kept across a restart or an overnight close, which is the case the feature
|
|
142
|
+
* exists for. A day later it is no longer that promise: the user has moved on, and a task
|
|
143
|
+
* springing back to life is a surprise rather than a service. Such a deadline is retired with a
|
|
144
|
+
* note instead of fired, so the only tasks a sweep can revive are ones someone is still waiting on.
|
|
145
|
+
*/
|
|
146
|
+
export const AUTO_RESUME_MISSED_WINDOW_MS = 24 * 60 * 60_000;
|
|
147
|
+
/**
|
|
148
|
+
* How often the queue checks that it is not wedged.
|
|
149
|
+
*
|
|
150
|
+
* A hold is the only thing in the engine that can make an idle queue CORRECT, so it is also the
|
|
151
|
+
* only thing that can make a wedged one look correct. This tick is the way out: cheap (a few
|
|
152
|
+
* in-memory checks), unref'd, and it only ever acts when idling has no justification left.
|
|
153
|
+
*/
|
|
154
|
+
export const QUEUE_WATCHDOG_MS = 60_000;
|
|
155
|
+
/** Shared empty holds for the common "nothing is held" pump — avoids allocating per sweep. */
|
|
156
|
+
const NO_HOLDS = { deadline: new Set(), inFlight: new Set() };
|
|
157
|
+
/**
|
|
158
|
+
* May this run start, given what its account is holding?
|
|
159
|
+
*
|
|
160
|
+
* The two kinds of hold bind different work, and getting that wrong has produced a bug in each
|
|
161
|
+
* direction (spec 2026-08-03-auto-resume-after-usage-limit):
|
|
162
|
+
*
|
|
163
|
+
* - a `deadline` hold means the window is KNOWN shut until an instant, so it blocks everything
|
|
164
|
+
* on that account — resumes included. Exempting them let four resumes fire at once and
|
|
165
|
+
* re-limit one after another, which is the stampede wearing a different hat.
|
|
166
|
+
* - an `inFlight` hold means a resume is testing the window right now and nothing is proven, so
|
|
167
|
+
* it blocks fresh work but not other resumes. Blocking those deadlocked a live workspace.
|
|
168
|
+
*/
|
|
169
|
+
function accountHeldFor(run, holds, fallbackRunner) {
|
|
170
|
+
const key = runAccountKey(run, fallbackRunner);
|
|
171
|
+
if (holds.deadline.has(key))
|
|
172
|
+
return true;
|
|
173
|
+
return holds.inFlight.has(key) && !resumeInFlight(run);
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Which agent ACCOUNT a run's work runs on — the thing a provider usage limit actually closes
|
|
177
|
+
* (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
178
|
+
*
|
|
179
|
+
* Backend plus agent account, because those are the two axes a limit is scoped to: a Claude
|
|
180
|
+
* limit must never stall a Codex task, and a second Claude login is a second budget. A record
|
|
181
|
+
* that names no runner has not started yet and will take the configured default, which is what
|
|
182
|
+
* `fallbackRunner` carries; a run that HAS started always carries its resolved runner (execute
|
|
183
|
+
* persists it), and only started runs can be holding.
|
|
184
|
+
*/
|
|
185
|
+
export function runAccountKey(run, fallbackRunner) {
|
|
186
|
+
return `${run.runner ?? fallbackRunner}:${run.agentProfile ?? 'default'}`;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Is this run an automatic resume that has not completed a turn yet?
|
|
190
|
+
*
|
|
191
|
+
* Such a run is the work the reopened window is FOR, so the hold must never apply to it — not
|
|
192
|
+
* its own, and not another resume's. Two resumes that hold each other is a deadlock the queue
|
|
193
|
+
* cannot recover from: both sit `queued` with a counter and no deadline, each waiting for the
|
|
194
|
+
* other to prove a window neither will ever get to test. That is the shape a live run produced
|
|
195
|
+
* — two scheduled tasks fired, both went `queued`, and nothing in the workspace moved again.
|
|
196
|
+
*
|
|
197
|
+
* The hold exists to stop NEW work walking into a closed window. A resume is not new work.
|
|
198
|
+
*/
|
|
199
|
+
function resumeInFlight(run) {
|
|
200
|
+
return (run.autoResumeAttempts !== undefined && (run.status === 'queued' || run.status === 'running'));
|
|
201
|
+
}
|
|
202
|
+
const AUTO_RESUME_PROMPT = 'The provider usage limit that interrupted this task has reset. Read the handoff file (CEZ_HANDOFF_FILE) to recover context, then continue the task from where you left off.';
|
|
203
|
+
/**
|
|
204
|
+
* The wake instant as a human reads it — local, to the SECOND, with the zone named. The
|
|
205
|
+
* transcript line is what someone scanning a stalled task actually reads, and "18:41" is not
|
|
206
|
+
* enough to tell a wait that is nearly over from one that just started; the machine-readable ISO
|
|
207
|
+
* copy lives on `RunRecord.autoResumeAt`. Server-side formatting is honest here because cezar is
|
|
208
|
+
* local-first: the process and the browser reading it are the same machine.
|
|
209
|
+
*/
|
|
210
|
+
function formatWakeInstant(at) {
|
|
211
|
+
return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'long' }).format(at);
|
|
212
|
+
}
|
|
116
213
|
/**
|
|
117
214
|
* The effective "extra" system prompt for a run (spec §protocol v2, R2 2.3):
|
|
118
215
|
* the per-run override (`POST /api/runs` `systemPrompt`) REPLACES the
|
|
@@ -137,6 +234,17 @@ export function composeSystemPrompt(...parts) {
|
|
|
137
234
|
.filter((p) => Boolean(p))
|
|
138
235
|
.join('\n\n---\n\n');
|
|
139
236
|
}
|
|
237
|
+
/**
|
|
238
|
+
* The directories a spawned agent may reach outside its worktree: the run-state
|
|
239
|
+
* folder that holds its handoff file, plus its own temp directory when this run
|
|
240
|
+
* got one (#785). Handing an agent a `TMPDIR` its file tools are not allowed to
|
|
241
|
+
* write would trade one silent failure for another, so the two travel together;
|
|
242
|
+
* under `CEZ_AGENT_TMPDIR=0` there is no per-run directory and the list is
|
|
243
|
+
* exactly what it always was.
|
|
244
|
+
*/
|
|
245
|
+
export function agentDirectories(runsDir, env) {
|
|
246
|
+
return env.TMPDIR ? [runsDir, env.TMPDIR] : [runsDir];
|
|
247
|
+
}
|
|
140
248
|
/**
|
|
141
249
|
* Materialized pasted attachment: the on-disk name/serving-URL pair the
|
|
142
250
|
* transcript already used, plus the absolute path that lets the agent
|
|
@@ -239,6 +347,12 @@ export class RunManager {
|
|
|
239
347
|
/** Messages that landed in the dequeue → session-open gap (#472), flushed as
|
|
240
348
|
* ordinary follow-up turns the moment the session opens. In-memory only. */
|
|
241
349
|
deferredMessages = new Map();
|
|
350
|
+
/** Armed usage-limit resumes, keyed by run id (spec
|
|
351
|
+
* 2026-08-03-auto-resume-after-usage-limit). The DEADLINE itself lives on the record
|
|
352
|
+
* (`autoResumeAt`) — this map holds only the process-local timer, so a restart rebuilds it
|
|
353
|
+
* from the record rather than losing the wait. Runs here are `failed` and therefore NOT in
|
|
354
|
+
* `active`, which is why the timer cannot live on an `ActiveRun` like the monitoring one. */
|
|
355
|
+
autoResumeTimers = new Map();
|
|
242
356
|
pumping = false;
|
|
243
357
|
/** A pump that arrived while one was in flight — replayed by `pump()`'s own
|
|
244
358
|
* loop so a slot freed mid-sweep is never a lost wakeup. */
|
|
@@ -258,6 +372,13 @@ export class RunManager {
|
|
|
258
372
|
/** Unsubscribe handle for the constructor's `onUsage` subscription — released
|
|
259
373
|
* by dispose() so a torn-down manager stops receiving sampler ticks. */
|
|
260
374
|
offUsage;
|
|
375
|
+
/** The stalled-queue watchdog (see `rescueStalledQueue`). */
|
|
376
|
+
queueWatchdog;
|
|
377
|
+
/** Set by the watchdog for exactly one sweep: ignore the usage-limit hold and make progress. */
|
|
378
|
+
forceNextPump = false;
|
|
379
|
+
/** Runs the watchdog started despite the hold. The spawn-time gate (`requeueWhileHeld`) would
|
|
380
|
+
* otherwise hand them straight back and the rescue would undo itself in a millisecond. */
|
|
381
|
+
forceStarted = new Set();
|
|
261
382
|
/** The workspace-wide parallel-cap semaphore + cached resource config
|
|
262
383
|
* (spec 2026-07-20, step 2.5). Boot constructs ONE and every manager shares
|
|
263
384
|
* it; the private fallback keeps single-manager callers and tests working. */
|
|
@@ -274,10 +395,13 @@ export class RunManager {
|
|
|
274
395
|
busySlots: () => this.busySlots(),
|
|
275
396
|
pump: () => this.pump(),
|
|
276
397
|
oldestQueuedAt: () => this.oldestQueuedAt(),
|
|
398
|
+
accountHolds: () => this.accountHolds(),
|
|
277
399
|
});
|
|
278
400
|
// Memory guard (#memory-guard): the shared process-tree sampler already ticks ~every 2 s for
|
|
279
401
|
// the runs table; piggyback on it to enforce the per-task memory ceiling.
|
|
280
402
|
this.offUsage = onUsage((snapshot) => void this.enforceMemoryLimit(snapshot));
|
|
403
|
+
this.queueWatchdog = setInterval(() => void this.rescueStalledQueue(), QUEUE_WATCHDOG_MS);
|
|
404
|
+
this.queueWatchdog.unref?.();
|
|
281
405
|
}
|
|
282
406
|
/**
|
|
283
407
|
* Release everything this manager owns without touching run records
|
|
@@ -292,6 +416,7 @@ export class RunManager {
|
|
|
292
416
|
dispose() {
|
|
293
417
|
this.offUsage();
|
|
294
418
|
this.offSemaphore();
|
|
419
|
+
clearInterval(this.queueWatchdog);
|
|
295
420
|
for (const [runId, state] of this.active) {
|
|
296
421
|
this.clearIdleTimer(state);
|
|
297
422
|
this.clearMonitoringWakeTimer(state, runId);
|
|
@@ -299,6 +424,9 @@ export class RunManager {
|
|
|
299
424
|
state.releaseRepoRoot?.();
|
|
300
425
|
state.releaseRepoRoot = undefined;
|
|
301
426
|
}
|
|
427
|
+
for (const timer of this.autoResumeTimers.values())
|
|
428
|
+
clearTimeout(timer);
|
|
429
|
+
this.autoResumeTimers.clear();
|
|
302
430
|
this.active.clear();
|
|
303
431
|
this.waiting.clear();
|
|
304
432
|
this.starting.clear();
|
|
@@ -364,21 +492,64 @@ export class RunManager {
|
|
|
364
492
|
* key would let a value inherited from *this* process through — a nested
|
|
365
493
|
* cezar (an agent running `cez serve`/`cez run`/the test suite) would then
|
|
366
494
|
* write follow-ups into the parent's inbox despite the opt-out. Empty is the
|
|
367
|
-
* established "absent" spelling — consumers guard with `if (todosFile)`.
|
|
495
|
+
* established "absent" spelling — consumers guard with `if (todosFile)`.
|
|
496
|
+
*
|
|
497
|
+
* `TMPDIR`/`TEMP`/`TMP` (#785) point at this run's own scratch directory
|
|
498
|
+
* instead of the machine-wide one every agent used to share. Created and
|
|
499
|
+
* write-probed here, on the last common path before a spawn, so an unusable
|
|
500
|
+
* temp directory throws `AgentTempDirError` at the caller rather than
|
|
501
|
+
* turning into empty command output inside a running agent. */
|
|
368
502
|
agentEnv(runId, generateFollowups = true) {
|
|
369
503
|
return {
|
|
370
504
|
CEZ_HANDOFF_FILE: handoffPath(this.dataDir, runId),
|
|
371
505
|
CEZ_TASK_ID: runId,
|
|
372
506
|
CEZ_TODOS_FILE: generateFollowups ? todosPath(this.dataDir) : '',
|
|
507
|
+
...agentTmpEnv(this.dataDir, runId),
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
/**
|
|
511
|
+
* `agentEnv` plus the agent-account variable for the profile this STEP runs under (spec
|
|
512
|
+
* 2026-07-29-agent-profiles), and the id it resolved to so the caller can record it.
|
|
513
|
+
*
|
|
514
|
+
* Resolved per step, not per run, because a workflow can mix backends: an override naming a
|
|
515
|
+
* Claude account says nothing about which Codex account a codex step should use. Resolution
|
|
516
|
+
* order, most specific first:
|
|
517
|
+
*
|
|
518
|
+
* 1. the step's ALREADY-RECORDED `profileId` — a resume or Continue must reattach to the
|
|
519
|
+
* account that created the session, whatever the project has since been switched to;
|
|
520
|
+
* 2. the run's composer override, but only for steps on the run's own runner;
|
|
521
|
+
* 3. the project's stored selection, and failing that the discovered default.
|
|
522
|
+
*
|
|
523
|
+
* Read fresh every time. `~/.cezar/config.json` is shared by every cezar process on this
|
|
524
|
+
* machine, so a cached snapshot is a staleness bug, and one small JSON read is free next to
|
|
525
|
+
* spawning a CLI. Never throws: an unreadable home degrades to the default profile, which is
|
|
526
|
+
* exactly the behaviour that predates profiles.
|
|
527
|
+
*/
|
|
528
|
+
async agentEnvForStep(runId, backend, options = {}) {
|
|
529
|
+
const run = this.store.getRun(runId);
|
|
530
|
+
const profileId = options.recordedProfileId
|
|
531
|
+
?? (backend === (run?.runner ?? 'claude') ? run?.agentProfile : undefined);
|
|
532
|
+
const resolved = await resolveProfileEnvForRoot(this.repoRoot, backend, profileId);
|
|
533
|
+
return {
|
|
534
|
+
env: { ...this.agentEnv(runId, options.generateFollowups), ...resolved.env },
|
|
535
|
+
profileId: resolved.profile.id,
|
|
373
536
|
};
|
|
374
537
|
}
|
|
375
538
|
startRun(workflow, input, group) {
|
|
539
|
+
// Sanitize at the manager boundary so CLI runs, workflows, variants, and
|
|
540
|
+
// direct callers cannot bypass the HTTP policy.
|
|
541
|
+
const effectiveInput = agentModelsLocked(this.repoRoot)
|
|
542
|
+
? { ...input, model: undefined }
|
|
543
|
+
: input;
|
|
376
544
|
const run = this.store.createRun({
|
|
377
545
|
title: makeRunTitle(input.task, workflow) + (group ? ` (${group.variant})` : ''),
|
|
378
546
|
workflow: workflow.name,
|
|
379
547
|
task: input.task,
|
|
380
|
-
model:
|
|
548
|
+
model: effectiveInput.model,
|
|
381
549
|
runner: input.runner,
|
|
550
|
+
// The composer's per-task account (spec 2026-07-29-agent-profiles). Persisted at creation
|
|
551
|
+
// so a queued run picks it up at dequeue and every later resume reads the same answer.
|
|
552
|
+
agentProfile: input.agentProfile,
|
|
382
553
|
// The global inbox is the ceiling on the per-run flag (#471). Enforced here rather than
|
|
383
554
|
// at the HTTP route because `cezar run`, the inbox's own "▶ Run" and variants all reach
|
|
384
555
|
// startRun directly — a route-level gate would leave those writing todos.json.
|
|
@@ -425,7 +596,7 @@ export class RunManager {
|
|
|
425
596
|
// above shows instantly; the namer's short title replaces it when (and if)
|
|
426
597
|
// the model answers. Never awaited, never fails the run.
|
|
427
598
|
void this.autoNameRun(run.id, skillHint, input.task);
|
|
428
|
-
this.pendingJobs.set(run.id, { workflow, input });
|
|
599
|
+
this.pendingJobs.set(run.id, { workflow, input: effectiveInput });
|
|
429
600
|
this.queue.push(run.id);
|
|
430
601
|
void this.pump();
|
|
431
602
|
return run;
|
|
@@ -494,6 +665,7 @@ export class RunManager {
|
|
|
494
665
|
*/
|
|
495
666
|
async pump() {
|
|
496
667
|
this.reconcileMonitoringWakeTimers();
|
|
668
|
+
this.reconcileAutoResumes();
|
|
497
669
|
// A pump requested while one is in flight can't just be dropped: the
|
|
498
670
|
// in-flight pass may already have read capacity (it awaits `getRepoInfo`
|
|
499
671
|
// before the first check), so a slot freed in that window would be lost
|
|
@@ -518,10 +690,46 @@ export class RunManager {
|
|
|
518
690
|
const capacity = () => this.semaphore.busy() < maxParallel &&
|
|
519
691
|
this.busySlots() < projectMax &&
|
|
520
692
|
(repo !== null || this.busySlots() < 1);
|
|
693
|
+
// The usage-limit hold (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
694
|
+
//
|
|
695
|
+
// A limit closes an ACCOUNT, not a run — so starting the next queued task walks it into
|
|
696
|
+
// the same wall. Measured before this gate existed: eight tasks under `maxParallel: 2`
|
|
697
|
+
// all failed within 517 ms, each spawning a CLI (and, outside worktree-opt-out mode, a
|
|
698
|
+
// worktree and a branch) only to be marked `scheduled`. The cap was respected at every
|
|
699
|
+
// instant and was no brake at all, because a doomed run lives ~200 ms.
|
|
700
|
+
//
|
|
701
|
+
// So: while any run on an account is waiting out a limit, nothing new starts on THAT
|
|
702
|
+
// account. Other accounts (a second login, a different backend) keep running — the hold
|
|
703
|
+
// is keyed, not global. The set is derived from the durable records rather than tracked
|
|
704
|
+
// separately, which is what makes it survive a restart, expire on its own, and lift the
|
|
705
|
+
// instant a user cancels a resume.
|
|
706
|
+
// The watchdog's one-shot override — read and cleared here, so a forced sweep never
|
|
707
|
+
// leaks into the next ordinary one.
|
|
708
|
+
const forced = this.forceNextPump;
|
|
709
|
+
this.forceNextPump = false;
|
|
710
|
+
const holds = this.queue.length > 0 && !forced ? this.semaphore.accountHolds() : NO_HOLDS;
|
|
711
|
+
const anyHold = holds.deadline.size > 0 || holds.inFlight.size > 0;
|
|
712
|
+
// Only pay for the config read when something is actually held: a queued record may name
|
|
713
|
+
// no runner, and then the account it would use is the configured default.
|
|
714
|
+
const defaultRunner = anyHold ? (await loadConfig(this.repoRoot)).defaultRunner : undefined;
|
|
521
715
|
while (this.queue.length > 0 && capacity()) {
|
|
522
|
-
|
|
716
|
+
// FIFO among the runs that CAN start; a held one keeps its place in the queue rather
|
|
717
|
+
// than being dequeued and re-queued (which would churn its position and its record).
|
|
718
|
+
const next = !anyHold
|
|
719
|
+
? 0
|
|
720
|
+
: this.queue.findIndex((id) => {
|
|
721
|
+
const queued = this.store.getRun(id);
|
|
722
|
+
return !queued || !accountHeldFor(queued, holds, defaultRunner ?? 'claude');
|
|
723
|
+
});
|
|
724
|
+
if (next === -1)
|
|
725
|
+
break; // everything queued is waiting on a held account
|
|
726
|
+
const runId = this.queue.splice(next, 1)[0];
|
|
523
727
|
if (!runId)
|
|
524
728
|
break;
|
|
729
|
+
// A forced sweep has to reach the spawn: the gate inside `execute` asks the same
|
|
730
|
+
// question and would send this run straight back to the queue.
|
|
731
|
+
if (forced)
|
|
732
|
+
this.forceStarted.add(runId);
|
|
525
733
|
const job = this.pendingJobs.get(runId);
|
|
526
734
|
const continuation = this.pendingContinuations.get(runId);
|
|
527
735
|
this.pendingJobs.delete(runId);
|
|
@@ -572,6 +780,87 @@ export class RunManager {
|
|
|
572
780
|
this.pumping = false;
|
|
573
781
|
}
|
|
574
782
|
}
|
|
783
|
+
/**
|
|
784
|
+
* Make one `queued` RECORD executable again — the engine half a queued run needs but does not
|
|
785
|
+
* persist (`pendingJobs` / `pendingContinuations` are process-local, the record is not).
|
|
786
|
+
*
|
|
787
|
+
* Two callers, one path: boot recovery re-adopts everything the previous process was holding,
|
|
788
|
+
* and the queue watchdog re-adopts anything the running process has somehow lost. A queued
|
|
789
|
+
* record with no work item behind it is invisible to `pump()` and would sit there for good,
|
|
790
|
+
* which is the worst failure this engine has — the task is neither running nor failed, just
|
|
791
|
+
* silently never going to happen.
|
|
792
|
+
*
|
|
793
|
+
* A continuation is reconstructed first: its executable details are gone, but the pending
|
|
794
|
+
* `continue-N` step and the session before it are durable, which is enough. Otherwise the
|
|
795
|
+
* workflow is revived from the record. A run that can be neither is failed loudly rather than
|
|
796
|
+
* left in the queue as a ghost.
|
|
797
|
+
*/
|
|
798
|
+
async reviveQueuedRun(run, reason) {
|
|
799
|
+
const queuedContinuation = [...run.steps]
|
|
800
|
+
.reverse()
|
|
801
|
+
.find((step) => step.status === 'pending' && step.id.startsWith('continue-'));
|
|
802
|
+
const sessionStep = queuedContinuation
|
|
803
|
+
? [...run.steps].reverse().find((step) => step.id !== queuedContinuation.id && step.sessionId)
|
|
804
|
+
: undefined;
|
|
805
|
+
if (queuedContinuation && sessionStep?.sessionId) {
|
|
806
|
+
const backend = run.runner ?? 'claude';
|
|
807
|
+
const sessionBackend = sessionStep.backend ?? backend;
|
|
808
|
+
this.pendingContinuations.set(run.id, {
|
|
809
|
+
stepId: queuedContinuation.id,
|
|
810
|
+
sessionId: sessionBackend === backend ? sessionStep.sessionId : undefined,
|
|
811
|
+
backend,
|
|
812
|
+
prompt: RESTART_CONTINUATION_PROMPT,
|
|
813
|
+
images: [],
|
|
814
|
+
});
|
|
815
|
+
this.queue.push(run.id);
|
|
816
|
+
this.store.appendEvent(run.id, {
|
|
817
|
+
type: 'lifecycle',
|
|
818
|
+
message: `${reason} — interrupted continuation re-queued`,
|
|
819
|
+
});
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
const workflow = await this.reviveWorkflow(run);
|
|
823
|
+
if (!workflow) {
|
|
824
|
+
this.store.updateRun(run.id, {
|
|
825
|
+
status: 'failed',
|
|
826
|
+
error: 'interrupted — workflow definition not recoverable after a restart',
|
|
827
|
+
finishedAt: new Date().toISOString(),
|
|
828
|
+
});
|
|
829
|
+
this.store.appendEvent(run.id, {
|
|
830
|
+
type: 'lifecycle',
|
|
831
|
+
message: `${reason} — workflow definition not recoverable, task failed`,
|
|
832
|
+
});
|
|
833
|
+
return;
|
|
834
|
+
}
|
|
835
|
+
// Re-apply the inbox ceiling (#471). `execute()` gates again at spawn time, so the agent is
|
|
836
|
+
// safe either way — but a run queued while the inbox was on and recovered after it was
|
|
837
|
+
// switched off would otherwise keep echoing `generateFollowups: true` on a run that
|
|
838
|
+
// demonstrably produced none. Normalize the record, the way startRun does.
|
|
839
|
+
const generateFollowups = followupsEnabled() ? run.generateFollowups : false;
|
|
840
|
+
if (generateFollowups !== run.generateFollowups) {
|
|
841
|
+
this.store.updateRun(run.id, { generateFollowups });
|
|
842
|
+
}
|
|
843
|
+
this.pendingJobs.set(run.id, {
|
|
844
|
+
workflow,
|
|
845
|
+
// Folded through the same helper `pump()` uses (#472) so a restart carries the stack.
|
|
846
|
+
// Idempotent: hydration always composes from `run.task` + the stack, never from an
|
|
847
|
+
// already-folded `input.task`, so re-hydrating at dequeue yields the same string.
|
|
848
|
+
input: this.hydrateQueuedInput(run.id, {
|
|
849
|
+
task: run.task,
|
|
850
|
+
model: run.model,
|
|
851
|
+
runner: run.runner,
|
|
852
|
+
generateFollowups,
|
|
853
|
+
// Re-thread autonomy (#489): the rebuilt input feeds `execute`, whose mid-run auto-nudge
|
|
854
|
+
// reads `input.autonomous`. Without this a recovered autonomous run would run
|
|
855
|
+
// non-autonomously and later wrongly park at `review`.
|
|
856
|
+
autonomous: run.autonomous,
|
|
857
|
+
// Preserve an explicit worktree opt-out across a queued restart.
|
|
858
|
+
worktree: run.worktree,
|
|
859
|
+
}),
|
|
860
|
+
});
|
|
861
|
+
this.queue.push(run.id);
|
|
862
|
+
this.store.appendEvent(run.id, { type: 'lifecycle', message: `${reason} — task re-queued` });
|
|
863
|
+
}
|
|
575
864
|
/**
|
|
576
865
|
* Startup recovery (#367) — re-adopt runs that were live when the previous
|
|
577
866
|
* cezar process exited (requires the store opened with `keepLive`):
|
|
@@ -588,83 +877,13 @@ export class RunManager {
|
|
|
588
877
|
.listRuns()
|
|
589
878
|
.filter((r) => ['queued', 'waiting', 'running'].includes(r.status))
|
|
590
879
|
.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
880
|
+
// A crash never reaches `dropActive`, so its temp directory (#785) outlived the run.
|
|
881
|
+
// Startup is the one moment we know which runs are still live, so sweep every other
|
|
882
|
+
// per-run directory here — bounded to `<dataDir>/tmp`, never a sibling.
|
|
883
|
+
sweepAgentTmpDirs(this.dataDir, live.map((r) => r.id));
|
|
591
884
|
for (const run of live) {
|
|
592
885
|
if (run.status === 'queued') {
|
|
593
|
-
|
|
594
|
-
// behind a capacity gate. Its executable details are process-local,
|
|
595
|
-
// but the queued continue step and preceding provider session are
|
|
596
|
-
// durable, so reconstruct that job before considering workflow revival.
|
|
597
|
-
const queuedContinuation = [...run.steps]
|
|
598
|
-
.reverse()
|
|
599
|
-
.find((step) => step.status === 'pending' && step.id.startsWith('continue-'));
|
|
600
|
-
const sessionStep = queuedContinuation
|
|
601
|
-
? [...run.steps]
|
|
602
|
-
.reverse()
|
|
603
|
-
.find((step) => step.id !== queuedContinuation.id && step.sessionId)
|
|
604
|
-
: undefined;
|
|
605
|
-
if (queuedContinuation && sessionStep?.sessionId) {
|
|
606
|
-
const backend = run.runner ?? 'claude';
|
|
607
|
-
const sessionBackend = sessionStep.backend ?? backend;
|
|
608
|
-
this.pendingContinuations.set(run.id, {
|
|
609
|
-
stepId: queuedContinuation.id,
|
|
610
|
-
sessionId: sessionBackend === backend ? sessionStep.sessionId : undefined,
|
|
611
|
-
backend,
|
|
612
|
-
prompt: RESTART_CONTINUATION_PROMPT,
|
|
613
|
-
images: [],
|
|
614
|
-
});
|
|
615
|
-
this.queue.push(run.id);
|
|
616
|
-
this.store.appendEvent(run.id, {
|
|
617
|
-
type: 'lifecycle',
|
|
618
|
-
message: 'cezar restarted — interrupted continuation re-queued',
|
|
619
|
-
});
|
|
620
|
-
continue;
|
|
621
|
-
}
|
|
622
|
-
const workflow = await this.reviveWorkflow(run);
|
|
623
|
-
if (workflow) {
|
|
624
|
-
// Re-apply the inbox ceiling (#471). `execute()` gates again at spawn time, so the
|
|
625
|
-
// agent is safe either way — but a run queued while the inbox was on and recovered
|
|
626
|
-
// after it was switched off would otherwise keep echoing `generateFollowups: true`
|
|
627
|
-
// on a run that demonstrably produced none. Normalize the record, the way startRun
|
|
628
|
-
// does, so the stored answer matches what actually happens.
|
|
629
|
-
const generateFollowups = followupsEnabled() ? run.generateFollowups : false;
|
|
630
|
-
if (generateFollowups !== run.generateFollowups) {
|
|
631
|
-
this.store.updateRun(run.id, { generateFollowups });
|
|
632
|
-
}
|
|
633
|
-
this.pendingJobs.set(run.id, {
|
|
634
|
-
workflow,
|
|
635
|
-
// Folded through the same helper `pump()` uses (#472) so a restart
|
|
636
|
-
// carries the stack. Idempotent: hydration always composes from
|
|
637
|
-
// `run.task` + the stack, never from an already-folded `input.task`,
|
|
638
|
-
// so re-hydrating at dequeue yields the same string, not a doubled one.
|
|
639
|
-
input: this.hydrateQueuedInput(run.id, {
|
|
640
|
-
task: run.task,
|
|
641
|
-
model: run.model,
|
|
642
|
-
runner: run.runner,
|
|
643
|
-
generateFollowups,
|
|
644
|
-
// Re-thread autonomy (#489): the rebuilt input feeds `execute`,
|
|
645
|
-
// whose mid-run auto-nudge reads `input.autonomous`. Without this a
|
|
646
|
-
// recovered queued autonomous run would run non-autonomously (no
|
|
647
|
-
// auto-nudge) and later wrongly park at `review`.
|
|
648
|
-
autonomous: run.autonomous,
|
|
649
|
-
// Preserve an explicit worktree opt-out across a queued restart.
|
|
650
|
-
// Missing on older records means the default isolated mode.
|
|
651
|
-
worktree: run.worktree,
|
|
652
|
-
}),
|
|
653
|
-
});
|
|
654
|
-
this.queue.push(run.id);
|
|
655
|
-
this.store.appendEvent(run.id, { type: 'lifecycle', message: 'cezar restarted — task re-queued' });
|
|
656
|
-
}
|
|
657
|
-
else {
|
|
658
|
-
this.store.updateRun(run.id, {
|
|
659
|
-
status: 'failed',
|
|
660
|
-
error: 'interrupted — workflow definition not recoverable after a restart',
|
|
661
|
-
finishedAt: new Date().toISOString(),
|
|
662
|
-
});
|
|
663
|
-
this.store.appendEvent(run.id, {
|
|
664
|
-
type: 'lifecycle',
|
|
665
|
-
message: 'cezar restarted — workflow definition not recoverable, task failed',
|
|
666
|
-
});
|
|
667
|
-
}
|
|
886
|
+
await this.reviveQueuedRun(run, 'cezar restarted');
|
|
668
887
|
continue;
|
|
669
888
|
}
|
|
670
889
|
if (run.status === 'waiting') {
|
|
@@ -704,6 +923,11 @@ export class RunManager {
|
|
|
704
923
|
: `cezar restarted — could not resume the interrupted task (${resumed.error ?? 'unknown'})`,
|
|
705
924
|
});
|
|
706
925
|
}
|
|
926
|
+
// Re-arm usage-limit resumes (spec 2026-08-03-auto-resume-after-usage-limit): the wait is
|
|
927
|
+
// routinely longer than a cezar session, so the deadline is durable and the timer is rebuilt
|
|
928
|
+
// from it. `pump()` reconciles again on every sweep, so this is the fast path, not the only
|
|
929
|
+
// one — see `reconcileAutoResumes`.
|
|
930
|
+
this.reconcileAutoResumes();
|
|
707
931
|
void this.pump();
|
|
708
932
|
}
|
|
709
933
|
/** The persisted definition when it looks sane, else the catalog by name. */
|
|
@@ -730,9 +954,20 @@ export class RunManager {
|
|
|
730
954
|
this.active.delete(runId);
|
|
731
955
|
this.memoryPausing.delete(runId);
|
|
732
956
|
this.lastNamerKey.delete(runId);
|
|
957
|
+
this.forceStarted.delete(runId);
|
|
733
958
|
// The run's slot is gone from busySlots() as of the deletes above — hand it
|
|
734
959
|
// to the workspace's oldest queued run, in ANY project. Every terminal path
|
|
735
960
|
// funnels through here, so this one call covers them all.
|
|
961
|
+
// Same reasoning as retention below — every terminal path funnels through here, so the
|
|
962
|
+
// usage-limit question ("did this run stop because the account is out of window, and when
|
|
963
|
+
// does that window reopen?") is asked once, in one place, off the record the failing path
|
|
964
|
+
// has already written. Nothing to do for any other outcome.
|
|
965
|
+
//
|
|
966
|
+
// BEFORE releasing the slot, and that order is the whole point: `releaseSlot` pumps every
|
|
967
|
+
// manager, and a pump reads the hold off the records. Publishing the schedule afterwards
|
|
968
|
+
// left a window — measured as exactly one extra task — where the queue saw a free slot and
|
|
969
|
+
// an account that looked healthy, and started work that was already doomed.
|
|
970
|
+
this.scheduleAutoResumeIfLimited(runId);
|
|
736
971
|
this.releaseSlot();
|
|
737
972
|
// A run leaving the active registry is a terminal transition (done/review/
|
|
738
973
|
// failed/cancelled) — the one moment the finished-worktree count can grow.
|
|
@@ -740,6 +975,338 @@ export class RunManager {
|
|
|
740
975
|
// terminal path. Fire-and-forget: retention must never delay or throw into
|
|
741
976
|
// the lifecycle.
|
|
742
977
|
void this.enforceRetention();
|
|
978
|
+
// The run's temp directory (#785) goes on the same terminal transition, and
|
|
979
|
+
// unconditionally — it is scratch, not an artifact, so unlike a worktree
|
|
980
|
+
// there is no keep-count to respect and nothing left to recover from it. A
|
|
981
|
+
// Continue (or an auto-resume) re-creates it through `agentEnv`.
|
|
982
|
+
removeAgentTmpDir(this.dataDir, runId);
|
|
983
|
+
}
|
|
984
|
+
// ---- usage-limit auto-resume (spec 2026-08-03-auto-resume-after-usage-limit) --------------
|
|
985
|
+
/**
|
|
986
|
+
* A run just failed: if the provider said "usage limit, back at T", promise to resume it at
|
|
987
|
+
* `T + AUTO_RESUME_GRACE_MS` instead of leaving the task dead until someone notices.
|
|
988
|
+
*
|
|
989
|
+
* Every refusal below is silent-but-honest — the run stays `failed` with its Continue button,
|
|
990
|
+
* which is exactly the pre-feature behavior — except the safety cap, which says so on the
|
|
991
|
+
* transcript, because a run that stops resuming itself needs to explain why.
|
|
992
|
+
*/
|
|
993
|
+
scheduleAutoResumeIfLimited(runId) {
|
|
994
|
+
if (this.autoResumeTimers.has(runId))
|
|
995
|
+
return; // already promised
|
|
996
|
+
const run = this.store.getRun(runId);
|
|
997
|
+
if (!run || run.status !== 'failed')
|
|
998
|
+
return;
|
|
999
|
+
// Archiving IS resigning from a task. Reviving one because a window happened to reopen would
|
|
1000
|
+
// be the feature working against the clearest signal the user can give it.
|
|
1001
|
+
if (run.archived)
|
|
1002
|
+
return;
|
|
1003
|
+
const limit = parseUsageLimit(run.error);
|
|
1004
|
+
if (!limit)
|
|
1005
|
+
return;
|
|
1006
|
+
if (!this.semaphore.autoResumeOnUsageLimit())
|
|
1007
|
+
return;
|
|
1008
|
+
// No session to resume = nothing this feature can do; `continueRun` would refuse anyway.
|
|
1009
|
+
if (!run.steps.some((step) => step.sessionId))
|
|
1010
|
+
return;
|
|
1011
|
+
const attempts = run.autoResumeAttempts ?? 0;
|
|
1012
|
+
if (attempts >= MAX_AUTO_RESUMES) {
|
|
1013
|
+
this.store.appendEvent(runId, {
|
|
1014
|
+
type: 'note',
|
|
1015
|
+
message: `automatic resume cap reached (${MAX_AUTO_RESUMES}) — continue this task manually`,
|
|
1016
|
+
});
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
const wakeAt = new Date(limit.resetAt.getTime() + AUTO_RESUME_GRACE_MS);
|
|
1020
|
+
this.armAutoResume(runId, wakeAt.getTime());
|
|
1021
|
+
this.store.appendEvent(runId, {
|
|
1022
|
+
type: 'lifecycle',
|
|
1023
|
+
message: `usage limit reached — resuming automatically at ${formatWakeInstant(wakeAt)}`,
|
|
1024
|
+
});
|
|
1025
|
+
}
|
|
1026
|
+
/** Publish the deadline on the record (the cockpit's only source) and arm the timer for it. */
|
|
1027
|
+
armAutoResume(runId, deadline) {
|
|
1028
|
+
this.store.updateRun(runId, { autoResumeAt: new Date(deadline).toISOString() });
|
|
1029
|
+
const timer = setTimeout(() => this.fireAutoResume(runId), Math.max(0, deadline - Date.now()));
|
|
1030
|
+
timer.unref?.();
|
|
1031
|
+
this.autoResumeTimers.set(runId, timer);
|
|
1032
|
+
}
|
|
1033
|
+
/**
|
|
1034
|
+
* The window has reopened. Re-check the record synchronously — hours may have passed, and the
|
|
1035
|
+
* user may have continued, deleted or cancelled the run in them — then hand the resume to the
|
|
1036
|
+
* ordinary queued-continuation path so it obeys both concurrency caps like any other work.
|
|
1037
|
+
*/
|
|
1038
|
+
fireAutoResume(runId) {
|
|
1039
|
+
this.autoResumeTimers.delete(runId);
|
|
1040
|
+
const run = this.store.getRun(runId);
|
|
1041
|
+
if (!run || run.status !== 'failed' || !run.autoResumeAt)
|
|
1042
|
+
return;
|
|
1043
|
+
// Belt and braces against the one gap `reconcileAutoResumes` cannot close: the setting going
|
|
1044
|
+
// off in the window between the last pump and this tick.
|
|
1045
|
+
if (!this.semaphore.autoResumeOnUsageLimit()) {
|
|
1046
|
+
this.clearAutoResume(runId);
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
1049
|
+
const attempts = (run.autoResumeAttempts ?? 0) + 1;
|
|
1050
|
+
// `continueRun` retires the pending resume (timer + record fields) on the way in — this is a
|
|
1051
|
+
// resume, not a user turn, so the counter is put back straight after.
|
|
1052
|
+
const resumed = this.continueRun(runId, { text: AUTO_RESUME_PROMPT }, true);
|
|
1053
|
+
if (!resumed.ok) {
|
|
1054
|
+
// Refusals happen before `continueRun` retires anything, so the deadline is still on the
|
|
1055
|
+
// record — and a deadline in the past is a promise the cockpit keeps displaying and the
|
|
1056
|
+
// engine will never keep. Retire it here instead, and say why.
|
|
1057
|
+
this.clearAutoResume(runId);
|
|
1058
|
+
this.store.appendEvent(runId, {
|
|
1059
|
+
type: 'note',
|
|
1060
|
+
message: `automatic resume could not start — ${resumed.error ?? 'unknown'}`,
|
|
1061
|
+
});
|
|
1062
|
+
return;
|
|
1063
|
+
}
|
|
1064
|
+
this.store.updateRun(runId, { autoResumeAttempts: attempts });
|
|
1065
|
+
this.store.appendEvent(runId, {
|
|
1066
|
+
type: 'lifecycle',
|
|
1067
|
+
message: `usage limit reset — resuming automatically (${attempts}/${MAX_AUTO_RESUMES})`,
|
|
1068
|
+
});
|
|
1069
|
+
// A deferred continuation only ENQUEUES itself; the queue moves when something pumps it, and
|
|
1070
|
+
// `recover()` — the other deferring caller — pumps once after its whole bulk sweep. A timer
|
|
1071
|
+
// firing on its own has no such follow-up, so without this the resumed run sits at `queued`
|
|
1072
|
+
// until some unrelated run happens to finish. This is the pump for it.
|
|
1073
|
+
void this.pump();
|
|
1074
|
+
}
|
|
1075
|
+
/**
|
|
1076
|
+
* Make the armed timers agree with the records and the current setting. Runs on every `pump()`
|
|
1077
|
+
* — which is where a settings change lands (a config PUT refreshes the shared semaphore, which
|
|
1078
|
+
* pumps every manager) — and once from `recover()`.
|
|
1079
|
+
*
|
|
1080
|
+
* It is a RECONCILE rather than a one-shot restore because the deadline is durable state and
|
|
1081
|
+
* the timer is not: a restart, a rebuilt project context, a manager disposed mid-wait, or a
|
|
1082
|
+
* refusal all leave a record promising a resume that no timer is holding. Rebuilding from the
|
|
1083
|
+
* record covers every one of those at once — the alternative is a hint counting down to a time
|
|
1084
|
+
* that has already passed, which is exactly the failure this method exists to make impossible.
|
|
1085
|
+
*
|
|
1086
|
+
* Cheap: an in-memory scan, and arming is skipped for every run already held.
|
|
1087
|
+
*/
|
|
1088
|
+
reconcileAutoResumes() {
|
|
1089
|
+
if (!this.semaphore.autoResumeOnUsageLimit()) {
|
|
1090
|
+
// Sweep the RECORDS, not the timer map. A record promising a resume that no timer is
|
|
1091
|
+
// holding is the exact population this method exists for, and it is also the one the
|
|
1092
|
+
// setting can be switched off in front of: cezar restarted while it was off, the config
|
|
1093
|
+
// was hand-edited, or the project context was disposed mid-wait. Retiring only the armed
|
|
1094
|
+
// timers leaves such a record with a live `autoResumeAt`, which `accountHolds()` reads as
|
|
1095
|
+
// a deadline hold — so nothing new starts on that account, `rescueStalledQueue` treats the
|
|
1096
|
+
// phantom appointment as a legitimate reason to sit still, and the cockpit shows a
|
|
1097
|
+
// `scheduled` row for a resume that will never come. `clearAutoResume` covers the armed
|
|
1098
|
+
// ones too, so this one loop is the whole cancellation.
|
|
1099
|
+
const pending = new Set([
|
|
1100
|
+
...this.autoResumeTimers.keys(),
|
|
1101
|
+
...this.store.listRuns().filter((run) => run.autoResumeAt !== undefined).map((run) => run.id),
|
|
1102
|
+
]);
|
|
1103
|
+
for (const runId of pending) {
|
|
1104
|
+
this.clearAutoResume(runId);
|
|
1105
|
+
this.store.appendEvent(runId, {
|
|
1106
|
+
type: 'note',
|
|
1107
|
+
message: 'automatic resume cancelled — auto-resume is switched off',
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
1112
|
+
for (const run of this.store.listRuns()) {
|
|
1113
|
+
if (run.status !== 'failed' || !run.autoResumeAt)
|
|
1114
|
+
continue;
|
|
1115
|
+
if (this.autoResumeTimers.has(run.id))
|
|
1116
|
+
continue;
|
|
1117
|
+
const deadline = Date.parse(run.autoResumeAt);
|
|
1118
|
+
// A deadline that is unreadable, belongs to a run that has spent its cap, or belongs to a
|
|
1119
|
+
// task the user has archived is retired rather than re-armed: it can only mislead. One
|
|
1120
|
+
// that has just passed arms at zero — the window is open, which is the point.
|
|
1121
|
+
if (run.archived
|
|
1122
|
+
|| !Number.isFinite(deadline)
|
|
1123
|
+
|| (run.autoResumeAttempts ?? 0) >= MAX_AUTO_RESUMES) {
|
|
1124
|
+
this.store.updateRun(run.id, { autoResumeAt: undefined });
|
|
1125
|
+
continue;
|
|
1126
|
+
}
|
|
1127
|
+
// …and one missed by more than a day is retired loudly: reviving a task from another era
|
|
1128
|
+
// is a surprise, not a service, and this is what keeps a sweep from resurrecting every
|
|
1129
|
+
// limit-stopped task a user has long since walked away from.
|
|
1130
|
+
if (Date.now() - deadline > AUTO_RESUME_MISSED_WINDOW_MS) {
|
|
1131
|
+
this.store.updateRun(run.id, { autoResumeAt: undefined });
|
|
1132
|
+
this.store.appendEvent(run.id, {
|
|
1133
|
+
type: 'note',
|
|
1134
|
+
message: 'automatic resume expired — its window reopened over a day ago; continue this task manually',
|
|
1135
|
+
});
|
|
1136
|
+
continue;
|
|
1137
|
+
}
|
|
1138
|
+
this.armAutoResume(run.id, deadline);
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
/**
|
|
1142
|
+
* Hand a run that has not spawned anything back to the queue, when the account it would run on
|
|
1143
|
+
* went into a usage-limit hold (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
1144
|
+
*
|
|
1145
|
+
* The dequeue-time gate in `pump()` cannot be the only one: a run can sit between dequeue and
|
|
1146
|
+
* spawn for a long time — an in-place run waiting for the exclusive repo-root lease is the
|
|
1147
|
+
* measured case — and the account can close in that gap. This is the last honest moment to
|
|
1148
|
+
* refuse, because everything after it costs a real agent turn.
|
|
1149
|
+
*
|
|
1150
|
+
* "Untouched" is the contract: the run has created no session and no worktree, so it goes back
|
|
1151
|
+
* as plain `queued` with its `startedAt` cleared, and `pump()` will pick it up when the window
|
|
1152
|
+
* reopens. Returns true when the caller must abandon the run.
|
|
1153
|
+
*/
|
|
1154
|
+
requeueWhileHeld(runId, workflow, input, runner, state) {
|
|
1155
|
+
const run = this.store.getRun(runId);
|
|
1156
|
+
if (!run || run.status === 'cancelled' || state?.cancelled)
|
|
1157
|
+
return false;
|
|
1158
|
+
// The watchdog sent this one through. Checked, never consumed: the spawn path asks this
|
|
1159
|
+
// question TWICE — here at the top of `execute`, and again after the exclusive repo-root
|
|
1160
|
+
// lease is granted — so a one-shot flag would clear at the first gate and let the second one
|
|
1161
|
+
// hand an in-place run straight back, re-wedging the queue the rescue had just freed.
|
|
1162
|
+
// `dropActive` retires the entry on every terminal path, so the set still cleans itself up.
|
|
1163
|
+
if (this.forceStarted.has(runId))
|
|
1164
|
+
return false;
|
|
1165
|
+
if (!accountHeldFor({ ...run, runner }, this.semaphore.accountHolds(), runner))
|
|
1166
|
+
return false;
|
|
1167
|
+
state?.releaseRepoRoot?.();
|
|
1168
|
+
if (state)
|
|
1169
|
+
state.releaseRepoRoot = undefined;
|
|
1170
|
+
this.pendingJobs.set(runId, { workflow, input });
|
|
1171
|
+
this.queue.push(runId);
|
|
1172
|
+
this.store.updateRun(runId, { status: 'queued', startedAt: undefined, currentStepId: undefined });
|
|
1173
|
+
this.store.appendEvent(runId, {
|
|
1174
|
+
type: 'note',
|
|
1175
|
+
message: 'held in the queue — this agent account is waiting out a usage limit',
|
|
1176
|
+
});
|
|
1177
|
+
this.dropActive(runId);
|
|
1178
|
+
return true;
|
|
1179
|
+
}
|
|
1180
|
+
/**
|
|
1181
|
+
* The failsafe: a queue must never be able to wedge.
|
|
1182
|
+
*
|
|
1183
|
+
* Everything else in this file makes an idle queue CORRECT under some condition — a slot cap, a
|
|
1184
|
+
* repo-root lease, and now a usage-limit hold. That is also what makes a wedged queue look
|
|
1185
|
+
* correct, and the hold has already produced one in the field: two resumes fired together, each
|
|
1186
|
+
* holding the account the other was waiting on, and the whole workspace stopped with every task
|
|
1187
|
+
* `queued`. That specific bug is fixed and tested, but "the queue stopped and nothing will ever
|
|
1188
|
+
* restart it" is too expensive a failure mode to leave resting on any single fix being right.
|
|
1189
|
+
*
|
|
1190
|
+
* The test is deliberately about JUSTIFICATION rather than about any particular bug: idling is
|
|
1191
|
+
* legitimate while work is running (here or in another project), or while a real appointment is
|
|
1192
|
+
* still ahead — a scheduled resume that will fire and pump on its own. Anything else is a
|
|
1193
|
+
* queue with work in it, nothing running anywhere, and no event coming to wake it. That gets one
|
|
1194
|
+
* forced sweep, which starts work under the ordinary caps and lets the account's real state
|
|
1195
|
+
* re-assert itself: if the window truly is shut, that task meets the limit and re-establishes an
|
|
1196
|
+
* honest hold, with a real deadline behind it this time.
|
|
1197
|
+
*
|
|
1198
|
+
* Public so a test can drive the wedge directly instead of waiting out the interval.
|
|
1199
|
+
*/
|
|
1200
|
+
async rescueStalledQueue(now = Date.now()) {
|
|
1201
|
+
// First, the worst shape: a record that says `queued` while the engine holds no job, no
|
|
1202
|
+
// continuation and no queue entry for it. `pump()` cannot see such a run — it iterates the
|
|
1203
|
+
// queue, and this one is not in it — so nothing will ever start it. Re-adopt it through the
|
|
1204
|
+
// same path boot recovery uses.
|
|
1205
|
+
for (const run of this.store.listRuns()) {
|
|
1206
|
+
if (run.status !== 'queued')
|
|
1207
|
+
continue;
|
|
1208
|
+
if (this.active.has(run.id) || this.starting.has(run.id))
|
|
1209
|
+
continue;
|
|
1210
|
+
if (this.pendingJobs.has(run.id) || this.pendingContinuations.has(run.id))
|
|
1211
|
+
continue;
|
|
1212
|
+
if (this.queue.includes(run.id))
|
|
1213
|
+
continue;
|
|
1214
|
+
console.warn(`[cez] queue watchdog: re-adopting queued run ${run.id} the engine had lost`);
|
|
1215
|
+
await this.reviveQueuedRun(run, 'queue watchdog');
|
|
1216
|
+
}
|
|
1217
|
+
if (this.queue.length === 0)
|
|
1218
|
+
return;
|
|
1219
|
+
if (this.busySlots() > 0 || this.starting.size > 0)
|
|
1220
|
+
return;
|
|
1221
|
+
if (this.semaphore.busy() > 0)
|
|
1222
|
+
return;
|
|
1223
|
+
// A future deadline is a real reason to sit still: that timer will fire and pump.
|
|
1224
|
+
for (const run of this.store.listRuns()) {
|
|
1225
|
+
if (run.status !== 'failed' || !run.autoResumeAt)
|
|
1226
|
+
continue;
|
|
1227
|
+
const deadline = Date.parse(run.autoResumeAt);
|
|
1228
|
+
if (Number.isFinite(deadline) && deadline > now)
|
|
1229
|
+
return;
|
|
1230
|
+
}
|
|
1231
|
+
if (this.semaphore.accountHolds().inFlight.size === 0) {
|
|
1232
|
+
// Not the hold, then — some other wakeup went missing. An ordinary pump is the whole fix,
|
|
1233
|
+
// and it is idempotent, so this stays quiet.
|
|
1234
|
+
void this.pump();
|
|
1235
|
+
return;
|
|
1236
|
+
}
|
|
1237
|
+
console.warn('[cez] queue watchdog: work is queued, nothing is running, and the usage-limit hold has no'
|
|
1238
|
+
+ ' deadline behind it — starting the next task anyway');
|
|
1239
|
+
this.forceNextPump = true;
|
|
1240
|
+
void this.pump();
|
|
1241
|
+
}
|
|
1242
|
+
/**
|
|
1243
|
+
* The accounts this project is currently holding: one key per run parked on a usage-limit
|
|
1244
|
+
* resume that has not come due yet (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
1245
|
+
*
|
|
1246
|
+
* Published to the shared semaphore so the hold spans PROJECTS — one Claude account can be
|
|
1247
|
+
* driving tasks in three repos, and a limit closes it for all of them. Derived from the
|
|
1248
|
+
* records on every ask rather than tracked as state: a deadline that passes, a resume that
|
|
1249
|
+
* fires, a cancel, an archive and a delete all lift the hold with no bookkeeping.
|
|
1250
|
+
*
|
|
1251
|
+
* Deliberately excludes a deadline that has already passed — that run is about to resume, and
|
|
1252
|
+
* holding the queue for it would only stall the very work the window reopened for.
|
|
1253
|
+
*/
|
|
1254
|
+
accountHolds(now = Date.now()) {
|
|
1255
|
+
const deadline = new Set();
|
|
1256
|
+
const inFlight = new Set();
|
|
1257
|
+
for (const run of this.store.listRuns()) {
|
|
1258
|
+
// A holding run always carries the runner it actually ran on, so the fallback is unused
|
|
1259
|
+
// here — it is spelled out rather than `!` so a future record shape degrades, not throws.
|
|
1260
|
+
const key = () => runAccountKey(run, run.runner ?? 'claude');
|
|
1261
|
+
if (run.status === 'failed' && run.autoResumeAt) {
|
|
1262
|
+
const at = Date.parse(run.autoResumeAt);
|
|
1263
|
+
if (Number.isFinite(at) && at > now)
|
|
1264
|
+
deadline.add(key());
|
|
1265
|
+
}
|
|
1266
|
+
else if (resumeInFlight(run)) {
|
|
1267
|
+
inFlight.add(key());
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
return { deadline, inFlight };
|
|
1271
|
+
}
|
|
1272
|
+
/**
|
|
1273
|
+
* The PER-TASK off switch (`DELETE /api/v1/runs/:id/auto-resume`, and the archive route):
|
|
1274
|
+
* stop resuming THIS task, without touching the workspace setting or any other task.
|
|
1275
|
+
*
|
|
1276
|
+
* Idempotent — a run with nothing pending answers the same way, because "this task will not
|
|
1277
|
+
* resume itself" is equally true either way. Returns false only when the run does not exist,
|
|
1278
|
+
* which is the route's 404.
|
|
1279
|
+
*/
|
|
1280
|
+
cancelAutoResume(runId) {
|
|
1281
|
+
const run = this.store.getRun(runId);
|
|
1282
|
+
if (!run)
|
|
1283
|
+
return false;
|
|
1284
|
+
const pending = run.autoResumeAt !== undefined || this.autoResumeTimers.has(runId);
|
|
1285
|
+
this.clearAutoResume(runId);
|
|
1286
|
+
if (pending) {
|
|
1287
|
+
this.store.appendEvent(runId, {
|
|
1288
|
+
type: 'note',
|
|
1289
|
+
message: 'automatic resume cancelled for this task',
|
|
1290
|
+
});
|
|
1291
|
+
// This run may have been the last thing holding its account's queue — nothing else will
|
|
1292
|
+
// notice, since the hold is derived and its release is not an event.
|
|
1293
|
+
void this.pump();
|
|
1294
|
+
}
|
|
1295
|
+
return true;
|
|
1296
|
+
}
|
|
1297
|
+
/** Retire a pending resume — timer, deadline and counter. The counter goes too because every
|
|
1298
|
+
* caller is a fresh epoch: a human Continue, or a resume that re-stamps its own count. */
|
|
1299
|
+
clearAutoResume(runId) {
|
|
1300
|
+
const timer = this.autoResumeTimers.get(runId);
|
|
1301
|
+
if (timer)
|
|
1302
|
+
clearTimeout(timer);
|
|
1303
|
+
this.autoResumeTimers.delete(runId);
|
|
1304
|
+
const run = this.store.getRun(runId);
|
|
1305
|
+
if (!run)
|
|
1306
|
+
return;
|
|
1307
|
+
if (run.autoResumeAt !== undefined || run.autoResumeAttempts !== undefined) {
|
|
1308
|
+
this.store.updateRun(runId, { autoResumeAt: undefined, autoResumeAttempts: undefined });
|
|
1309
|
+
}
|
|
743
1310
|
}
|
|
744
1311
|
/** Reclaim finished worktrees beyond the keep-limit (#483) — directory only,
|
|
745
1312
|
* `cez/<id8>` branch kept. Best-effort; a failure never affects run
|
|
@@ -1201,6 +1768,9 @@ export class RunManager {
|
|
|
1201
1768
|
/** Restart recovery may discover several interrupted tasks at once. Those
|
|
1202
1769
|
* continuations are queued; an explicit user Continue remains immediate. */
|
|
1203
1770
|
deferForCapacity = false) {
|
|
1771
|
+
if (agentModelsLocked(this.repoRoot) && opts.model?.trim()) {
|
|
1772
|
+
return { ok: false, error: AGENT_MODELS_LOCKED_ERROR };
|
|
1773
|
+
}
|
|
1204
1774
|
if (this.active.has(runId))
|
|
1205
1775
|
return { ok: false, error: 'run is still active' };
|
|
1206
1776
|
const run = this.store.getRun(runId);
|
|
@@ -1251,6 +1821,11 @@ export class RunManager {
|
|
|
1251
1821
|
: {}),
|
|
1252
1822
|
});
|
|
1253
1823
|
}
|
|
1824
|
+
// Everything that could refuse this continuation has now passed, so a pending usage-limit
|
|
1825
|
+
// resume is superseded either way: this IS that resume (it re-stamps its own counter), or a
|
|
1826
|
+
// human got there first — and then the counter starts over, because the cap only exists to
|
|
1827
|
+
// bound UNATTENDED resumes.
|
|
1828
|
+
this.clearAutoResume(runId);
|
|
1254
1829
|
const continuations = run.steps.filter((s) => s.id.startsWith('continue-')).length;
|
|
1255
1830
|
const stepId = `continue-${continuations + 1}`;
|
|
1256
1831
|
this.store.addStep(runId, { id: stepId, name: 'Continue', kind: 'agent' });
|
|
@@ -1338,6 +1913,11 @@ export class RunManager {
|
|
|
1338
1913
|
this.armAutosave(state);
|
|
1339
1914
|
if (record)
|
|
1340
1915
|
seedHandoffFile(this.dataDir, record); // idempotent — normally already there
|
|
1916
|
+
// Registry snapshot for `/skill` expansion. `execute` loads this for the workflow's own
|
|
1917
|
+
// sessions; a continuation builds its OWN ActiveRun, and without this the resumed session
|
|
1918
|
+
// expanded against an empty registry and leaked `/om-...` verbatim to the backend, which
|
|
1919
|
+
// answered "Unknown skill" (#811). Best-effort — discovery must never break Continue.
|
|
1920
|
+
state.skills = await discoverSkills(this.repoRoot).catch(() => []);
|
|
1341
1921
|
this.store.updateRun(runId, {
|
|
1342
1922
|
status: 'running',
|
|
1343
1923
|
error: undefined,
|
|
@@ -1353,17 +1933,6 @@ export class RunManager {
|
|
|
1353
1933
|
backend,
|
|
1354
1934
|
});
|
|
1355
1935
|
this.store.appendEvent(runId, { type: 'step-start', stepId, name: 'Continue', kind: 'agent', iteration: 1 });
|
|
1356
|
-
// Registry snapshot for `/skill` expansion. `execute` loads this for the workflow's own
|
|
1357
|
-
// sessions; a continuation builds its OWN ActiveRun, and without this the resumed session
|
|
1358
|
-
// expanded against an empty registry and leaked `/om-...` verbatim to the backend, which
|
|
1359
|
-
// answered "Unknown skill" (#811). Best-effort — discovery must never break Continue.
|
|
1360
|
-
//
|
|
1361
|
-
// Loaded AFTER the run is marked `running`, not before: `continueRun` returns the moment it
|
|
1362
|
-
// schedules this, so every await ahead of that write is a window in which the run still
|
|
1363
|
-
// reads with its previous TERMINAL status. A directory scan there is long enough to be
|
|
1364
|
-
// observed. Nothing between here and `startSession` reads `state.skills`, and no message can
|
|
1365
|
-
// be delivered into a session that does not exist yet, so the invariant is unaffected.
|
|
1366
|
-
state.skills = await discoverSkills(this.repoRoot).catch(() => []);
|
|
1367
1936
|
// Attachments pasted into the follow-up composer, on the same terms as a live-session
|
|
1368
1937
|
// message (#357): persisted to the run's own image store so the thread renders the bubble's
|
|
1369
1938
|
// images rather than a bare count, and handed to the agent BOTH as base64 blocks (so it can
|
|
@@ -1384,6 +1953,7 @@ export class RunManager {
|
|
|
1384
1953
|
});
|
|
1385
1954
|
let stepCost = 0;
|
|
1386
1955
|
let turnText = '';
|
|
1956
|
+
let sessionError;
|
|
1387
1957
|
const sink = this.makeUiSink(runId, stepId);
|
|
1388
1958
|
const onEvent = (event) => {
|
|
1389
1959
|
if (event.type === 'image') {
|
|
@@ -1400,6 +1970,13 @@ export class RunManager {
|
|
|
1400
1970
|
return;
|
|
1401
1971
|
}
|
|
1402
1972
|
this.store.appendEvent(runId, { ...event, stepId });
|
|
1973
|
+
if (event.type === 'error') {
|
|
1974
|
+
sessionError ??= event.message;
|
|
1975
|
+
state.session?.interrupt();
|
|
1976
|
+
return;
|
|
1977
|
+
}
|
|
1978
|
+
if (sessionError)
|
|
1979
|
+
return;
|
|
1403
1980
|
if (event.type === 'session') {
|
|
1404
1981
|
this.store.updateStep(runId, stepId, { sessionId: event.sessionId, backend });
|
|
1405
1982
|
}
|
|
@@ -1478,21 +2055,51 @@ export class RunManager {
|
|
|
1478
2055
|
this.releaseSlot();
|
|
1479
2056
|
}
|
|
1480
2057
|
}
|
|
2058
|
+
// A turn that completed is the ONLY evidence the provider's window actually reopened, so
|
|
2059
|
+
// it is what retires the consecutive-resume counter — which in turn releases the account
|
|
2060
|
+
// hold for every other task queued behind it (spec
|
|
2061
|
+
// 2026-08-03-auto-resume-after-usage-limit). `settleSuccess` does the same for a run that
|
|
2062
|
+
// finishes outright; this covers the far more common "parked for the user" ending.
|
|
2063
|
+
if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
|
|
2064
|
+
this.store.updateRun(runId, { autoResumeAttempts: undefined });
|
|
2065
|
+
}
|
|
1481
2066
|
appendHandoffHeartbeat(this.dataDir, runId, `turn complete — status=${monitoring ? 'monitoring' : sessionOpen ? 'waiting' : 'running'}`);
|
|
1482
2067
|
}
|
|
1483
2068
|
};
|
|
1484
2069
|
// Backend + model come off the record: the run's current backend by default, or the
|
|
1485
2070
|
// follow-up override that `continueRun` persisted before scheduling (#401).
|
|
1486
2071
|
const continueBackend = backend;
|
|
2072
|
+
/** Settle this turn as a failure before anything is spawned — the shape both
|
|
2073
|
+
* pre-spawn gates below need (model identity, #405; temp directory, #785). */
|
|
2074
|
+
const failBeforeSpawn = (message) => {
|
|
2075
|
+
const failedAt = new Date().toISOString();
|
|
2076
|
+
sink.sessionEnded('error', message);
|
|
2077
|
+
this.store.updateStep(runId, stepId, {
|
|
2078
|
+
status: 'failed',
|
|
2079
|
+
error: message,
|
|
2080
|
+
finishedAt: failedAt,
|
|
2081
|
+
});
|
|
2082
|
+
this.store.updateRun(runId, {
|
|
2083
|
+
status: 'failed',
|
|
2084
|
+
error: `continue failed: ${message}`,
|
|
2085
|
+
finishedAt: failedAt,
|
|
2086
|
+
currentStepId: undefined,
|
|
2087
|
+
});
|
|
2088
|
+
this.store.appendEvent(runId, {
|
|
2089
|
+
type: 'lifecycle',
|
|
2090
|
+
message: `continue failed — ${message}`,
|
|
2091
|
+
});
|
|
2092
|
+
this.dropActive(runId);
|
|
2093
|
+
};
|
|
1487
2094
|
// Apply the SAME canonical-identity gate the first spawn applies (#405, review M1).
|
|
1488
2095
|
// A follow-up may switch both runner and model (#401), so without this the record keeps
|
|
1489
2096
|
// asserting the identity the run STARTED with while a different model serves the turn —
|
|
1490
|
-
// the exact defect
|
|
2097
|
+
// the exact defect that PR existed to remove — and the raw record string reaches the CLI
|
|
1491
2098
|
// in the un-normalised wire form the first step already converted away (`anthropic/opus`
|
|
1492
2099
|
// instead of `opus`). Fail loud here too rather than let the backend pick a default.
|
|
1493
2100
|
let continueModel;
|
|
1494
2101
|
try {
|
|
1495
|
-
const normalized = normalizeModelForBackend(continueBackend, record?.model);
|
|
2102
|
+
const normalized = normalizeModelForBackend(continueBackend, agentModelsLocked(this.repoRoot) ? undefined : record?.model, { configuredProvider: await configuredModelProvider(continueBackend, state.cwd) });
|
|
1496
2103
|
continueModel = normalized?.backendModel;
|
|
1497
2104
|
this.store.updateRun(runId, {
|
|
1498
2105
|
modelIdentity: normalized ? formatModelIdentity(normalized.identity) : undefined,
|
|
@@ -1501,26 +2108,32 @@ export class RunManager {
|
|
|
1501
2108
|
catch (err) {
|
|
1502
2109
|
if (!(err instanceof ModelIdentityError))
|
|
1503
2110
|
throw err;
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
2111
|
+
failBeforeSpawn(err.message);
|
|
2112
|
+
return;
|
|
2113
|
+
}
|
|
2114
|
+
// Resuming reattaches to a session that lives inside ONE account's config dir, so the
|
|
2115
|
+
// continuation must run under the account that created it — not whatever the project has
|
|
2116
|
+
// been switched to since. The owning step is the one carrying this session id.
|
|
2117
|
+
const resumedProfileId = sessionId === undefined
|
|
2118
|
+
? undefined
|
|
2119
|
+
: record?.steps.find((s) => s.sessionId === sessionId)?.profileId;
|
|
2120
|
+
// The temp-directory preflight (#785) rides along with the account resolution: a resumed
|
|
2121
|
+
// turn hits the same broken `/tmp` a fresh one would, and an agent whose shell silently
|
|
2122
|
+
// returns nothing is worse than a turn that refuses to start and says why.
|
|
2123
|
+
let continueProfile;
|
|
2124
|
+
try {
|
|
2125
|
+
continueProfile = await this.agentEnvForStep(runId, continueBackend, {
|
|
2126
|
+
generateFollowups,
|
|
2127
|
+
recordedProfileId: resumedProfileId,
|
|
1520
2128
|
});
|
|
1521
|
-
|
|
2129
|
+
}
|
|
2130
|
+
catch (err) {
|
|
2131
|
+
if (!(err instanceof AgentTempDirError))
|
|
2132
|
+
throw err;
|
|
2133
|
+
failBeforeSpawn(err.message);
|
|
1522
2134
|
return;
|
|
1523
2135
|
}
|
|
2136
|
+
this.store.updateStep(runId, stepId, { profileId: continueProfile.profileId });
|
|
1524
2137
|
const runner = createRunner(continueBackend);
|
|
1525
2138
|
state.currentStepId = stepId;
|
|
1526
2139
|
this.beginUsageInvocation(runId, state, stepId);
|
|
@@ -1540,8 +2153,8 @@ export class RunManager {
|
|
|
1540
2153
|
...(openingImages.length ? { images: openingImages } : {}),
|
|
1541
2154
|
cwd: state.cwd,
|
|
1542
2155
|
allowedTools: DEFAULT_ALLOWED_TOOLS,
|
|
1543
|
-
additionalDirectories:
|
|
1544
|
-
env:
|
|
2156
|
+
additionalDirectories: agentDirectories(join(this.dataDir, 'runs'), continueProfile.env),
|
|
2157
|
+
env: continueProfile.env,
|
|
1545
2158
|
model: continueModel,
|
|
1546
2159
|
sessionId,
|
|
1547
2160
|
resume: sessionId !== undefined,
|
|
@@ -1556,6 +2169,8 @@ export class RunManager {
|
|
|
1556
2169
|
const finishedAt = () => new Date().toISOString();
|
|
1557
2170
|
try {
|
|
1558
2171
|
await session.result;
|
|
2172
|
+
if (sessionError)
|
|
2173
|
+
throw new Error(sessionError);
|
|
1559
2174
|
sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
|
|
1560
2175
|
if (state.cancelled) {
|
|
1561
2176
|
this.store.updateStep(runId, stepId, { status: 'cancelled', finishedAt: finishedAt() });
|
|
@@ -1608,6 +2223,12 @@ export class RunManager {
|
|
|
1608
2223
|
// the config default. Per-step `runner` can still override it below.
|
|
1609
2224
|
const config = await loadConfig(this.repoRoot);
|
|
1610
2225
|
const taskBackend = input.runner ?? config.defaultRunner;
|
|
2226
|
+
// The account may have gone into a usage-limit hold since this run was dequeued — the queue
|
|
2227
|
+
// gate cannot be the only one, because dequeue is not the moment of no return. Nothing has
|
|
2228
|
+
// happened yet here, so the run goes back to the queue untouched (spec
|
|
2229
|
+
// 2026-08-03-auto-resume-after-usage-limit).
|
|
2230
|
+
if (this.requeueWhileHeld(runId, workflow, input, taskBackend))
|
|
2231
|
+
return;
|
|
1611
2232
|
// Extra system prompt (R2 2.3): POST override > config default; echoed on
|
|
1612
2233
|
// the record so the UI/API can show what the run actually used.
|
|
1613
2234
|
const extraSystemPrompt = resolveExtraSystemPrompt(input.systemPrompt, config.systemPrompt);
|
|
@@ -1617,7 +2238,7 @@ export class RunManager {
|
|
|
1617
2238
|
// can still override below); the authoritative fail-loud gate is at spawn.
|
|
1618
2239
|
let modelIdentity;
|
|
1619
2240
|
try {
|
|
1620
|
-
const normalized = normalizeModelForBackend(taskBackend, input.model);
|
|
2241
|
+
const normalized = normalizeModelForBackend(taskBackend, agentModelsLocked(this.repoRoot) ? undefined : input.model, { configuredProvider: await configuredModelProvider(taskBackend, this.repoRoot) });
|
|
1621
2242
|
modelIdentity = normalized ? formatModelIdentity(normalized.identity) : undefined;
|
|
1622
2243
|
}
|
|
1623
2244
|
catch {
|
|
@@ -1728,6 +2349,15 @@ export class RunManager {
|
|
|
1728
2349
|
// run through the usual path.
|
|
1729
2350
|
await this.acquireRepoRoot(runId, state);
|
|
1730
2351
|
}
|
|
2352
|
+
// THE window that matters for an in-place run. Waiting for the exclusive tree can take
|
|
2353
|
+
// minutes, and a run parked on that lease holds no slot (#347) — so the queue keeps
|
|
2354
|
+
// advancing behind it and the dequeue-time gate is long past. Measured with five in-place
|
|
2355
|
+
// tasks and `maxParallel: 2`: four of them started. Re-ask here, where the very next thing
|
|
2356
|
+
// is a spawn, and hand the run back to the queue if the account closed meanwhile. This
|
|
2357
|
+
// check also covers the explicit lock-bypass path, where the account may close while the
|
|
2358
|
+
// run is preparing its first step.
|
|
2359
|
+
if (this.requeueWhileHeld(runId, workflow, input, taskBackend, state))
|
|
2360
|
+
return;
|
|
1731
2361
|
}
|
|
1732
2362
|
// Handoff journal (spec 007) — seeded after the worktree exists so the
|
|
1733
2363
|
// header can name the branch. Idempotent: an existing file stays as-is.
|
|
@@ -1917,6 +2547,7 @@ export class RunManager {
|
|
|
1917
2547
|
const startTokens = stepRecord?.tokensUsed ?? 0;
|
|
1918
2548
|
let stepCost = stepRecord?.costUsd ?? 0;
|
|
1919
2549
|
let turnText = '';
|
|
2550
|
+
let sessionError;
|
|
1920
2551
|
const sink = this.makeUiSink(runId, step.id);
|
|
1921
2552
|
const onEvent = (event) => {
|
|
1922
2553
|
if (event.type === 'image') {
|
|
@@ -1933,6 +2564,13 @@ export class RunManager {
|
|
|
1933
2564
|
return;
|
|
1934
2565
|
}
|
|
1935
2566
|
emit({ ...event, stepId: step.id });
|
|
2567
|
+
if (event.type === 'error') {
|
|
2568
|
+
sessionError ??= event.message;
|
|
2569
|
+
state.session?.interrupt();
|
|
2570
|
+
return;
|
|
2571
|
+
}
|
|
2572
|
+
if (sessionError)
|
|
2573
|
+
return;
|
|
1936
2574
|
if (event.type === 'session') {
|
|
1937
2575
|
// Codex/OpenCode mint their own session id — persist it so resume works.
|
|
1938
2576
|
this.store.updateStep(runId, step.id, { sessionId: event.sessionId, backend });
|
|
@@ -2001,6 +2639,10 @@ export class RunManager {
|
|
|
2001
2639
|
this.armIdleTimer(runId, state);
|
|
2002
2640
|
this.releaseSlot(); // the freed slot can start a queued run right away — in any project
|
|
2003
2641
|
}
|
|
2642
|
+
// The window is proven open — see the twin in `runContinuation`.
|
|
2643
|
+
if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
|
|
2644
|
+
this.store.updateRun(runId, { autoResumeAttempts: undefined });
|
|
2645
|
+
}
|
|
2004
2646
|
// Cez's own heartbeat — the handoff stays current even when the
|
|
2005
2647
|
// agent forgets to write (spec 007).
|
|
2006
2648
|
appendHandoffHeartbeat(this.dataDir, runId, `turn complete — status=${monitoring ? 'monitoring' : waiting ? 'waiting' : 'running'}`);
|
|
@@ -2013,7 +2655,7 @@ export class RunManager {
|
|
|
2013
2655
|
// instead of letting the backend silently substitute its default.
|
|
2014
2656
|
let backendModel;
|
|
2015
2657
|
try {
|
|
2016
|
-
const normalized = normalizeModelForBackend(stepBackend, step.model ?? input.model);
|
|
2658
|
+
const normalized = normalizeModelForBackend(stepBackend, agentModelsLocked(this.repoRoot) ? undefined : step.model ?? input.model, { configuredProvider: await configuredModelProvider(stepBackend, state.cwd) });
|
|
2017
2659
|
backendModel = normalized?.backendModel;
|
|
2018
2660
|
// Persist the identity of what ACTUALLY runs (#405, review M1). The run-start echo
|
|
2019
2661
|
// (line ~993) is best-effort from `taskBackend`/`input.model`; a per-step `runner`/`model`
|
|
@@ -2028,6 +2670,23 @@ export class RunManager {
|
|
|
2028
2670
|
return err.message;
|
|
2029
2671
|
throw err;
|
|
2030
2672
|
}
|
|
2673
|
+
// Which agent account this step spawns under, and — recorded on the step before the spawn —
|
|
2674
|
+
// which one its session belongs to. `sessionId` and `profileId` are a pair: a resume that
|
|
2675
|
+
// reads the wrong account's config dir finds no session and silently starts a fresh one.
|
|
2676
|
+
// Resolved together with the temp-directory preflight (#785): the step fails with a named,
|
|
2677
|
+
// actionable error instead of spawning a backend whose shell would return empty output.
|
|
2678
|
+
let stepProfile;
|
|
2679
|
+
try {
|
|
2680
|
+
stepProfile = await this.agentEnvForStep(runId, stepBackend, {
|
|
2681
|
+
generateFollowups: followupsEnabled() && input.generateFollowups !== false,
|
|
2682
|
+
});
|
|
2683
|
+
}
|
|
2684
|
+
catch (err) {
|
|
2685
|
+
if (err instanceof AgentTempDirError)
|
|
2686
|
+
return err.message;
|
|
2687
|
+
throw err;
|
|
2688
|
+
}
|
|
2689
|
+
this.store.updateStep(runId, step.id, { profileId: stepProfile.profileId });
|
|
2031
2690
|
const runner = createRunner(stepBackend);
|
|
2032
2691
|
let session;
|
|
2033
2692
|
state.currentStepId = step.id;
|
|
@@ -2045,8 +2704,8 @@ export class RunManager {
|
|
|
2045
2704
|
allowedTools: step.allowedTools ?? DEFAULT_ALLOWED_TOOLS,
|
|
2046
2705
|
bashAllowlist: step.bashAllowlist,
|
|
2047
2706
|
// The handoff file lives outside the worktree — grant access.
|
|
2048
|
-
additionalDirectories:
|
|
2049
|
-
env:
|
|
2707
|
+
additionalDirectories: agentDirectories(join(this.dataDir, 'runs'), stepProfile.env),
|
|
2708
|
+
env: stepProfile.env,
|
|
2050
2709
|
model: backendModel,
|
|
2051
2710
|
sessionId,
|
|
2052
2711
|
// Interactive sessions have no wall clock — the idle timer rules.
|
|
@@ -2069,6 +2728,10 @@ export class RunManager {
|
|
|
2069
2728
|
registerRunProcess(runId, session.pid);
|
|
2070
2729
|
try {
|
|
2071
2730
|
const result = await session.result;
|
|
2731
|
+
if (sessionError) {
|
|
2732
|
+
sink.sessionEnded('error', sessionError);
|
|
2733
|
+
return sessionError;
|
|
2734
|
+
}
|
|
2072
2735
|
// v2 counterpart of v1's `done` (spec: the mappers leave session-close
|
|
2073
2736
|
// events to the RunManager — only it knows how the session settled).
|
|
2074
2737
|
sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
|
|
@@ -2371,6 +3034,10 @@ export class RunManager {
|
|
|
2371
3034
|
status: review ? 'review' : 'done',
|
|
2372
3035
|
finishedAt: new Date().toISOString(),
|
|
2373
3036
|
currentStepId: undefined,
|
|
3037
|
+
// A run that got all the way to a settled turn is not in a limit loop, so the resume
|
|
3038
|
+
// counter starts over — otherwise a task that legitimately met the limit once a week would
|
|
3039
|
+
// creep toward the cap forever and stop resuming for no reason anyone could see.
|
|
3040
|
+
autoResumeAttempts: undefined,
|
|
2374
3041
|
});
|
|
2375
3042
|
this.store.appendEvent(runId, {
|
|
2376
3043
|
type: 'lifecycle',
|