@open-mercato/cezar 0.9.1 → 0.9.2-pr710.1256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +157 -32
- package/dist/agent-config/account-identity.d.ts +44 -0
- package/dist/agent-config/account-identity.js +128 -0
- package/dist/agent-config/account-identity.js.map +1 -0
- package/dist/agent-config/catalog.d.ts +9 -1
- package/dist/agent-config/catalog.js +18 -0
- package/dist/agent-config/catalog.js.map +1 -1
- package/dist/agent-config/files.d.ts +1 -1
- package/dist/agent-config/model-settings/claude.d.ts +2 -0
- package/dist/agent-config/model-settings/claude.js +11 -0
- package/dist/agent-config/model-settings/claude.js.map +1 -0
- package/dist/agent-config/model-settings/codex.d.ts +2 -0
- package/dist/agent-config/model-settings/codex.js +16 -0
- package/dist/agent-config/model-settings/codex.js.map +1 -0
- package/dist/agent-config/model-settings/opencode.d.ts +2 -0
- package/dist/agent-config/model-settings/opencode.js +8 -0
- package/dist/agent-config/model-settings/opencode.js.map +1 -0
- package/dist/agent-config/model-settings/pi.d.ts +12 -0
- package/dist/agent-config/model-settings/pi.js +18 -0
- package/dist/agent-config/model-settings/pi.js.map +1 -0
- package/dist/agent-config/model-settings/shared.d.ts +9 -0
- package/dist/agent-config/model-settings/shared.js +93 -0
- package/dist/agent-config/model-settings/shared.js.map +1 -0
- package/dist/agent-config/model-settings/types.d.ts +9 -0
- package/dist/agent-config/model-settings/types.js +2 -0
- package/dist/agent-config/model-settings/types.js.map +1 -0
- package/dist/agent-config/models.d.ts +9 -0
- package/dist/agent-config/models.js +32 -0
- package/dist/agent-config/models.js.map +1 -0
- package/dist/agent-config/service.d.ts +1 -1
- package/dist/agent-config/service.js +4 -4
- package/dist/agent-config/service.js.map +1 -1
- package/dist/agent-config/validate.d.ts +1 -1
- package/dist/automations/coordinator.d.ts +26 -0
- package/dist/automations/coordinator.js +66 -0
- package/dist/automations/coordinator.js.map +1 -0
- package/dist/automations/github-poller.d.ts +74 -0
- package/dist/automations/github-poller.js +234 -0
- package/dist/automations/github-poller.js.map +1 -0
- package/dist/automations/scheduler.d.ts +42 -0
- package/dist/automations/scheduler.js +192 -0
- package/dist/automations/scheduler.js.map +1 -0
- package/dist/automations/store.d.ts +65 -0
- package/dist/automations/store.js +298 -0
- package/dist/automations/store.js.map +1 -0
- package/dist/automations/task-template.d.ts +18 -0
- package/dist/automations/task-template.js +95 -0
- package/dist/automations/task-template.js.map +1 -0
- package/dist/automations/types.d.ts +255 -0
- package/dist/automations/types.js +159 -0
- package/dist/automations/types.js.map +1 -0
- package/dist/config.d.ts +19 -112
- package/dist/config.js +48 -5
- package/dist/config.js.map +1 -1
- package/dist/contract/agent-config.d.ts +153 -0
- package/dist/contract/agent-profiles.d.ts +347 -0
- package/dist/contract/automations.d.ts +935 -0
- package/dist/contract/events.d.ts +104 -0
- package/dist/contract/github.d.ts +467 -0
- package/dist/contract/health.d.ts +102 -0
- package/dist/contract/index.d.ts +16 -0
- package/dist/contract/index.js +1830 -0
- package/dist/contract/projects.d.ts +210 -0
- package/dist/contract/repo.d.ts +234 -0
- package/dist/contract/runs.d.ts +1407 -0
- package/dist/contract/skills.d.ts +232 -0
- package/dist/contract/workflows.d.ts +218 -0
- package/dist/contract/workspace.d.ts +577 -0
- package/dist/core/agent-env.d.ts +1 -1
- package/dist/core/agent-env.js +41 -16
- package/dist/core/agent-env.js.map +1 -1
- package/dist/core/agent-model-policy.d.ts +10 -0
- package/dist/core/agent-model-policy.js +36 -0
- package/dist/core/agent-model-policy.js.map +1 -0
- package/dist/core/agent-profiles.d.ts +62 -0
- package/dist/core/agent-profiles.js +90 -0
- package/dist/core/agent-profiles.js.map +1 -0
- package/dist/core/agent-runner.d.ts +49 -7
- package/dist/core/agent-runner.js +48 -3
- package/dist/core/agent-runner.js.map +1 -1
- package/dist/core/ask.d.ts +45 -124
- package/dist/core/ask.js +103 -13
- package/dist/core/ask.js.map +1 -1
- package/dist/core/backend-detect.d.ts +5 -5
- package/dist/core/backend-detect.js +44 -7
- package/dist/core/backend-detect.js.map +1 -1
- package/dist/core/claude-cli-runner.d.ts +2 -2
- package/dist/core/claude-cli-runner.js +61 -13
- package/dist/core/claude-cli-runner.js.map +1 -1
- package/dist/core/claude-ui-mapper.d.ts +1 -1
- package/dist/core/codex-app-server-runner.d.ts +1 -1
- package/dist/core/codex-app-server-runner.js +65 -3
- package/dist/core/codex-app-server-runner.js.map +1 -1
- package/dist/core/codex-app-server-transport.d.ts +6 -1
- package/dist/core/codex-app-server-transport.js +17 -3
- package/dist/core/codex-app-server-transport.js.map +1 -1
- package/dist/core/codex-model-catalog.d.ts +1 -1
- package/dist/core/codex-ui-mapper.d.ts +5 -1
- package/dist/core/codex-ui-mapper.js +74 -17
- package/dist/core/codex-ui-mapper.js.map +1 -1
- package/dist/core/model-identity.d.ts +24 -12
- package/dist/core/model-identity.js +41 -24
- package/dist/core/model-identity.js.map +1 -1
- package/dist/core/model-presets.d.ts +10 -4
- package/dist/core/model-presets.js +52 -9
- package/dist/core/model-presets.js.map +1 -1
- package/dist/core/opencode-model-catalog.d.ts +30 -0
- package/dist/core/opencode-model-catalog.js +126 -0
- package/dist/core/opencode-model-catalog.js.map +1 -0
- package/dist/core/opencode-server-runner.d.ts +2 -2
- package/dist/core/opencode-ui-mapper.d.ts +4 -1
- package/dist/core/opencode-ui-mapper.js +45 -7
- package/dist/core/opencode-ui-mapper.js.map +1 -1
- package/dist/core/pi-runner.d.ts +24 -0
- package/dist/core/pi-runner.js +349 -0
- package/dist/core/pi-runner.js.map +1 -0
- package/dist/core/pi-ui-mapper.d.ts +34 -0
- package/dist/core/pi-ui-mapper.js +270 -0
- package/dist/core/pi-ui-mapper.js.map +1 -0
- package/dist/core/provider-auth.d.ts +154 -0
- package/dist/core/provider-auth.js +509 -0
- package/dist/core/provider-auth.js.map +1 -0
- package/dist/core/provider-availability.d.ts +3 -0
- package/dist/core/provider-availability.js +14 -0
- package/dist/core/provider-availability.js.map +1 -0
- package/dist/core/runner-factory.d.ts +1 -1
- package/dist/core/runner-factory.js +3 -0
- package/dist/core/runner-factory.js.map +1 -1
- package/dist/core/runner-model-catalog.d.ts +1 -1
- package/dist/core/shell-env.d.ts +28 -0
- package/dist/core/shell-env.js +56 -0
- package/dist/core/shell-env.js.map +1 -0
- package/dist/core/tool-display.d.ts +1 -1
- package/dist/core/ui-events.d.ts +2 -2
- package/dist/core/ui-events.js +1 -1
- package/dist/core/usage-limit.d.ts +42 -0
- package/dist/core/usage-limit.js +209 -0
- package/dist/core/usage-limit.js.map +1 -0
- package/dist/git-diff-base.d.ts +85 -0
- package/dist/git-diff-base.js +180 -0
- package/dist/git-diff-base.js.map +1 -0
- package/dist/git-worktree.d.ts +46 -6
- package/dist/git-worktree.js +46 -9
- package/dist/git-worktree.js.map +1 -1
- package/dist/handoff.d.ts +2 -2
- package/dist/handoff.js +1 -1
- package/dist/index.js +66 -8
- package/dist/index.js.map +1 -1
- package/dist/paths.d.ts +69 -6
- package/dist/paths.js +86 -9
- package/dist/paths.js.map +1 -1
- package/dist/planner.d.ts +2 -2
- package/dist/planner.js +7 -1
- package/dist/planner.js.map +1 -1
- package/dist/release/manifests.d.ts +81 -0
- package/dist/release/manifests.js +74 -0
- package/dist/release/manifests.js.map +1 -0
- package/dist/release/snapshot.d.ts +9 -23
- package/dist/release/snapshot.js +9 -22
- package/dist/release/snapshot.js.map +1 -1
- package/dist/release/stable.d.ts +15 -29
- package/dist/release/stable.js +14 -27
- package/dist/release/stable.js.map +1 -1
- package/dist/runs/agent-tmpdir.d.ts +41 -0
- package/dist/runs/agent-tmpdir.js +183 -0
- package/dist/runs/agent-tmpdir.js.map +1 -0
- package/dist/runs/auto-name.d.ts +1 -1
- package/dist/runs/auto-name.js +6 -1
- package/dist/runs/auto-name.js.map +1 -1
- package/dist/runs/event-history.d.ts +50 -0
- package/dist/runs/event-history.js +613 -0
- package/dist/runs/event-history.js.map +1 -0
- package/dist/runs/retention.d.ts +1 -1
- package/dist/runs/run-index.d.ts +19 -0
- package/dist/runs/run-index.js +43 -0
- package/dist/runs/run-index.js.map +1 -0
- package/dist/runs/store.d.ts +209 -344
- package/dist/runs/store.js +298 -39
- package/dist/runs/store.js.map +1 -1
- package/dist/runs/ui-event-sink.d.ts +1 -1
- package/dist/server/app-type.d.ts +21 -0
- package/dist/server/app-type.js +2 -0
- package/dist/server/app-type.js.map +1 -0
- package/dist/server/capabilities.d.ts +21 -6
- package/dist/server/capabilities.js +27 -1
- package/dist/server/capabilities.js.map +1 -1
- package/dist/server/forge/github.d.ts +58 -151
- package/dist/server/forge/github.js +530 -6
- package/dist/server/forge/github.js.map +1 -1
- package/dist/server/forge/index.d.ts +19 -3
- package/dist/server/forge/index.js +28 -1
- package/dist/server/forge/index.js.map +1 -1
- package/dist/server/forge/types.d.ts +86 -1
- package/dist/server/git-changes.d.ts +10 -8
- package/dist/server/git-changes.js +19 -17
- package/dist/server/git-changes.js.map +1 -1
- package/dist/server/git.d.ts +2 -0
- package/dist/server/git.js +9 -0
- package/dist/server/git.js.map +1 -1
- package/dist/server/github.d.ts +3 -3
- package/dist/server/github.js +1 -1
- package/dist/server/github.js.map +1 -1
- package/dist/server/open-in-app.d.ts +2 -1
- package/dist/server/open-in-app.js +3 -1
- package/dist/server/open-in-app.js.map +1 -1
- package/dist/server/open-in-terminal.d.ts +46 -1
- package/dist/server/open-in-terminal.js +72 -13
- package/dist/server/open-in-terminal.js.map +1 -1
- package/dist/server/pr.d.ts +2 -2
- package/dist/server/project-context.d.ts +20 -3
- package/dist/server/project-context.js +30 -1
- package/dist/server/project-context.js.map +1 -1
- package/dist/server/provider-action-gate.d.ts +8 -0
- package/dist/server/provider-action-gate.js +56 -0
- package/dist/server/provider-action-gate.js.map +1 -0
- package/dist/server/provider-auth-runtime.d.ts +21 -0
- package/dist/server/provider-auth-runtime.js +66 -0
- package/dist/server/provider-auth-runtime.js.map +1 -0
- package/dist/server/server.d.ts +12802 -47
- package/dist/server/server.js +2857 -949
- package/dist/server/server.js.map +1 -1
- package/dist/server/validators.d.ts +97 -0
- package/dist/server/validators.js +86 -0
- package/dist/server/validators.js.map +1 -0
- package/dist/server/ws.d.ts +2 -2
- package/dist/server/ws.js +2 -2
- package/dist/server/ws.js.map +1 -1
- package/dist/server-install/engine.d.ts +1 -1
- package/dist/server-install/engine.js +5 -18
- package/dist/server-install/engine.js.map +1 -1
- package/dist/server-install/platforms/macosx-ngrok.d.ts +1 -1
- package/dist/server-install/platforms/ubuntu-vps.d.ts +16 -1
- package/dist/server-install/platforms/ubuntu-vps.js +83 -1
- package/dist/server-install/platforms/ubuntu-vps.js.map +1 -1
- package/dist/server-install/state.d.ts +1 -1
- package/dist/server-install/steps.d.ts +2 -2
- package/dist/server-install/steps.js +6 -2
- package/dist/server-install/steps.js.map +1 -1
- package/dist/server-install/strategies.d.ts +1 -1
- package/dist/server-install/types.d.ts +38 -424
- package/dist/server-install/ui.d.ts +1 -1
- package/dist/skills-banner.d.ts +7 -6
- package/dist/skills-banner.js +7 -6
- package/dist/skills-banner.js.map +1 -1
- package/dist/skills-remote.d.ts +2 -2
- package/dist/skills-update.d.ts +88 -0
- package/dist/skills-update.js +392 -0
- package/dist/skills-update.js.map +1 -0
- package/dist/skills.d.ts +2 -0
- package/dist/skills.js +7 -2
- package/dist/skills.js.map +1 -1
- package/dist/todos.d.ts +2 -28
- package/dist/todos.js +1 -1
- package/dist/ui-state.d.ts +16 -2
- package/dist/ui-state.js +14 -1
- package/dist/ui-state.js.map +1 -1
- package/dist/workflows/load.d.ts +1 -1
- package/dist/workflows/run.d.ts +276 -15
- package/dist/workflows/run.js +1261 -159
- package/dist/workflows/run.js.map +1 -1
- package/dist/workflows/types.d.ts +62 -224
- package/dist/workflows/types.js +25 -2
- package/dist/workflows/types.js.map +1 -1
- package/dist/workspace/agent-accounts.d.ts +153 -0
- package/dist/workspace/agent-accounts.js +304 -0
- package/dist/workspace/agent-accounts.js.map +1 -0
- package/dist/workspace/agent-profiles.d.ts +78 -0
- package/dist/workspace/agent-profiles.js +115 -0
- package/dist/workspace/agent-profiles.js.map +1 -0
- package/dist/workspace/config.d.ts +94 -206
- package/dist/workspace/config.js +226 -14
- package/dist/workspace/config.js.map +1 -1
- package/dist/workspace/migrations.d.ts +1 -1
- package/dist/workspace/migrations.js +19 -4
- package/dist/workspace/migrations.js.map +1 -1
- package/dist/workspace/projects-cli.js +54 -4
- package/dist/workspace/projects-cli.js.map +1 -1
- package/dist/workspace/projects.d.ts +26 -2
- package/dist/workspace/projects.js +42 -3
- package/dist/workspace/projects.js.map +1 -1
- package/dist/workspace/semaphore.d.ts +51 -0
- package/dist/workspace/semaphore.js +53 -3
- package/dist/workspace/semaphore.js.map +1 -1
- package/dist/workspace/ui-state.d.ts +16 -5
- package/dist/workspace/ui-state.js +18 -7
- package/dist/workspace/ui-state.js.map +1 -1
- package/package.json +17 -39
- package/scripts/inline-contract.mjs +112 -0
- package/scripts/mock-claude.mjs +59 -1
- package/scripts/mock-pi-rpc.mjs +83 -0
- package/scripts/sync-readme.mjs +20 -0
- package/web/dist/assets/alert-dialog-BVs3QSmP.js +1 -0
- package/web/dist/assets/arrow-down-D7T7XuF6.js +1 -0
- package/web/dist/assets/arrow-left-B1dRFd8y.js +1 -0
- package/web/dist/assets/bundle-mjs-BT31bpU6.js +1 -0
- package/web/dist/assets/centered-state-DdRLYi9u.js +43 -0
- package/web/dist/assets/chevron-right-Dh0t3AL8.js +1 -0
- package/web/dist/assets/{chunk-BO2N2NFS-DtrdTCWa.js → chunk-BO2N2NFS-DE6qKn3r.js} +8 -8
- package/web/dist/assets/circle-check-4FDOlh9u.js +1 -0
- package/web/dist/assets/circle-x-CgTVbqX5.js +1 -0
- package/web/dist/assets/collapsible-BoADzTok.js +1 -0
- package/web/dist/assets/commit-list-BCcCkRbU.js +1 -0
- package/web/dist/assets/compare-variants-B7gTgGqR.js +1 -0
- package/web/dist/assets/{core-DePtBHcl.js → core-BCsw8oQw.js} +1 -1
- package/web/dist/assets/diff-ecfV3-op.js +3 -0
- package/web/dist/assets/diff-stat-C2KGLVnZ.js +1 -0
- package/web/dist/assets/{diff-view-DCeSegKk.js → diff-view-CQtonQ4L.js} +4 -4
- package/web/dist/assets/dropdown-menu-WcGoirf2.js +1 -0
- package/web/dist/assets/editable-title-SqNK-wuI.js +1 -0
- package/web/dist/assets/{ellipsis-vertical-CNnSLn5a.js → ellipsis-vertical-ClTFrwit.js} +1 -1
- package/web/dist/assets/{file-D4GBh1ao.js → file-BU0mWida.js} +1 -1
- package/web/dist/assets/folder-rsgCF0aw.js +1 -0
- package/web/dist/assets/{git-pull-request-DnR8Xh4_.js → git-pull-request-BFhIKb83.js} +1 -1
- package/web/dist/assets/git-toolbar-CF1_tzqz.js +1 -0
- package/web/dist/assets/github-CyjPz-Ks.js +1 -0
- package/web/dist/assets/highlighted-body-OFNGDK62-BL-MfFgl.js +1 -0
- package/web/dist/assets/highlighter-CdHqIwFr.js +3 -0
- package/web/dist/assets/image-preview-iEzCyEmz.js +1 -0
- package/web/dist/assets/index-BN09RpYY.css +2 -0
- package/web/dist/assets/index-CyG7aE1W.js +7 -0
- package/web/dist/assets/lib-BQXq3kEf.js +1 -0
- package/web/dist/assets/lib-BxQXEXDF.js +1 -0
- package/web/dist/assets/markdown-CNAa22LB.js +2 -0
- package/web/dist/assets/mermaid-GHXKKRXX-C4jxqUUb.js +1 -0
- package/web/dist/assets/new-task-form-CWhQZbsX.js +1 -0
- package/web/dist/assets/pill-BO5s2x3N.js +1 -0
- package/web/dist/assets/project-router-Dm4OhAPE.js +1 -0
- package/web/dist/assets/prompt-templates-CP__OCDQ.js +15 -0
- package/web/dist/assets/react-runtime-CCIEwYL0.js +9 -0
- package/web/dist/assets/{refresh-cw-BT8E96Tm.js → refresh-cw-DyiysCoP.js} +1 -1
- package/web/dist/assets/repo-git-B1WM6N8T.js +1 -0
- package/web/dist/assets/rolldown-runtime-QTnfLwEv.js +1 -0
- package/web/dist/assets/run-diff-CKx4d9Oo.js +3 -0
- package/web/dist/assets/run-header-DpbfOtNi.js +1 -0
- package/web/dist/assets/{search-x-BDjaMrzG.js → search-x-DRvk-6x4.js} +1 -1
- package/web/dist/assets/skill-empty-hint-D5vsHfMr.js +1 -0
- package/web/dist/assets/skills-BiWfHmCR.js +1 -0
- package/web/dist/assets/skills-XGltyIJ_.js +1 -0
- package/web/dist/assets/sparkles-BugkDxWO.js +1 -0
- package/web/dist/assets/{square-terminal-Ckjjn0vb.js → square-terminal-Drp2h-o8.js} +1 -1
- package/web/dist/assets/tab-link-CiRLRlnY.js +1 -0
- package/web/dist/assets/task-changes-C2hKkEtc.js +1 -0
- package/web/dist/assets/task-commits-CwGs_Opa.js +1 -0
- package/web/dist/assets/task-files-5KUgFzX3.js +2 -0
- package/web/dist/assets/task-thread-B3AFO3SS.js +9 -0
- package/web/dist/assets/textarea-Dn90Eun6.js +1 -0
- package/web/dist/assets/thread-loading-_1hPuk76.js +1 -0
- package/web/dist/assets/{trash-2-DAyZ0VSi.js → trash-2-BKEc5Sp2.js} +1 -1
- package/web/dist/assets/{triangle-alert-Uy8rokUh.js → triangle-alert-BGGPY2C2.js} +1 -1
- package/web/dist/assets/{upload-CDdyPPo_.js → upload-B0AQw-mO.js} +1 -1
- package/web/dist/assets/use-desktop-CegyTp3X.js +1 -0
- package/web/dist/assets/use-submit-shortcut-Mle-9uJp.js +1 -0
- package/web/dist/assets/utils-D9GHDNza.js +64 -0
- package/web/dist/assets/workflows-3PCz57Z6.js +11 -0
- package/web/dist/assets/zoomable-image-DTq7s8CZ.js +1 -0
- package/web/dist/index.html +33 -13
- package/scripts/dev.mjs +0 -74
- package/scripts/release-snapshot.mjs +0 -130
- package/scripts/release.mjs +0 -119
- package/web/dist/assets/arrow-left-CnSdp92h.js +0 -1
- package/web/dist/assets/bundle-mjs-BegPhL5c.js +0 -1
- package/web/dist/assets/centered-state-DOsTcJZF.js +0 -43
- package/web/dist/assets/commit-list-BrxBEzRK.js +0 -1
- package/web/dist/assets/compare-variants-Bv2c9ORF.js +0 -1
- package/web/dist/assets/dist-B6XQOhgM.js +0 -1
- package/web/dist/assets/git-toolbar-pIC1ubPM.js +0 -1
- package/web/dist/assets/github-DTTSG0ht.js +0 -1
- package/web/dist/assets/highlighted-body-OFNGDK62-BuFJwunK.js +0 -1
- package/web/dist/assets/image-preview-DEhW0TFz.js +0 -1
- package/web/dist/assets/index-Cijmlz7q.js +0 -30
- package/web/dist/assets/index-DzroxqBG.css +0 -2
- package/web/dist/assets/lib-DPEZDBUN.js +0 -1
- package/web/dist/assets/lib-uHpeUPzK.js +0 -1
- package/web/dist/assets/mermaid-GHXKKRXX-CN2zQue0.js +0 -1
- package/web/dist/assets/open-mercato-toBr6SOa.svg +0 -11
- package/web/dist/assets/project-router-BMx4e8m-.js +0 -1
- package/web/dist/assets/repo-git-BusYfa3t.js +0 -1
- package/web/dist/assets/run-diff-wOJ3odA-.js +0 -3
- package/web/dist/assets/run-header-Cg5incil.js +0 -1
- package/web/dist/assets/skill-empty-hint-DTUSp6Vl.js +0 -1
- package/web/dist/assets/skills-PpP2owbM.js +0 -1
- package/web/dist/assets/tab-link-F3uQrdm2.js +0 -1
- package/web/dist/assets/task-changes-iD9kS79I.js +0 -1
- package/web/dist/assets/task-commits-COkLaUri.js +0 -1
- package/web/dist/assets/task-files-Ckttz7XH.js +0 -2
- package/web/dist/assets/task-thread-CQD91VPt.js +0 -9
- package/web/dist/assets/textarea-Bj_LCreg.js +0 -1
- package/web/dist/assets/use-desktop-DUh0STzq.js +0 -3
- package/web/dist/assets/utils-B8s5qIcK.js +0 -1
- package/web/dist/assets/workflows-BFk8C0co.js +0 -11
- package/web/dist/assets/zoomable-image-PEDOqyLQ.js +0 -1
- /package/web/{open-mercato.svg → dist/open-mercato.svg} +0 -0
package/dist/workflows/run.js
CHANGED
|
@@ -2,29 +2,37 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { spawn } from 'node:child_process';
|
|
3
3
|
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
|
-
import {
|
|
5
|
+
import { parseAskMarkerResult, stripAskMarker, } from '../core/ask.js';
|
|
6
6
|
import { onUsage, registerRunProcess, unregisterRunProcess } from '../core/process-usage.js';
|
|
7
|
+
import { parseUsageLimit } from '../core/usage-limit.js';
|
|
7
8
|
import { createRunner } from '../core/runner-factory.js';
|
|
8
9
|
import { modelConflictsWithRunner } from '../core/model-presets.js';
|
|
10
|
+
import { AGENT_MODELS_LOCKED_ERROR, agentModelsLocked } from '../core/agent-model-policy.js';
|
|
9
11
|
import { ModelIdentityError, formatModelIdentity, normalizeModelForBackend, } from '../core/model-identity.js';
|
|
10
12
|
import { HANDOFF_ONLY_INSTRUCTIONS, HANDOFF_INSTRUCTIONS, appendHandoffHeartbeat, followupsEnabled, handoffPath, seedHandoffFile, } from '../handoff.js';
|
|
11
13
|
import { todosPath } from '../todos.js';
|
|
12
14
|
import { discoverSkills } from '../skills.js';
|
|
13
15
|
import { materializeSkillDir } from '../skills-remote.js';
|
|
14
16
|
import { seedAgentConfigLocalLayer } from '../agent-config/seed.js';
|
|
17
|
+
import { readAgentModelProvider } from '../agent-config/models.js';
|
|
15
18
|
import { loadConfig, resolveWorktreeRetention } from '../config.js';
|
|
16
19
|
import { autosaveCommit, createWorktree, resolveBaseRef, worktreeDiff, worktreeShortstat } from '../git-worktree.js';
|
|
17
|
-
import { getRepoInfo } from '../server/git.js';
|
|
20
|
+
import { getHeadCommit, getRepoInfo } from '../server/git.js';
|
|
18
21
|
import { loadWorkflows } from './load.js';
|
|
19
22
|
import { reclaimWorktrees, rematerializeReclaimedWorktree } from '../runs/retention.js';
|
|
23
|
+
import { AgentTempDirError, agentTmpEnv, removeAgentTmpDir, sweepAgentTmpDirs, } from '../runs/agent-tmpdir.js';
|
|
20
24
|
import { extractTaskRefs, refineTaskRefs, titleRefNumber } from '../runs/task-refs.js';
|
|
21
25
|
import { parseTaskMarkers, stripTaskMarkers } from '../runs/task-markers.js';
|
|
22
26
|
import { autoNamingActive, generateRunName, liveTitleUpdatesEnabled, postValidateTitle } from '../runs/auto-name.js';
|
|
23
27
|
import { reviewGateEnabled } from '../runs/review-gate.js';
|
|
28
|
+
import { resolveProfileEnvForRoot } from '../workspace/agent-profiles.js';
|
|
24
29
|
import { WorkspaceSemaphore } from '../workspace/semaphore.js';
|
|
25
30
|
import { UiEventSink } from '../runs/ui-event-sink.js';
|
|
26
31
|
import { chainStepNote, DEFAULT_ALLOWED_TOOLS, stepKind } from './types.js';
|
|
27
32
|
const CHECK_OUTPUT_CAP = 20_000;
|
|
33
|
+
async function configuredModelProvider(backend, repoRoot) {
|
|
34
|
+
return readAgentModelProvider(backend, repoRoot).catch(() => undefined);
|
|
35
|
+
}
|
|
28
36
|
/** An interactive session that hears nothing from the user closes itself. */
|
|
29
37
|
export const IDLE_TIMEOUT_MS = 15 * 60_000;
|
|
30
38
|
/**
|
|
@@ -77,6 +85,18 @@ function emitAskRequested(sink, ask) {
|
|
|
77
85
|
sink.handle({ type: 'ask.requested', requestId, questions: ask.questions });
|
|
78
86
|
return requestId;
|
|
79
87
|
}
|
|
88
|
+
/** A persisted, non-fatal explanation for protocol-shaped text that could not
|
|
89
|
+
* become an ask card. Never include the raw payload in this diagnostic. */
|
|
90
|
+
function askMarkerRejection(result) {
|
|
91
|
+
if (result.kind === 'invalid-json') {
|
|
92
|
+
return 'structured question ignored — CEZ:ASK payload is not valid JSON';
|
|
93
|
+
}
|
|
94
|
+
if (result.kind !== 'invalid-structure')
|
|
95
|
+
return undefined;
|
|
96
|
+
const issue = result.issues[0];
|
|
97
|
+
const location = issue?.path.length ? ` at ${issue.path.join('.')}` : '';
|
|
98
|
+
return `structured question ignored — CEZ:ASK payload failed validation${location}${issue ? `: ${issue.message}` : ''}`;
|
|
99
|
+
}
|
|
80
100
|
/** Periodic "cezar autosave" commit in the task worktree (spec 006). */
|
|
81
101
|
export const AUTOSAVE_INTERVAL_MS = 90_000;
|
|
82
102
|
/** The periodic autosave timer is opt-in (#471): off, a task branch carries only the
|
|
@@ -86,9 +106,110 @@ export const AUTOSAVE_INTERVAL_MS = 90_000;
|
|
|
86
106
|
export function periodicAutosaveEnabled(env = process.env) {
|
|
87
107
|
return env.CEZ_AUTOSAVE === '1';
|
|
88
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* Explicitly opt out of the repository-root lease for runs that execute in the
|
|
111
|
+
* current checkout. This covers explicit worktree opt-out, non-Git degradation,
|
|
112
|
+
* and continuations whose worktree cannot be restored (spec 006 hardening, #438).
|
|
113
|
+
* This is intentionally unsafe: concurrent agents may overwrite each other's
|
|
114
|
+
* files or Git state. Isolated worktree runs are unaffected.
|
|
115
|
+
*/
|
|
116
|
+
export function repositoryRootLockDisabled(env = process.env) {
|
|
117
|
+
return env.CEZ_DISABLE_REPO_LOCK === '1';
|
|
118
|
+
}
|
|
119
|
+
const REPOSITORY_ROOT_LOCK_DISABLED_NOTE = 'repository-root lock disabled by CEZ_DISABLE_REPO_LOCK=1 (shared checkout is unsafe)';
|
|
89
120
|
/** Safety cap on autonomous auto-continues per run — stops a stuck agent from nudging forever. */
|
|
90
121
|
const MAX_AUTO_CONTINUES = 40;
|
|
91
122
|
const AUTONOMOUS_NUDGE = 'Continue working autonomously until the task is fully complete. Do not ask me for confirmation or clarification — make reasonable assumptions and proceed. When everything is done, end the session with your done signal.';
|
|
123
|
+
const MONITORING_WAKE_NUDGE = 'Re-check the downstream work you were monitoring. Continue toward the task goal; emit CEZ:MONITORING again only if it is still pending.';
|
|
124
|
+
/**
|
|
125
|
+
* Auto-resume after a provider usage limit (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
126
|
+
*
|
|
127
|
+
* The wait is the provider's own reset instant plus this grace: resuming AT the boundary races the
|
|
128
|
+
* provider's clock (and its rounding), and one failed resume costs the whole window over again.
|
|
129
|
+
* Thirty seconds is cheap next to five hours and long enough to be past any sane skew.
|
|
130
|
+
*/
|
|
131
|
+
export const AUTO_RESUME_GRACE_MS = 30_000;
|
|
132
|
+
/**
|
|
133
|
+
* Consecutive automatic resumes allowed without a human turn. A resume can only fire after a real
|
|
134
|
+
* reset instant, so this is not a throttle — it is the backstop for the pathological case (a
|
|
135
|
+
* provider that answers "limit reached, retry now" in a loop), and it is deliberately generous
|
|
136
|
+
* enough to sit through a couple of days of five-hour windows.
|
|
137
|
+
*/
|
|
138
|
+
export const MAX_AUTO_RESUMES = 12;
|
|
139
|
+
/**
|
|
140
|
+
* How long a missed deadline stays worth acting on. The promise is "we pick this up when the
|
|
141
|
+
* window reopens" — kept across a restart or an overnight close, which is the case the feature
|
|
142
|
+
* exists for. A day later it is no longer that promise: the user has moved on, and a task
|
|
143
|
+
* springing back to life is a surprise rather than a service. Such a deadline is retired with a
|
|
144
|
+
* note instead of fired, so the only tasks a sweep can revive are ones someone is still waiting on.
|
|
145
|
+
*/
|
|
146
|
+
export const AUTO_RESUME_MISSED_WINDOW_MS = 24 * 60 * 60_000;
|
|
147
|
+
/**
|
|
148
|
+
* How often the queue checks that it is not wedged.
|
|
149
|
+
*
|
|
150
|
+
* A hold is the only thing in the engine that can make an idle queue CORRECT, so it is also the
|
|
151
|
+
* only thing that can make a wedged one look correct. This tick is the way out: cheap (a few
|
|
152
|
+
* in-memory checks), unref'd, and it only ever acts when idling has no justification left.
|
|
153
|
+
*/
|
|
154
|
+
export const QUEUE_WATCHDOG_MS = 60_000;
|
|
155
|
+
/** Shared empty holds for the common "nothing is held" pump — avoids allocating per sweep. */
|
|
156
|
+
const NO_HOLDS = { deadline: new Set(), inFlight: new Set() };
|
|
157
|
+
/**
|
|
158
|
+
* May this run start, given what its account is holding?
|
|
159
|
+
*
|
|
160
|
+
* The two kinds of hold bind different work, and getting that wrong has produced a bug in each
|
|
161
|
+
* direction (spec 2026-08-03-auto-resume-after-usage-limit):
|
|
162
|
+
*
|
|
163
|
+
* - a `deadline` hold means the window is KNOWN shut until an instant, so it blocks everything
|
|
164
|
+
* on that account — resumes included. Exempting them let four resumes fire at once and
|
|
165
|
+
* re-limit one after another, which is the stampede wearing a different hat.
|
|
166
|
+
* - an `inFlight` hold means a resume is testing the window right now and nothing is proven, so
|
|
167
|
+
* it blocks fresh work but not other resumes. Blocking those deadlocked a live workspace.
|
|
168
|
+
*/
|
|
169
|
+
function accountHeldFor(run, holds, fallbackRunner) {
|
|
170
|
+
const key = runAccountKey(run, fallbackRunner);
|
|
171
|
+
if (holds.deadline.has(key))
|
|
172
|
+
return true;
|
|
173
|
+
return holds.inFlight.has(key) && !resumeInFlight(run);
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Which agent ACCOUNT a run's work runs on — the thing a provider usage limit actually closes
|
|
177
|
+
* (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
178
|
+
*
|
|
179
|
+
* Backend plus agent account, because those are the two axes a limit is scoped to: a Claude
|
|
180
|
+
* limit must never stall a Codex task, and a second Claude login is a second budget. A record
|
|
181
|
+
* that names no runner has not started yet and will take the configured default, which is what
|
|
182
|
+
* `fallbackRunner` carries; a run that HAS started always carries its resolved runner (execute
|
|
183
|
+
* persists it), and only started runs can be holding.
|
|
184
|
+
*/
|
|
185
|
+
export function runAccountKey(run, fallbackRunner) {
|
|
186
|
+
return `${run.runner ?? fallbackRunner}:${run.agentProfile ?? 'default'}`;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Is this run an automatic resume that has not completed a turn yet?
|
|
190
|
+
*
|
|
191
|
+
* Such a run is the work the reopened window is FOR, so the hold must never apply to it — not
|
|
192
|
+
* its own, and not another resume's. Two resumes that hold each other is a deadlock the queue
|
|
193
|
+
* cannot recover from: both sit `queued` with a counter and no deadline, each waiting for the
|
|
194
|
+
* other to prove a window neither will ever get to test. That is the shape a live run produced
|
|
195
|
+
* — two scheduled tasks fired, both went `queued`, and nothing in the workspace moved again.
|
|
196
|
+
*
|
|
197
|
+
* The hold exists to stop NEW work walking into a closed window. A resume is not new work.
|
|
198
|
+
*/
|
|
199
|
+
function resumeInFlight(run) {
|
|
200
|
+
return (run.autoResumeAttempts !== undefined && (run.status === 'queued' || run.status === 'running'));
|
|
201
|
+
}
|
|
202
|
+
const AUTO_RESUME_PROMPT = 'The provider usage limit that interrupted this task has reset. Read the handoff file (CEZ_HANDOFF_FILE) to recover context, then continue the task from where you left off.';
|
|
203
|
+
/**
|
|
204
|
+
* The wake instant as a human reads it — local, to the SECOND, with the zone named. The
|
|
205
|
+
* transcript line is what someone scanning a stalled task actually reads, and "18:41" is not
|
|
206
|
+
* enough to tell a wait that is nearly over from one that just started; the machine-readable ISO
|
|
207
|
+
* copy lives on `RunRecord.autoResumeAt`. Server-side formatting is honest here because cezar is
|
|
208
|
+
* local-first: the process and the browser reading it are the same machine.
|
|
209
|
+
*/
|
|
210
|
+
function formatWakeInstant(at) {
|
|
211
|
+
return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'long' }).format(at);
|
|
212
|
+
}
|
|
92
213
|
/**
|
|
93
214
|
* The effective "extra" system prompt for a run (spec §protocol v2, R2 2.3):
|
|
94
215
|
* the per-run override (`POST /api/runs` `systemPrompt`) REPLACES the
|
|
@@ -113,6 +234,17 @@ export function composeSystemPrompt(...parts) {
|
|
|
113
234
|
.filter((p) => Boolean(p))
|
|
114
235
|
.join('\n\n---\n\n');
|
|
115
236
|
}
|
|
237
|
+
/**
|
|
238
|
+
* The directories a spawned agent may reach outside its worktree: the run-state
|
|
239
|
+
* folder that holds its handoff file, plus its own temp directory when this run
|
|
240
|
+
* got one (#785). Handing an agent a `TMPDIR` its file tools are not allowed to
|
|
241
|
+
* write would trade one silent failure for another, so the two travel together;
|
|
242
|
+
* under `CEZ_AGENT_TMPDIR=0` there is no per-run directory and the list is
|
|
243
|
+
* exactly what it always was.
|
|
244
|
+
*/
|
|
245
|
+
export function agentDirectories(runsDir, env) {
|
|
246
|
+
return env.TMPDIR ? [runsDir, env.TMPDIR] : [runsDir];
|
|
247
|
+
}
|
|
116
248
|
/**
|
|
117
249
|
* Materialized pasted attachment: the on-disk name/serving-URL pair the
|
|
118
250
|
* transcript already used, plus the absolute path that lets the agent
|
|
@@ -171,6 +303,7 @@ const VARIANT_HINTS = {
|
|
|
171
303
|
B: 'Approach hint: prefer the minimal, surgical change.',
|
|
172
304
|
C: 'Approach hint: prefer a thorough, structural approach.',
|
|
173
305
|
};
|
|
306
|
+
const RESTART_CONTINUATION_PROMPT = 'The cezar process restarted while you were working on this task. Read the handoff file (CEZ_HANDOFF_FILE) to recover context, then continue the task from where you left off.';
|
|
174
307
|
/**
|
|
175
308
|
* The mini workflow engine: executes a `WorkflowDef` against a repo, one step
|
|
176
309
|
* at a time, persisting every event to the RunStore (which the SSE endpoints
|
|
@@ -200,7 +333,13 @@ export class RunManager {
|
|
|
200
333
|
// timeout already bounds how long a session can sit open. Invariant:
|
|
201
334
|
// `waiting ⊆ active` — always cleared together via dropActive().
|
|
202
335
|
waiting = new Set();
|
|
336
|
+
/** Durable monitoring subset. Only the configured number receives the waiting-slot exemption. */
|
|
337
|
+
monitoring = new Set();
|
|
203
338
|
pendingJobs = new Map();
|
|
339
|
+
/** Interrupted agent turns recovered after a process restart. Unlike an
|
|
340
|
+
* explicit user Continue, these are bulk scheduler work and must re-enter
|
|
341
|
+
* through `pump()` so both workspace and per-project caps are honored. */
|
|
342
|
+
pendingContinuations = new Map();
|
|
204
343
|
/** Per-run image counter behind `pasted-<n>` / `screenshot-<n>` (#472). Lives on
|
|
205
344
|
* the manager rather than the `ActiveRun` so a *queued* run — which has no
|
|
206
345
|
* `ActiveRun` at all — can persist attachments. Seeded lazily from disk. */
|
|
@@ -208,14 +347,21 @@ export class RunManager {
|
|
|
208
347
|
/** Messages that landed in the dequeue → session-open gap (#472), flushed as
|
|
209
348
|
* ordinary follow-up turns the moment the session opens. In-memory only. */
|
|
210
349
|
deferredMessages = new Map();
|
|
350
|
+
/** Armed usage-limit resumes, keyed by run id (spec
|
|
351
|
+
* 2026-08-03-auto-resume-after-usage-limit). The DEADLINE itself lives on the record
|
|
352
|
+
* (`autoResumeAt`) — this map holds only the process-local timer, so a restart rebuilds it
|
|
353
|
+
* from the record rather than losing the wait. Runs here are `failed` and therefore NOT in
|
|
354
|
+
* `active`, which is why the timer cannot live on an `ActiveRun` like the monitoring one. */
|
|
355
|
+
autoResumeTimers = new Map();
|
|
211
356
|
pumping = false;
|
|
212
357
|
/** A pump that arrived while one was in flight — replayed by `pump()`'s own
|
|
213
358
|
* loop so a slot freed mid-sweep is never a lost wakeup. */
|
|
214
359
|
pumpAgain = false;
|
|
215
360
|
/**
|
|
216
361
|
* Runs normally isolate in worktrees and may execute in parallel. When that
|
|
217
|
-
* isolation is unavailable (or explicitly disabled),
|
|
218
|
-
*
|
|
362
|
+
* isolation is unavailable (or explicitly disabled), access to `repoRoot` is
|
|
363
|
+
* serialized by default so two agents cannot edit/revert the same files
|
|
364
|
+
* (#438). `CEZ_DISABLE_REPO_LOCK=1` deliberately bypasses this safety lease.
|
|
219
365
|
*/
|
|
220
366
|
repoRootTail = Promise.resolve();
|
|
221
367
|
/** `.ai/cezar` — where the per-task handoff files and todos.json live. */
|
|
@@ -226,6 +372,13 @@ export class RunManager {
|
|
|
226
372
|
/** Unsubscribe handle for the constructor's `onUsage` subscription — released
|
|
227
373
|
* by dispose() so a torn-down manager stops receiving sampler ticks. */
|
|
228
374
|
offUsage;
|
|
375
|
+
/** The stalled-queue watchdog (see `rescueStalledQueue`). */
|
|
376
|
+
queueWatchdog;
|
|
377
|
+
/** Set by the watchdog for exactly one sweep: ignore the usage-limit hold and make progress. */
|
|
378
|
+
forceNextPump = false;
|
|
379
|
+
/** Runs the watchdog started despite the hold. The spawn-time gate (`requeueWhileHeld`) would
|
|
380
|
+
* otherwise hand them straight back and the rescue would undo itself in a millisecond. */
|
|
381
|
+
forceStarted = new Set();
|
|
229
382
|
/** The workspace-wide parallel-cap semaphore + cached resource config
|
|
230
383
|
* (spec 2026-07-20, step 2.5). Boot constructs ONE and every manager shares
|
|
231
384
|
* it; the private fallback keeps single-manager callers and tests working. */
|
|
@@ -242,10 +395,13 @@ export class RunManager {
|
|
|
242
395
|
busySlots: () => this.busySlots(),
|
|
243
396
|
pump: () => this.pump(),
|
|
244
397
|
oldestQueuedAt: () => this.oldestQueuedAt(),
|
|
398
|
+
accountHolds: () => this.accountHolds(),
|
|
245
399
|
});
|
|
246
400
|
// Memory guard (#memory-guard): the shared process-tree sampler already ticks ~every 2 s for
|
|
247
401
|
// the runs table; piggyback on it to enforce the per-task memory ceiling.
|
|
248
402
|
this.offUsage = onUsage((snapshot) => void this.enforceMemoryLimit(snapshot));
|
|
403
|
+
this.queueWatchdog = setInterval(() => void this.rescueStalledQueue(), QUEUE_WATCHDOG_MS);
|
|
404
|
+
this.queueWatchdog.unref?.();
|
|
249
405
|
}
|
|
250
406
|
/**
|
|
251
407
|
* Release everything this manager owns without touching run records
|
|
@@ -260,17 +416,23 @@ export class RunManager {
|
|
|
260
416
|
dispose() {
|
|
261
417
|
this.offUsage();
|
|
262
418
|
this.offSemaphore();
|
|
263
|
-
|
|
419
|
+
clearInterval(this.queueWatchdog);
|
|
420
|
+
for (const [runId, state] of this.active) {
|
|
264
421
|
this.clearIdleTimer(state);
|
|
422
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
265
423
|
this.clearAutosaveTimer(state);
|
|
266
424
|
state.releaseRepoRoot?.();
|
|
267
425
|
state.releaseRepoRoot = undefined;
|
|
268
426
|
}
|
|
427
|
+
for (const timer of this.autoResumeTimers.values())
|
|
428
|
+
clearTimeout(timer);
|
|
429
|
+
this.autoResumeTimers.clear();
|
|
269
430
|
this.active.clear();
|
|
270
431
|
this.waiting.clear();
|
|
271
432
|
this.starting.clear();
|
|
272
433
|
this.queue.length = 0;
|
|
273
434
|
this.pendingJobs.clear();
|
|
435
|
+
this.pendingContinuations.clear();
|
|
274
436
|
this.memoryPausing.clear();
|
|
275
437
|
this.lastNamerKey.clear();
|
|
276
438
|
}
|
|
@@ -330,21 +492,64 @@ export class RunManager {
|
|
|
330
492
|
* key would let a value inherited from *this* process through — a nested
|
|
331
493
|
* cezar (an agent running `cez serve`/`cez run`/the test suite) would then
|
|
332
494
|
* write follow-ups into the parent's inbox despite the opt-out. Empty is the
|
|
333
|
-
* established "absent" spelling — consumers guard with `if (todosFile)`.
|
|
495
|
+
* established "absent" spelling — consumers guard with `if (todosFile)`.
|
|
496
|
+
*
|
|
497
|
+
* `TMPDIR`/`TEMP`/`TMP` (#785) point at this run's own scratch directory
|
|
498
|
+
* instead of the machine-wide one every agent used to share. Created and
|
|
499
|
+
* write-probed here, on the last common path before a spawn, so an unusable
|
|
500
|
+
* temp directory throws `AgentTempDirError` at the caller rather than
|
|
501
|
+
* turning into empty command output inside a running agent. */
|
|
334
502
|
agentEnv(runId, generateFollowups = true) {
|
|
335
503
|
return {
|
|
336
504
|
CEZ_HANDOFF_FILE: handoffPath(this.dataDir, runId),
|
|
337
505
|
CEZ_TASK_ID: runId,
|
|
338
506
|
CEZ_TODOS_FILE: generateFollowups ? todosPath(this.dataDir) : '',
|
|
507
|
+
...agentTmpEnv(this.dataDir, runId),
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
/**
|
|
511
|
+
* `agentEnv` plus the agent-account variable for the profile this STEP runs under (spec
|
|
512
|
+
* 2026-07-29-agent-profiles), and the id it resolved to so the caller can record it.
|
|
513
|
+
*
|
|
514
|
+
* Resolved per step, not per run, because a workflow can mix backends: an override naming a
|
|
515
|
+
* Claude account says nothing about which Codex account a codex step should use. Resolution
|
|
516
|
+
* order, most specific first:
|
|
517
|
+
*
|
|
518
|
+
* 1. the step's ALREADY-RECORDED `profileId` — a resume or Continue must reattach to the
|
|
519
|
+
* account that created the session, whatever the project has since been switched to;
|
|
520
|
+
* 2. the run's composer override, but only for steps on the run's own runner;
|
|
521
|
+
* 3. the project's stored selection, and failing that the discovered default.
|
|
522
|
+
*
|
|
523
|
+
* Read fresh every time. `~/.cezar/config.json` is shared by every cezar process on this
|
|
524
|
+
* machine, so a cached snapshot is a staleness bug, and one small JSON read is free next to
|
|
525
|
+
* spawning a CLI. Never throws: an unreadable home degrades to the default profile, which is
|
|
526
|
+
* exactly the behaviour that predates profiles.
|
|
527
|
+
*/
|
|
528
|
+
async agentEnvForStep(runId, backend, options = {}) {
|
|
529
|
+
const run = this.store.getRun(runId);
|
|
530
|
+
const profileId = options.recordedProfileId
|
|
531
|
+
?? (backend === (run?.runner ?? 'claude') ? run?.agentProfile : undefined);
|
|
532
|
+
const resolved = await resolveProfileEnvForRoot(this.repoRoot, backend, profileId);
|
|
533
|
+
return {
|
|
534
|
+
env: { ...this.agentEnv(runId, options.generateFollowups), ...resolved.env },
|
|
535
|
+
profileId: resolved.profile.id,
|
|
339
536
|
};
|
|
340
537
|
}
|
|
341
538
|
startRun(workflow, input, group) {
|
|
539
|
+
// Sanitize at the manager boundary so CLI runs, workflows, variants, and
|
|
540
|
+
// direct callers cannot bypass the HTTP policy.
|
|
541
|
+
const effectiveInput = agentModelsLocked(this.repoRoot)
|
|
542
|
+
? { ...input, model: undefined }
|
|
543
|
+
: input;
|
|
342
544
|
const run = this.store.createRun({
|
|
343
545
|
title: makeRunTitle(input.task, workflow) + (group ? ` (${group.variant})` : ''),
|
|
344
546
|
workflow: workflow.name,
|
|
345
547
|
task: input.task,
|
|
346
|
-
model:
|
|
548
|
+
model: effectiveInput.model,
|
|
347
549
|
runner: input.runner,
|
|
550
|
+
// The composer's per-task account (spec 2026-07-29-agent-profiles). Persisted at creation
|
|
551
|
+
// so a queued run picks it up at dequeue and every later resume reads the same answer.
|
|
552
|
+
agentProfile: input.agentProfile,
|
|
348
553
|
// The global inbox is the ceiling on the per-run flag (#471). Enforced here rather than
|
|
349
554
|
// at the HTTP route because `cezar run`, the inbox's own "▶ Run" and variants all reach
|
|
350
555
|
// startRun directly — a route-level gate would leave those writing todos.json.
|
|
@@ -354,6 +559,9 @@ export class RunManager {
|
|
|
354
559
|
// auto-nudge reads `input.autonomous` (`execute`), but the record is the
|
|
355
560
|
// only source those after-the-fact consumers have.
|
|
356
561
|
autonomous: input.autonomous === true,
|
|
562
|
+
// Persist the explicit opt-out so queued-run restart recovery and the
|
|
563
|
+
// session Git routes can distinguish it from a removed isolated worktree.
|
|
564
|
+
worktree: !group && input.worktree === false ? false : undefined,
|
|
357
565
|
groupId: group?.groupId,
|
|
358
566
|
variant: group?.variant,
|
|
359
567
|
steps: workflow.steps.map((s) => ({ id: s.id, name: s.name ?? s.id, kind: stepKind(s) })),
|
|
@@ -388,7 +596,7 @@ export class RunManager {
|
|
|
388
596
|
// above shows instantly; the namer's short title replaces it when (and if)
|
|
389
597
|
// the model answers. Never awaited, never fails the run.
|
|
390
598
|
void this.autoNameRun(run.id, skillHint, input.task);
|
|
391
|
-
this.pendingJobs.set(run.id, { workflow, input });
|
|
599
|
+
this.pendingJobs.set(run.id, { workflow, input: effectiveInput });
|
|
392
600
|
this.queue.push(run.id);
|
|
393
601
|
void this.pump();
|
|
394
602
|
return run;
|
|
@@ -405,7 +613,7 @@ export class RunManager {
|
|
|
405
613
|
return VARIANT_LETTERS.slice(0, Math.min(Math.max(count, 1), VARIANT_LETTERS.length)).map((variant) => {
|
|
406
614
|
const hint = VARIANT_HINTS[variant];
|
|
407
615
|
const task = hint ? `${input.task}\n\n${hint}` : input.task;
|
|
408
|
-
return this.startRun(workflow, { ...input, task }, { groupId, variant });
|
|
616
|
+
return this.startRun(workflow, { ...input, task, worktree: undefined }, { groupId, variant });
|
|
409
617
|
});
|
|
410
618
|
}
|
|
411
619
|
/**
|
|
@@ -418,7 +626,9 @@ export class RunManager {
|
|
|
418
626
|
* `maxParallel`, including when other projects saturate the cap.
|
|
419
627
|
*/
|
|
420
628
|
busySlots() {
|
|
421
|
-
|
|
629
|
+
const ordinaryWaiting = this.waiting.size - this.monitoring.size;
|
|
630
|
+
const exemptMonitoring = Math.min(this.monitoring.size, this.semaphore.maxMonitoringSessions());
|
|
631
|
+
return this.active.size + this.starting.size - ordinaryWaiting - exemptMonitoring;
|
|
422
632
|
}
|
|
423
633
|
/** Epoch ms of this manager's oldest queued run (the semaphore's fairness
|
|
424
634
|
* key when a freed slot is broadcast), or null when nothing is queued.
|
|
@@ -454,6 +664,8 @@ export class RunManager {
|
|
|
454
664
|
* root (spec 006 degradation rule), which is always the tighter bound.
|
|
455
665
|
*/
|
|
456
666
|
async pump() {
|
|
667
|
+
this.reconcileMonitoringWakeTimers();
|
|
668
|
+
this.reconcileAutoResumes();
|
|
457
669
|
// A pump requested while one is in flight can't just be dropped: the
|
|
458
670
|
// in-flight pass may already have read capacity (it awaits `getRepoInfo`
|
|
459
671
|
// before the first check), so a slot freed in that window would be lost
|
|
@@ -478,15 +690,69 @@ export class RunManager {
|
|
|
478
690
|
const capacity = () => this.semaphore.busy() < maxParallel &&
|
|
479
691
|
this.busySlots() < projectMax &&
|
|
480
692
|
(repo !== null || this.busySlots() < 1);
|
|
693
|
+
// The usage-limit hold (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
694
|
+
//
|
|
695
|
+
// A limit closes an ACCOUNT, not a run — so starting the next queued task walks it into
|
|
696
|
+
// the same wall. Measured before this gate existed: eight tasks under `maxParallel: 2`
|
|
697
|
+
// all failed within 517 ms, each spawning a CLI (and, outside worktree-opt-out mode, a
|
|
698
|
+
// worktree and a branch) only to be marked `scheduled`. The cap was respected at every
|
|
699
|
+
// instant and was no brake at all, because a doomed run lives ~200 ms.
|
|
700
|
+
//
|
|
701
|
+
// So: while any run on an account is waiting out a limit, nothing new starts on THAT
|
|
702
|
+
// account. Other accounts (a second login, a different backend) keep running — the hold
|
|
703
|
+
// is keyed, not global. The set is derived from the durable records rather than tracked
|
|
704
|
+
// separately, which is what makes it survive a restart, expire on its own, and lift the
|
|
705
|
+
// instant a user cancels a resume.
|
|
706
|
+
// The watchdog's one-shot override — read and cleared here, so a forced sweep never
|
|
707
|
+
// leaks into the next ordinary one.
|
|
708
|
+
const forced = this.forceNextPump;
|
|
709
|
+
this.forceNextPump = false;
|
|
710
|
+
const holds = this.queue.length > 0 && !forced ? this.semaphore.accountHolds() : NO_HOLDS;
|
|
711
|
+
const anyHold = holds.deadline.size > 0 || holds.inFlight.size > 0;
|
|
712
|
+
// Only pay for the config read when something is actually held: a queued record may name
|
|
713
|
+
// no runner, and then the account it would use is the configured default.
|
|
714
|
+
const defaultRunner = anyHold ? (await loadConfig(this.repoRoot)).defaultRunner : undefined;
|
|
481
715
|
while (this.queue.length > 0 && capacity()) {
|
|
482
|
-
|
|
716
|
+
// FIFO among the runs that CAN start; a held one keeps its place in the queue rather
|
|
717
|
+
// than being dequeued and re-queued (which would churn its position and its record).
|
|
718
|
+
const next = !anyHold
|
|
719
|
+
? 0
|
|
720
|
+
: this.queue.findIndex((id) => {
|
|
721
|
+
const queued = this.store.getRun(id);
|
|
722
|
+
return !queued || !accountHeldFor(queued, holds, defaultRunner ?? 'claude');
|
|
723
|
+
});
|
|
724
|
+
if (next === -1)
|
|
725
|
+
break; // everything queued is waiting on a held account
|
|
726
|
+
const runId = this.queue.splice(next, 1)[0];
|
|
483
727
|
if (!runId)
|
|
484
728
|
break;
|
|
729
|
+
// A forced sweep has to reach the spawn: the gate inside `execute` asks the same
|
|
730
|
+
// question and would send this run straight back to the queue.
|
|
731
|
+
if (forced)
|
|
732
|
+
this.forceStarted.add(runId);
|
|
485
733
|
const job = this.pendingJobs.get(runId);
|
|
734
|
+
const continuation = this.pendingContinuations.get(runId);
|
|
486
735
|
this.pendingJobs.delete(runId);
|
|
487
|
-
|
|
736
|
+
this.pendingContinuations.delete(runId);
|
|
737
|
+
if (!job && !continuation)
|
|
488
738
|
continue;
|
|
489
739
|
this.starting.add(runId);
|
|
740
|
+
if (continuation) {
|
|
741
|
+
const hydrated = this.hydrateQueuedContinuation(runId, continuation);
|
|
742
|
+
void this.runContinuation(runId, hydrated.stepId, hydrated.sessionId, hydrated.backend, hydrated.prompt, hydrated.images, hydrated.persistedImages, hydrated.persistedAttachments).catch((err) => {
|
|
743
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
744
|
+
this.store.updateRun(runId, {
|
|
745
|
+
status: 'failed',
|
|
746
|
+
error: `continue crashed: ${message}`,
|
|
747
|
+
finishedAt: new Date().toISOString(),
|
|
748
|
+
});
|
|
749
|
+
this.starting.delete(runId);
|
|
750
|
+
this.dropActive(runId);
|
|
751
|
+
});
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
if (!job)
|
|
755
|
+
continue;
|
|
490
756
|
// Rebuild the prompt from the store at the last instant (#472), so an edit
|
|
491
757
|
// or a stacked message that landed while the run waited is honored. Entered
|
|
492
758
|
// in the same synchronous tick as the `pendingJobs.delete` above, so no
|
|
@@ -514,6 +780,87 @@ export class RunManager {
|
|
|
514
780
|
this.pumping = false;
|
|
515
781
|
}
|
|
516
782
|
}
|
|
783
|
+
/**
|
|
784
|
+
* Make one `queued` RECORD executable again — the engine half a queued run needs but does not
|
|
785
|
+
* persist (`pendingJobs` / `pendingContinuations` are process-local, the record is not).
|
|
786
|
+
*
|
|
787
|
+
* Two callers, one path: boot recovery re-adopts everything the previous process was holding,
|
|
788
|
+
* and the queue watchdog re-adopts anything the running process has somehow lost. A queued
|
|
789
|
+
* record with no work item behind it is invisible to `pump()` and would sit there for good,
|
|
790
|
+
* which is the worst failure this engine has — the task is neither running nor failed, just
|
|
791
|
+
* silently never going to happen.
|
|
792
|
+
*
|
|
793
|
+
* A continuation is reconstructed first: its executable details are gone, but the pending
|
|
794
|
+
* `continue-N` step and the session before it are durable, which is enough. Otherwise the
|
|
795
|
+
* workflow is revived from the record. A run that can be neither is failed loudly rather than
|
|
796
|
+
* left in the queue as a ghost.
|
|
797
|
+
*/
|
|
798
|
+
async reviveQueuedRun(run, reason) {
|
|
799
|
+
const queuedContinuation = [...run.steps]
|
|
800
|
+
.reverse()
|
|
801
|
+
.find((step) => step.status === 'pending' && step.id.startsWith('continue-'));
|
|
802
|
+
const sessionStep = queuedContinuation
|
|
803
|
+
? [...run.steps].reverse().find((step) => step.id !== queuedContinuation.id && step.sessionId)
|
|
804
|
+
: undefined;
|
|
805
|
+
if (queuedContinuation && sessionStep?.sessionId) {
|
|
806
|
+
const backend = run.runner ?? 'claude';
|
|
807
|
+
const sessionBackend = sessionStep.backend ?? backend;
|
|
808
|
+
this.pendingContinuations.set(run.id, {
|
|
809
|
+
stepId: queuedContinuation.id,
|
|
810
|
+
sessionId: sessionBackend === backend ? sessionStep.sessionId : undefined,
|
|
811
|
+
backend,
|
|
812
|
+
prompt: RESTART_CONTINUATION_PROMPT,
|
|
813
|
+
images: [],
|
|
814
|
+
});
|
|
815
|
+
this.queue.push(run.id);
|
|
816
|
+
this.store.appendEvent(run.id, {
|
|
817
|
+
type: 'lifecycle',
|
|
818
|
+
message: `${reason} — interrupted continuation re-queued`,
|
|
819
|
+
});
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
const workflow = await this.reviveWorkflow(run);
|
|
823
|
+
if (!workflow) {
|
|
824
|
+
this.store.updateRun(run.id, {
|
|
825
|
+
status: 'failed',
|
|
826
|
+
error: 'interrupted — workflow definition not recoverable after a restart',
|
|
827
|
+
finishedAt: new Date().toISOString(),
|
|
828
|
+
});
|
|
829
|
+
this.store.appendEvent(run.id, {
|
|
830
|
+
type: 'lifecycle',
|
|
831
|
+
message: `${reason} — workflow definition not recoverable, task failed`,
|
|
832
|
+
});
|
|
833
|
+
return;
|
|
834
|
+
}
|
|
835
|
+
// Re-apply the inbox ceiling (#471). `execute()` gates again at spawn time, so the agent is
|
|
836
|
+
// safe either way — but a run queued while the inbox was on and recovered after it was
|
|
837
|
+
// switched off would otherwise keep echoing `generateFollowups: true` on a run that
|
|
838
|
+
// demonstrably produced none. Normalize the record, the way startRun does.
|
|
839
|
+
const generateFollowups = followupsEnabled() ? run.generateFollowups : false;
|
|
840
|
+
if (generateFollowups !== run.generateFollowups) {
|
|
841
|
+
this.store.updateRun(run.id, { generateFollowups });
|
|
842
|
+
}
|
|
843
|
+
this.pendingJobs.set(run.id, {
|
|
844
|
+
workflow,
|
|
845
|
+
// Folded through the same helper `pump()` uses (#472) so a restart carries the stack.
|
|
846
|
+
// Idempotent: hydration always composes from `run.task` + the stack, never from an
|
|
847
|
+
// already-folded `input.task`, so re-hydrating at dequeue yields the same string.
|
|
848
|
+
input: this.hydrateQueuedInput(run.id, {
|
|
849
|
+
task: run.task,
|
|
850
|
+
model: run.model,
|
|
851
|
+
runner: run.runner,
|
|
852
|
+
generateFollowups,
|
|
853
|
+
// Re-thread autonomy (#489): the rebuilt input feeds `execute`, whose mid-run auto-nudge
|
|
854
|
+
// reads `input.autonomous`. Without this a recovered autonomous run would run
|
|
855
|
+
// non-autonomously and later wrongly park at `review`.
|
|
856
|
+
autonomous: run.autonomous,
|
|
857
|
+
// Preserve an explicit worktree opt-out across a queued restart.
|
|
858
|
+
worktree: run.worktree,
|
|
859
|
+
}),
|
|
860
|
+
});
|
|
861
|
+
this.queue.push(run.id);
|
|
862
|
+
this.store.appendEvent(run.id, { type: 'lifecycle', message: `${reason} — task re-queued` });
|
|
863
|
+
}
|
|
517
864
|
/**
|
|
518
865
|
* Startup recovery (#367) — re-adopt runs that were live when the previous
|
|
519
866
|
* cezar process exited (requires the store opened with `keepLive`):
|
|
@@ -530,51 +877,13 @@ export class RunManager {
|
|
|
530
877
|
.listRuns()
|
|
531
878
|
.filter((r) => ['queued', 'waiting', 'running'].includes(r.status))
|
|
532
879
|
.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
880
|
+
// A crash never reaches `dropActive`, so its temp directory (#785) outlived the run.
|
|
881
|
+
// Startup is the one moment we know which runs are still live, so sweep every other
|
|
882
|
+
// per-run directory here — bounded to `<dataDir>/tmp`, never a sibling.
|
|
883
|
+
sweepAgentTmpDirs(this.dataDir, live.map((r) => r.id));
|
|
533
884
|
for (const run of live) {
|
|
534
885
|
if (run.status === 'queued') {
|
|
535
|
-
|
|
536
|
-
if (workflow) {
|
|
537
|
-
// Re-apply the inbox ceiling (#471). `execute()` gates again at spawn time, so the
|
|
538
|
-
// agent is safe either way — but a run queued while the inbox was on and recovered
|
|
539
|
-
// after it was switched off would otherwise keep echoing `generateFollowups: true`
|
|
540
|
-
// on a run that demonstrably produced none. Normalize the record, the way startRun
|
|
541
|
-
// does, so the stored answer matches what actually happens.
|
|
542
|
-
const generateFollowups = followupsEnabled() ? run.generateFollowups : false;
|
|
543
|
-
if (generateFollowups !== run.generateFollowups) {
|
|
544
|
-
this.store.updateRun(run.id, { generateFollowups });
|
|
545
|
-
}
|
|
546
|
-
this.pendingJobs.set(run.id, {
|
|
547
|
-
workflow,
|
|
548
|
-
// Folded through the same helper `pump()` uses (#472) so a restart
|
|
549
|
-
// carries the stack. Idempotent: hydration always composes from
|
|
550
|
-
// `run.task` + the stack, never from an already-folded `input.task`,
|
|
551
|
-
// so re-hydrating at dequeue yields the same string, not a doubled one.
|
|
552
|
-
input: this.hydrateQueuedInput(run.id, {
|
|
553
|
-
task: run.task,
|
|
554
|
-
model: run.model,
|
|
555
|
-
runner: run.runner,
|
|
556
|
-
generateFollowups,
|
|
557
|
-
// Re-thread autonomy (#489): the rebuilt input feeds `execute`,
|
|
558
|
-
// whose mid-run auto-nudge reads `input.autonomous`. Without this a
|
|
559
|
-
// recovered queued autonomous run would run non-autonomously (no
|
|
560
|
-
// auto-nudge) and later wrongly park at `review`.
|
|
561
|
-
autonomous: run.autonomous,
|
|
562
|
-
}),
|
|
563
|
-
});
|
|
564
|
-
this.queue.push(run.id);
|
|
565
|
-
this.store.appendEvent(run.id, { type: 'lifecycle', message: 'cezar restarted — task re-queued' });
|
|
566
|
-
}
|
|
567
|
-
else {
|
|
568
|
-
this.store.updateRun(run.id, {
|
|
569
|
-
status: 'failed',
|
|
570
|
-
error: 'interrupted — workflow definition not recoverable after a restart',
|
|
571
|
-
finishedAt: new Date().toISOString(),
|
|
572
|
-
});
|
|
573
|
-
this.store.appendEvent(run.id, {
|
|
574
|
-
type: 'lifecycle',
|
|
575
|
-
message: 'cezar restarted — workflow definition not recoverable, task failed',
|
|
576
|
-
});
|
|
577
|
-
}
|
|
886
|
+
await this.reviveQueuedRun(run, 'cezar restarted');
|
|
578
887
|
continue;
|
|
579
888
|
}
|
|
580
889
|
if (run.status === 'waiting') {
|
|
@@ -605,8 +914,8 @@ export class RunManager {
|
|
|
605
914
|
currentStepId: undefined,
|
|
606
915
|
});
|
|
607
916
|
const resumed = this.continueRun(run.id, {
|
|
608
|
-
text:
|
|
609
|
-
});
|
|
917
|
+
text: RESTART_CONTINUATION_PROMPT,
|
|
918
|
+
}, true);
|
|
610
919
|
this.store.appendEvent(run.id, {
|
|
611
920
|
type: 'lifecycle',
|
|
612
921
|
message: resumed.ok
|
|
@@ -614,14 +923,21 @@ export class RunManager {
|
|
|
614
923
|
: `cezar restarted — could not resume the interrupted task (${resumed.error ?? 'unknown'})`,
|
|
615
924
|
});
|
|
616
925
|
}
|
|
926
|
+
// Re-arm usage-limit resumes (spec 2026-08-03-auto-resume-after-usage-limit): the wait is
|
|
927
|
+
// routinely longer than a cezar session, so the deadline is durable and the timer is rebuilt
|
|
928
|
+
// from it. `pump()` reconciles again on every sweep, so this is the fast path, not the only
|
|
929
|
+
// one — see `reconcileAutoResumes`.
|
|
930
|
+
this.reconcileAutoResumes();
|
|
617
931
|
void this.pump();
|
|
618
932
|
}
|
|
619
933
|
/** The persisted definition when it looks sane, else the catalog by name. */
|
|
620
934
|
async reviveWorkflow(run) {
|
|
935
|
+
// "Looks sane" is the STORE's job now: it parses `workflowDef` against the definition schema
|
|
936
|
+
// and `.catch`es a def that no longer fits to `undefined`, so anything present here already
|
|
937
|
+
// has the `steps` array the old inline `Array.isArray` check was asking for.
|
|
621
938
|
const def = run.workflowDef;
|
|
622
|
-
if (def
|
|
939
|
+
if (def)
|
|
623
940
|
return def;
|
|
624
|
-
}
|
|
625
941
|
const { workflows } = await loadWorkflows(this.repoRoot);
|
|
626
942
|
return workflows.find((w) => w.name === run.workflow) ?? null;
|
|
627
943
|
}
|
|
@@ -632,12 +948,26 @@ export class RunManager {
|
|
|
632
948
|
if (state)
|
|
633
949
|
state.releaseRepoRoot = undefined;
|
|
634
950
|
this.waiting.delete(runId);
|
|
951
|
+
this.monitoring.delete(runId);
|
|
952
|
+
if (state)
|
|
953
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
635
954
|
this.active.delete(runId);
|
|
636
955
|
this.memoryPausing.delete(runId);
|
|
637
956
|
this.lastNamerKey.delete(runId);
|
|
957
|
+
this.forceStarted.delete(runId);
|
|
638
958
|
// The run's slot is gone from busySlots() as of the deletes above — hand it
|
|
639
959
|
// to the workspace's oldest queued run, in ANY project. Every terminal path
|
|
640
960
|
// funnels through here, so this one call covers them all.
|
|
961
|
+
// Same reasoning as retention below — every terminal path funnels through here, so the
|
|
962
|
+
// usage-limit question ("did this run stop because the account is out of window, and when
|
|
963
|
+
// does that window reopen?") is asked once, in one place, off the record the failing path
|
|
964
|
+
// has already written. Nothing to do for any other outcome.
|
|
965
|
+
//
|
|
966
|
+
// BEFORE releasing the slot, and that order is the whole point: `releaseSlot` pumps every
|
|
967
|
+
// manager, and a pump reads the hold off the records. Publishing the schedule afterwards
|
|
968
|
+
// left a window — measured as exactly one extra task — where the queue saw a free slot and
|
|
969
|
+
// an account that looked healthy, and started work that was already doomed.
|
|
970
|
+
this.scheduleAutoResumeIfLimited(runId);
|
|
641
971
|
this.releaseSlot();
|
|
642
972
|
// A run leaving the active registry is a terminal transition (done/review/
|
|
643
973
|
// failed/cancelled) — the one moment the finished-worktree count can grow.
|
|
@@ -645,6 +975,338 @@ export class RunManager {
|
|
|
645
975
|
// terminal path. Fire-and-forget: retention must never delay or throw into
|
|
646
976
|
// the lifecycle.
|
|
647
977
|
void this.enforceRetention();
|
|
978
|
+
// The run's temp directory (#785) goes on the same terminal transition, and
|
|
979
|
+
// unconditionally — it is scratch, not an artifact, so unlike a worktree
|
|
980
|
+
// there is no keep-count to respect and nothing left to recover from it. A
|
|
981
|
+
// Continue (or an auto-resume) re-creates it through `agentEnv`.
|
|
982
|
+
removeAgentTmpDir(this.dataDir, runId);
|
|
983
|
+
}
|
|
984
|
+
// ---- usage-limit auto-resume (spec 2026-08-03-auto-resume-after-usage-limit) --------------
|
|
985
|
+
/**
|
|
986
|
+
* A run just failed: if the provider said "usage limit, back at T", promise to resume it at
|
|
987
|
+
* `T + AUTO_RESUME_GRACE_MS` instead of leaving the task dead until someone notices.
|
|
988
|
+
*
|
|
989
|
+
* Every refusal below is silent-but-honest — the run stays `failed` with its Continue button,
|
|
990
|
+
* which is exactly the pre-feature behavior — except the safety cap, which says so on the
|
|
991
|
+
* transcript, because a run that stops resuming itself needs to explain why.
|
|
992
|
+
*/
|
|
993
|
+
scheduleAutoResumeIfLimited(runId) {
|
|
994
|
+
if (this.autoResumeTimers.has(runId))
|
|
995
|
+
return; // already promised
|
|
996
|
+
const run = this.store.getRun(runId);
|
|
997
|
+
if (!run || run.status !== 'failed')
|
|
998
|
+
return;
|
|
999
|
+
// Archiving IS resigning from a task. Reviving one because a window happened to reopen would
|
|
1000
|
+
// be the feature working against the clearest signal the user can give it.
|
|
1001
|
+
if (run.archived)
|
|
1002
|
+
return;
|
|
1003
|
+
const limit = parseUsageLimit(run.error);
|
|
1004
|
+
if (!limit)
|
|
1005
|
+
return;
|
|
1006
|
+
if (!this.semaphore.autoResumeOnUsageLimit())
|
|
1007
|
+
return;
|
|
1008
|
+
// No session to resume = nothing this feature can do; `continueRun` would refuse anyway.
|
|
1009
|
+
if (!run.steps.some((step) => step.sessionId))
|
|
1010
|
+
return;
|
|
1011
|
+
const attempts = run.autoResumeAttempts ?? 0;
|
|
1012
|
+
if (attempts >= MAX_AUTO_RESUMES) {
|
|
1013
|
+
this.store.appendEvent(runId, {
|
|
1014
|
+
type: 'note',
|
|
1015
|
+
message: `automatic resume cap reached (${MAX_AUTO_RESUMES}) — continue this task manually`,
|
|
1016
|
+
});
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
const wakeAt = new Date(limit.resetAt.getTime() + AUTO_RESUME_GRACE_MS);
|
|
1020
|
+
this.armAutoResume(runId, wakeAt.getTime());
|
|
1021
|
+
this.store.appendEvent(runId, {
|
|
1022
|
+
type: 'lifecycle',
|
|
1023
|
+
message: `usage limit reached — resuming automatically at ${formatWakeInstant(wakeAt)}`,
|
|
1024
|
+
});
|
|
1025
|
+
}
|
|
1026
|
+
/** Publish the deadline on the record (the cockpit's only source) and arm the timer for it. */
|
|
1027
|
+
armAutoResume(runId, deadline) {
|
|
1028
|
+
this.store.updateRun(runId, { autoResumeAt: new Date(deadline).toISOString() });
|
|
1029
|
+
const timer = setTimeout(() => this.fireAutoResume(runId), Math.max(0, deadline - Date.now()));
|
|
1030
|
+
timer.unref?.();
|
|
1031
|
+
this.autoResumeTimers.set(runId, timer);
|
|
1032
|
+
}
|
|
1033
|
+
/**
|
|
1034
|
+
* The window has reopened. Re-check the record synchronously — hours may have passed, and the
|
|
1035
|
+
* user may have continued, deleted or cancelled the run in them — then hand the resume to the
|
|
1036
|
+
* ordinary queued-continuation path so it obeys both concurrency caps like any other work.
|
|
1037
|
+
*/
|
|
1038
|
+
fireAutoResume(runId) {
|
|
1039
|
+
this.autoResumeTimers.delete(runId);
|
|
1040
|
+
const run = this.store.getRun(runId);
|
|
1041
|
+
if (!run || run.status !== 'failed' || !run.autoResumeAt)
|
|
1042
|
+
return;
|
|
1043
|
+
// Belt and braces against the one gap `reconcileAutoResumes` cannot close: the setting going
|
|
1044
|
+
// off in the window between the last pump and this tick.
|
|
1045
|
+
if (!this.semaphore.autoResumeOnUsageLimit()) {
|
|
1046
|
+
this.clearAutoResume(runId);
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
1049
|
+
const attempts = (run.autoResumeAttempts ?? 0) + 1;
|
|
1050
|
+
// `continueRun` retires the pending resume (timer + record fields) on the way in — this is a
|
|
1051
|
+
// resume, not a user turn, so the counter is put back straight after.
|
|
1052
|
+
const resumed = this.continueRun(runId, { text: AUTO_RESUME_PROMPT }, true);
|
|
1053
|
+
if (!resumed.ok) {
|
|
1054
|
+
// Refusals happen before `continueRun` retires anything, so the deadline is still on the
|
|
1055
|
+
// record — and a deadline in the past is a promise the cockpit keeps displaying and the
|
|
1056
|
+
// engine will never keep. Retire it here instead, and say why.
|
|
1057
|
+
this.clearAutoResume(runId);
|
|
1058
|
+
this.store.appendEvent(runId, {
|
|
1059
|
+
type: 'note',
|
|
1060
|
+
message: `automatic resume could not start — ${resumed.error ?? 'unknown'}`,
|
|
1061
|
+
});
|
|
1062
|
+
return;
|
|
1063
|
+
}
|
|
1064
|
+
this.store.updateRun(runId, { autoResumeAttempts: attempts });
|
|
1065
|
+
this.store.appendEvent(runId, {
|
|
1066
|
+
type: 'lifecycle',
|
|
1067
|
+
message: `usage limit reset — resuming automatically (${attempts}/${MAX_AUTO_RESUMES})`,
|
|
1068
|
+
});
|
|
1069
|
+
// A deferred continuation only ENQUEUES itself; the queue moves when something pumps it, and
|
|
1070
|
+
// `recover()` — the other deferring caller — pumps once after its whole bulk sweep. A timer
|
|
1071
|
+
// firing on its own has no such follow-up, so without this the resumed run sits at `queued`
|
|
1072
|
+
// until some unrelated run happens to finish. This is the pump for it.
|
|
1073
|
+
void this.pump();
|
|
1074
|
+
}
|
|
1075
|
+
/**
|
|
1076
|
+
* Make the armed timers agree with the records and the current setting. Runs on every `pump()`
|
|
1077
|
+
* — which is where a settings change lands (a config PUT refreshes the shared semaphore, which
|
|
1078
|
+
* pumps every manager) — and once from `recover()`.
|
|
1079
|
+
*
|
|
1080
|
+
* It is a RECONCILE rather than a one-shot restore because the deadline is durable state and
|
|
1081
|
+
* the timer is not: a restart, a rebuilt project context, a manager disposed mid-wait, or a
|
|
1082
|
+
* refusal all leave a record promising a resume that no timer is holding. Rebuilding from the
|
|
1083
|
+
* record covers every one of those at once — the alternative is a hint counting down to a time
|
|
1084
|
+
* that has already passed, which is exactly the failure this method exists to make impossible.
|
|
1085
|
+
*
|
|
1086
|
+
* Cheap: an in-memory scan, and arming is skipped for every run already held.
|
|
1087
|
+
*/
|
|
1088
|
+
reconcileAutoResumes() {
|
|
1089
|
+
if (!this.semaphore.autoResumeOnUsageLimit()) {
|
|
1090
|
+
// Sweep the RECORDS, not the timer map. A record promising a resume that no timer is
|
|
1091
|
+
// holding is the exact population this method exists for, and it is also the one the
|
|
1092
|
+
// setting can be switched off in front of: cezar restarted while it was off, the config
|
|
1093
|
+
// was hand-edited, or the project context was disposed mid-wait. Retiring only the armed
|
|
1094
|
+
// timers leaves such a record with a live `autoResumeAt`, which `accountHolds()` reads as
|
|
1095
|
+
// a deadline hold — so nothing new starts on that account, `rescueStalledQueue` treats the
|
|
1096
|
+
// phantom appointment as a legitimate reason to sit still, and the cockpit shows a
|
|
1097
|
+
// `scheduled` row for a resume that will never come. `clearAutoResume` covers the armed
|
|
1098
|
+
// ones too, so this one loop is the whole cancellation.
|
|
1099
|
+
const pending = new Set([
|
|
1100
|
+
...this.autoResumeTimers.keys(),
|
|
1101
|
+
...this.store.listRuns().filter((run) => run.autoResumeAt !== undefined).map((run) => run.id),
|
|
1102
|
+
]);
|
|
1103
|
+
for (const runId of pending) {
|
|
1104
|
+
this.clearAutoResume(runId);
|
|
1105
|
+
this.store.appendEvent(runId, {
|
|
1106
|
+
type: 'note',
|
|
1107
|
+
message: 'automatic resume cancelled — auto-resume is switched off',
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
1112
|
+
for (const run of this.store.listRuns()) {
|
|
1113
|
+
if (run.status !== 'failed' || !run.autoResumeAt)
|
|
1114
|
+
continue;
|
|
1115
|
+
if (this.autoResumeTimers.has(run.id))
|
|
1116
|
+
continue;
|
|
1117
|
+
const deadline = Date.parse(run.autoResumeAt);
|
|
1118
|
+
// A deadline that is unreadable, belongs to a run that has spent its cap, or belongs to a
|
|
1119
|
+
// task the user has archived is retired rather than re-armed: it can only mislead. One
|
|
1120
|
+
// that has just passed arms at zero — the window is open, which is the point.
|
|
1121
|
+
if (run.archived
|
|
1122
|
+
|| !Number.isFinite(deadline)
|
|
1123
|
+
|| (run.autoResumeAttempts ?? 0) >= MAX_AUTO_RESUMES) {
|
|
1124
|
+
this.store.updateRun(run.id, { autoResumeAt: undefined });
|
|
1125
|
+
continue;
|
|
1126
|
+
}
|
|
1127
|
+
// …and one missed by more than a day is retired loudly: reviving a task from another era
|
|
1128
|
+
// is a surprise, not a service, and this is what keeps a sweep from resurrecting every
|
|
1129
|
+
// limit-stopped task a user has long since walked away from.
|
|
1130
|
+
if (Date.now() - deadline > AUTO_RESUME_MISSED_WINDOW_MS) {
|
|
1131
|
+
this.store.updateRun(run.id, { autoResumeAt: undefined });
|
|
1132
|
+
this.store.appendEvent(run.id, {
|
|
1133
|
+
type: 'note',
|
|
1134
|
+
message: 'automatic resume expired — its window reopened over a day ago; continue this task manually',
|
|
1135
|
+
});
|
|
1136
|
+
continue;
|
|
1137
|
+
}
|
|
1138
|
+
this.armAutoResume(run.id, deadline);
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
/**
|
|
1142
|
+
* Hand a run that has not spawned anything back to the queue, when the account it would run on
|
|
1143
|
+
* went into a usage-limit hold (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
1144
|
+
*
|
|
1145
|
+
* The dequeue-time gate in `pump()` cannot be the only one: a run can sit between dequeue and
|
|
1146
|
+
* spawn for a long time — an in-place run waiting for the exclusive repo-root lease is the
|
|
1147
|
+
* measured case — and the account can close in that gap. This is the last honest moment to
|
|
1148
|
+
* refuse, because everything after it costs a real agent turn.
|
|
1149
|
+
*
|
|
1150
|
+
* "Untouched" is the contract: the run has created no session and no worktree, so it goes back
|
|
1151
|
+
* as plain `queued` with its `startedAt` cleared, and `pump()` will pick it up when the window
|
|
1152
|
+
* reopens. Returns true when the caller must abandon the run.
|
|
1153
|
+
*/
|
|
1154
|
+
requeueWhileHeld(runId, workflow, input, runner, state) {
|
|
1155
|
+
const run = this.store.getRun(runId);
|
|
1156
|
+
if (!run || run.status === 'cancelled' || state?.cancelled)
|
|
1157
|
+
return false;
|
|
1158
|
+
// The watchdog sent this one through. Checked, never consumed: the spawn path asks this
|
|
1159
|
+
// question TWICE — here at the top of `execute`, and again after the exclusive repo-root
|
|
1160
|
+
// lease is granted — so a one-shot flag would clear at the first gate and let the second one
|
|
1161
|
+
// hand an in-place run straight back, re-wedging the queue the rescue had just freed.
|
|
1162
|
+
// `dropActive` retires the entry on every terminal path, so the set still cleans itself up.
|
|
1163
|
+
if (this.forceStarted.has(runId))
|
|
1164
|
+
return false;
|
|
1165
|
+
if (!accountHeldFor({ ...run, runner }, this.semaphore.accountHolds(), runner))
|
|
1166
|
+
return false;
|
|
1167
|
+
state?.releaseRepoRoot?.();
|
|
1168
|
+
if (state)
|
|
1169
|
+
state.releaseRepoRoot = undefined;
|
|
1170
|
+
this.pendingJobs.set(runId, { workflow, input });
|
|
1171
|
+
this.queue.push(runId);
|
|
1172
|
+
this.store.updateRun(runId, { status: 'queued', startedAt: undefined, currentStepId: undefined });
|
|
1173
|
+
this.store.appendEvent(runId, {
|
|
1174
|
+
type: 'note',
|
|
1175
|
+
message: 'held in the queue — this agent account is waiting out a usage limit',
|
|
1176
|
+
});
|
|
1177
|
+
this.dropActive(runId);
|
|
1178
|
+
return true;
|
|
1179
|
+
}
|
|
1180
|
+
/**
|
|
1181
|
+
* The failsafe: a queue must never be able to wedge.
|
|
1182
|
+
*
|
|
1183
|
+
* Everything else in this file makes an idle queue CORRECT under some condition — a slot cap, a
|
|
1184
|
+
* repo-root lease, and now a usage-limit hold. That is also what makes a wedged queue look
|
|
1185
|
+
* correct, and the hold has already produced one in the field: two resumes fired together, each
|
|
1186
|
+
* holding the account the other was waiting on, and the whole workspace stopped with every task
|
|
1187
|
+
* `queued`. That specific bug is fixed and tested, but "the queue stopped and nothing will ever
|
|
1188
|
+
* restart it" is too expensive a failure mode to leave resting on any single fix being right.
|
|
1189
|
+
*
|
|
1190
|
+
* The test is deliberately about JUSTIFICATION rather than about any particular bug: idling is
|
|
1191
|
+
* legitimate while work is running (here or in another project), or while a real appointment is
|
|
1192
|
+
* still ahead — a scheduled resume that will fire and pump on its own. Anything else is a
|
|
1193
|
+
* queue with work in it, nothing running anywhere, and no event coming to wake it. That gets one
|
|
1194
|
+
* forced sweep, which starts work under the ordinary caps and lets the account's real state
|
|
1195
|
+
* re-assert itself: if the window truly is shut, that task meets the limit and re-establishes an
|
|
1196
|
+
* honest hold, with a real deadline behind it this time.
|
|
1197
|
+
*
|
|
1198
|
+
* Public so a test can drive the wedge directly instead of waiting out the interval.
|
|
1199
|
+
*/
|
|
1200
|
+
async rescueStalledQueue(now = Date.now()) {
|
|
1201
|
+
// First, the worst shape: a record that says `queued` while the engine holds no job, no
|
|
1202
|
+
// continuation and no queue entry for it. `pump()` cannot see such a run — it iterates the
|
|
1203
|
+
// queue, and this one is not in it — so nothing will ever start it. Re-adopt it through the
|
|
1204
|
+
// same path boot recovery uses.
|
|
1205
|
+
for (const run of this.store.listRuns()) {
|
|
1206
|
+
if (run.status !== 'queued')
|
|
1207
|
+
continue;
|
|
1208
|
+
if (this.active.has(run.id) || this.starting.has(run.id))
|
|
1209
|
+
continue;
|
|
1210
|
+
if (this.pendingJobs.has(run.id) || this.pendingContinuations.has(run.id))
|
|
1211
|
+
continue;
|
|
1212
|
+
if (this.queue.includes(run.id))
|
|
1213
|
+
continue;
|
|
1214
|
+
console.warn(`[cez] queue watchdog: re-adopting queued run ${run.id} the engine had lost`);
|
|
1215
|
+
await this.reviveQueuedRun(run, 'queue watchdog');
|
|
1216
|
+
}
|
|
1217
|
+
if (this.queue.length === 0)
|
|
1218
|
+
return;
|
|
1219
|
+
if (this.busySlots() > 0 || this.starting.size > 0)
|
|
1220
|
+
return;
|
|
1221
|
+
if (this.semaphore.busy() > 0)
|
|
1222
|
+
return;
|
|
1223
|
+
// A future deadline is a real reason to sit still: that timer will fire and pump.
|
|
1224
|
+
for (const run of this.store.listRuns()) {
|
|
1225
|
+
if (run.status !== 'failed' || !run.autoResumeAt)
|
|
1226
|
+
continue;
|
|
1227
|
+
const deadline = Date.parse(run.autoResumeAt);
|
|
1228
|
+
if (Number.isFinite(deadline) && deadline > now)
|
|
1229
|
+
return;
|
|
1230
|
+
}
|
|
1231
|
+
if (this.semaphore.accountHolds().inFlight.size === 0) {
|
|
1232
|
+
// Not the hold, then — some other wakeup went missing. An ordinary pump is the whole fix,
|
|
1233
|
+
// and it is idempotent, so this stays quiet.
|
|
1234
|
+
void this.pump();
|
|
1235
|
+
return;
|
|
1236
|
+
}
|
|
1237
|
+
console.warn('[cez] queue watchdog: work is queued, nothing is running, and the usage-limit hold has no'
|
|
1238
|
+
+ ' deadline behind it — starting the next task anyway');
|
|
1239
|
+
this.forceNextPump = true;
|
|
1240
|
+
void this.pump();
|
|
1241
|
+
}
|
|
1242
|
+
/**
|
|
1243
|
+
* The accounts this project is currently holding: one key per run parked on a usage-limit
|
|
1244
|
+
* resume that has not come due yet (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
1245
|
+
*
|
|
1246
|
+
* Published to the shared semaphore so the hold spans PROJECTS — one Claude account can be
|
|
1247
|
+
* driving tasks in three repos, and a limit closes it for all of them. Derived from the
|
|
1248
|
+
* records on every ask rather than tracked as state: a deadline that passes, a resume that
|
|
1249
|
+
* fires, a cancel, an archive and a delete all lift the hold with no bookkeeping.
|
|
1250
|
+
*
|
|
1251
|
+
* Deliberately excludes a deadline that has already passed — that run is about to resume, and
|
|
1252
|
+
* holding the queue for it would only stall the very work the window reopened for.
|
|
1253
|
+
*/
|
|
1254
|
+
accountHolds(now = Date.now()) {
|
|
1255
|
+
const deadline = new Set();
|
|
1256
|
+
const inFlight = new Set();
|
|
1257
|
+
for (const run of this.store.listRuns()) {
|
|
1258
|
+
// A holding run always carries the runner it actually ran on, so the fallback is unused
|
|
1259
|
+
// here — it is spelled out rather than `!` so a future record shape degrades, not throws.
|
|
1260
|
+
const key = () => runAccountKey(run, run.runner ?? 'claude');
|
|
1261
|
+
if (run.status === 'failed' && run.autoResumeAt) {
|
|
1262
|
+
const at = Date.parse(run.autoResumeAt);
|
|
1263
|
+
if (Number.isFinite(at) && at > now)
|
|
1264
|
+
deadline.add(key());
|
|
1265
|
+
}
|
|
1266
|
+
else if (resumeInFlight(run)) {
|
|
1267
|
+
inFlight.add(key());
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
return { deadline, inFlight };
|
|
1271
|
+
}
|
|
1272
|
+
/**
|
|
1273
|
+
* The PER-TASK off switch (`DELETE /api/v1/runs/:id/auto-resume`, and the archive route):
|
|
1274
|
+
* stop resuming THIS task, without touching the workspace setting or any other task.
|
|
1275
|
+
*
|
|
1276
|
+
* Idempotent — a run with nothing pending answers the same way, because "this task will not
|
|
1277
|
+
* resume itself" is equally true either way. Returns false only when the run does not exist,
|
|
1278
|
+
* which is the route's 404.
|
|
1279
|
+
*/
|
|
1280
|
+
cancelAutoResume(runId) {
|
|
1281
|
+
const run = this.store.getRun(runId);
|
|
1282
|
+
if (!run)
|
|
1283
|
+
return false;
|
|
1284
|
+
const pending = run.autoResumeAt !== undefined || this.autoResumeTimers.has(runId);
|
|
1285
|
+
this.clearAutoResume(runId);
|
|
1286
|
+
if (pending) {
|
|
1287
|
+
this.store.appendEvent(runId, {
|
|
1288
|
+
type: 'note',
|
|
1289
|
+
message: 'automatic resume cancelled for this task',
|
|
1290
|
+
});
|
|
1291
|
+
// This run may have been the last thing holding its account's queue — nothing else will
|
|
1292
|
+
// notice, since the hold is derived and its release is not an event.
|
|
1293
|
+
void this.pump();
|
|
1294
|
+
}
|
|
1295
|
+
return true;
|
|
1296
|
+
}
|
|
1297
|
+
/** Retire a pending resume — timer, deadline and counter. The counter goes too because every
|
|
1298
|
+
* caller is a fresh epoch: a human Continue, or a resume that re-stamps its own count. */
|
|
1299
|
+
clearAutoResume(runId) {
|
|
1300
|
+
const timer = this.autoResumeTimers.get(runId);
|
|
1301
|
+
if (timer)
|
|
1302
|
+
clearTimeout(timer);
|
|
1303
|
+
this.autoResumeTimers.delete(runId);
|
|
1304
|
+
const run = this.store.getRun(runId);
|
|
1305
|
+
if (!run)
|
|
1306
|
+
return;
|
|
1307
|
+
if (run.autoResumeAt !== undefined || run.autoResumeAttempts !== undefined) {
|
|
1308
|
+
this.store.updateRun(runId, { autoResumeAt: undefined, autoResumeAttempts: undefined });
|
|
1309
|
+
}
|
|
648
1310
|
}
|
|
649
1311
|
/** Reclaim finished worktrees beyond the keep-limit (#483) — directory only,
|
|
650
1312
|
* `cez/<id8>` branch kept. Best-effort; a failure never affects run
|
|
@@ -723,6 +1385,7 @@ export class RunManager {
|
|
|
723
1385
|
if (queuedAt >= 0) {
|
|
724
1386
|
this.queue.splice(queuedAt, 1);
|
|
725
1387
|
this.pendingJobs.delete(runId);
|
|
1388
|
+
this.pendingContinuations.delete(runId);
|
|
726
1389
|
this.store.updateRun(runId, { status: 'cancelled', finishedAt: new Date().toISOString() });
|
|
727
1390
|
this.store.appendEvent(runId, { type: 'lifecycle', message: 'cancelled while queued' });
|
|
728
1391
|
return true;
|
|
@@ -763,35 +1426,13 @@ export class RunManager {
|
|
|
763
1426
|
.map((part) => part.trim())
|
|
764
1427
|
.filter((part) => part.length > 0)
|
|
765
1428
|
.join('\n\n');
|
|
766
|
-
const readImages = (urls, kind) => {
|
|
767
|
-
const images = [];
|
|
768
|
-
for (const url of urls) {
|
|
769
|
-
const name = url.split('/').pop();
|
|
770
|
-
if (!name || name.includes('..') || name.includes('/') || name.includes('\\'))
|
|
771
|
-
continue;
|
|
772
|
-
try {
|
|
773
|
-
const data = readFileSync(join(this.dataDir, 'runs', `${runId}-images`, name));
|
|
774
|
-
images.push({
|
|
775
|
-
type: 'image',
|
|
776
|
-
source: { type: 'base64', media_type: mediaTypeFor(name), data: data.toString('base64') },
|
|
777
|
-
});
|
|
778
|
-
}
|
|
779
|
-
catch {
|
|
780
|
-
// Degrade, never fail the boot (AGENTS.md): the user deleted `.ai/cezar/`
|
|
781
|
-
// or the file is unreadable — start with the text and say which image went.
|
|
782
|
-
this.store.appendEvent(runId, {
|
|
783
|
-
type: 'note',
|
|
784
|
-
message: `${kind} attachment ${name} could not be read — starting without it`,
|
|
785
|
-
});
|
|
786
|
-
}
|
|
787
|
-
}
|
|
788
|
-
return images;
|
|
789
|
-
};
|
|
790
1429
|
// Keep the original in-memory blocks for a live process (including the
|
|
791
1430
|
// best-effort case where persistence failed). Recovery has no such copy,
|
|
792
1431
|
// so rebuild it from the durable task-image URLs.
|
|
793
|
-
const images = input.images?.length
|
|
794
|
-
|
|
1432
|
+
const images = input.images?.length
|
|
1433
|
+
? input.images
|
|
1434
|
+
: this.readPersistedImages(runId, run.taskImages ?? [], 'task').blocks;
|
|
1435
|
+
const stackedImages = this.readPersistedImages(runId, stack.flatMap((m) => m.images ?? []), 'queued').blocks;
|
|
795
1436
|
return {
|
|
796
1437
|
...input,
|
|
797
1438
|
task,
|
|
@@ -799,15 +1440,67 @@ export class RunManager {
|
|
|
799
1440
|
...(stackedImages.length ? { stackedImages } : { stackedImages: undefined }),
|
|
800
1441
|
};
|
|
801
1442
|
}
|
|
1443
|
+
/** Apply edits and messages made while a restart continuation waits for
|
|
1444
|
+
* capacity. The durable record remains the source of truth, just as it is for
|
|
1445
|
+
* an ordinary queued workflow (#472), so a second restart reconstructs and
|
|
1446
|
+
* hydrates the same amendments instead of dropping them. */
|
|
1447
|
+
hydrateQueuedContinuation(runId, continuation) {
|
|
1448
|
+
const run = this.store.getRun(runId);
|
|
1449
|
+
if (!run) {
|
|
1450
|
+
return { ...continuation, persistedImages: [], persistedAttachments: [] };
|
|
1451
|
+
}
|
|
1452
|
+
const stack = run.queuedMessages ?? [];
|
|
1453
|
+
const amendedTask = [run.task, ...stack.map((message) => message.text)]
|
|
1454
|
+
.map((part) => part.trim())
|
|
1455
|
+
.filter((part) => part.length > 0)
|
|
1456
|
+
.join('\n\n');
|
|
1457
|
+
const prompt = amendedTask
|
|
1458
|
+
? `${continuation.prompt}\n\nCurrent task and queued updates:\n\n${amendedTask}`
|
|
1459
|
+
: continuation.prompt;
|
|
1460
|
+
const persisted = this.readPersistedImages(runId, stack.flatMap((message) => message.images ?? []), 'queued');
|
|
1461
|
+
return {
|
|
1462
|
+
...continuation,
|
|
1463
|
+
prompt,
|
|
1464
|
+
persistedImages: persisted.blocks,
|
|
1465
|
+
persistedAttachments: persisted.attachments,
|
|
1466
|
+
};
|
|
1467
|
+
}
|
|
1468
|
+
readPersistedImages(runId, urls, kind) {
|
|
1469
|
+
const blocks = [];
|
|
1470
|
+
const attachments = [];
|
|
1471
|
+
for (const url of urls) {
|
|
1472
|
+
const name = url.split('/').pop();
|
|
1473
|
+
if (!name || name.includes('..') || name.includes('/') || name.includes('\\'))
|
|
1474
|
+
continue;
|
|
1475
|
+
const path = join(this.dataDir, 'runs', `${runId}-images`, name);
|
|
1476
|
+
try {
|
|
1477
|
+
const data = readFileSync(path);
|
|
1478
|
+
blocks.push({
|
|
1479
|
+
type: 'image',
|
|
1480
|
+
source: { type: 'base64', media_type: mediaTypeFor(name), data: data.toString('base64') },
|
|
1481
|
+
});
|
|
1482
|
+
attachments.push({ name, url, path });
|
|
1483
|
+
}
|
|
1484
|
+
catch {
|
|
1485
|
+
// Degrade, never fail the boot (AGENTS.md): the user deleted `.ai/cezar/`
|
|
1486
|
+
// or the file is unreadable — start with the text and say which image went.
|
|
1487
|
+
this.store.appendEvent(runId, {
|
|
1488
|
+
type: 'note',
|
|
1489
|
+
message: `${kind} attachment ${name} could not be read — starting without it`,
|
|
1490
|
+
});
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
return { blocks, attachments };
|
|
1494
|
+
}
|
|
802
1495
|
/**
|
|
803
1496
|
* Still waiting for a slot? Checked against the engine's own queue rather than
|
|
804
1497
|
* the record's `status` (#472): the record is written by `execute()` a tick
|
|
805
1498
|
* after `pump()` dequeues, so a status read can see `queued` for a run that has
|
|
806
|
-
* already started.
|
|
807
|
-
* the authoritative answer for "can this prompt still be amended".
|
|
1499
|
+
* already started. The pending maps are deleted synchronously at dequeue, so
|
|
1500
|
+
* they are the authoritative answer for "can this prompt still be amended".
|
|
808
1501
|
*/
|
|
809
1502
|
isQueued(runId) {
|
|
810
|
-
return this.pendingJobs.has(runId);
|
|
1503
|
+
return this.pendingJobs.has(runId) || this.pendingContinuations.has(runId);
|
|
811
1504
|
}
|
|
812
1505
|
/** Split `ContentBlock[]` into the persisted shape a stacked message holds. */
|
|
813
1506
|
toQueuedMessage(runId, content) {
|
|
@@ -987,6 +1680,18 @@ export class RunManager {
|
|
|
987
1680
|
* then offers "Continue" instead.
|
|
988
1681
|
*/
|
|
989
1682
|
sendMessage(runId, content) {
|
|
1683
|
+
const delivered = this.deliverMessage(runId, content, true);
|
|
1684
|
+
if (delivered) {
|
|
1685
|
+
const state = this.active.get(runId);
|
|
1686
|
+
if (state)
|
|
1687
|
+
state.monitoringWakeups = 0;
|
|
1688
|
+
this.store.updateRun(runId, { monitoringWakeCapReached: undefined });
|
|
1689
|
+
}
|
|
1690
|
+
return delivered;
|
|
1691
|
+
}
|
|
1692
|
+
/** Shared live-session delivery. Synthetic scheduler prompts reuse lifecycle
|
|
1693
|
+
* bookkeeping without masquerading as user-authored transcript messages. */
|
|
1694
|
+
deliverMessage(runId, content, userAuthored) {
|
|
990
1695
|
const state = this.active.get(runId);
|
|
991
1696
|
if (!state?.session?.open || state.cancelled)
|
|
992
1697
|
return false;
|
|
@@ -997,27 +1702,32 @@ export class RunManager {
|
|
|
997
1702
|
// Persist the attached images so the thread can render them (not just count them) — the same
|
|
998
1703
|
// on-disk store + `/images/` route the agent's own screenshots use. `pasted` prefix marks
|
|
999
1704
|
// these as user attachments (vs. agent tool screenshots) on disk (#357).
|
|
1000
|
-
const persisted = content
|
|
1705
|
+
const persisted = userAuthored ? content
|
|
1001
1706
|
.filter((b) => b.type === 'image')
|
|
1002
1707
|
.map((b) => this.persistImage(runId, b.source.media_type, b.source.data, 'pasted'))
|
|
1003
|
-
.filter((saved) => saved !== null);
|
|
1708
|
+
.filter((saved) => saved !== null) : [];
|
|
1004
1709
|
const images = persisted.map((saved) => saved.url);
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1710
|
+
if (userAuthored) {
|
|
1711
|
+
this.store.appendEvent(runId, {
|
|
1712
|
+
type: 'user-message',
|
|
1713
|
+
stepId: state.currentStepId,
|
|
1714
|
+
text,
|
|
1715
|
+
imageCount: content.filter((b) => b.type === 'image').length,
|
|
1716
|
+
images,
|
|
1717
|
+
});
|
|
1718
|
+
}
|
|
1012
1719
|
// Tell the agent where the pasted files live on disk (#357): the base64 blocks below still
|
|
1013
1720
|
// ride along so the model can *view* them, but a real path is what lets it *operate* on them
|
|
1014
1721
|
// (save, `cp`, attach to a GitHub issue/PR) — and it's the only usable reference on backends
|
|
1015
1722
|
// (codex, opencode) that drop image blocks entirely before reaching the model.
|
|
1016
|
-
const
|
|
1723
|
+
const expanded = userAuthored ? expandRegistrySlashSkill(content, state.skills ?? []) : content;
|
|
1724
|
+
const deliverable = persisted.length ? [...expanded, pastedAttachmentsNote(persisted)] : expanded;
|
|
1017
1725
|
const delivered = state.session.sendMessage(deliverable);
|
|
1018
1726
|
if (delivered) {
|
|
1019
1727
|
this.clearIdleTimer(state);
|
|
1728
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1020
1729
|
this.waiting.delete(runId); // resumed — the run counts against slots again
|
|
1730
|
+
this.monitoring.delete(runId);
|
|
1021
1731
|
// Clear any `monitoring` activity — the agent is actively working again
|
|
1022
1732
|
// (spec 2026-07-18-subagent-monitoring-status, #490).
|
|
1023
1733
|
this.store.updateRun(runId, { status: 'running', activity: undefined });
|
|
@@ -1054,7 +1764,13 @@ export class RunManager {
|
|
|
1054
1764
|
* behaves exactly like an interactive step: `waiting` after each turn,
|
|
1055
1765
|
* messages via sendMessage, closed by finish/idle/cancel.
|
|
1056
1766
|
*/
|
|
1057
|
-
continueRun(runId, opts = {}
|
|
1767
|
+
continueRun(runId, opts = {},
|
|
1768
|
+
/** Restart recovery may discover several interrupted tasks at once. Those
|
|
1769
|
+
* continuations are queued; an explicit user Continue remains immediate. */
|
|
1770
|
+
deferForCapacity = false) {
|
|
1771
|
+
if (agentModelsLocked(this.repoRoot) && opts.model?.trim()) {
|
|
1772
|
+
return { ok: false, error: AGENT_MODELS_LOCKED_ERROR };
|
|
1773
|
+
}
|
|
1058
1774
|
if (this.active.has(runId))
|
|
1059
1775
|
return { ok: false, error: 'run is still active' };
|
|
1060
1776
|
const run = this.store.getRun(runId);
|
|
@@ -1105,10 +1821,34 @@ export class RunManager {
|
|
|
1105
1821
|
: {}),
|
|
1106
1822
|
});
|
|
1107
1823
|
}
|
|
1824
|
+
// Everything that could refuse this continuation has now passed, so a pending usage-limit
|
|
1825
|
+
// resume is superseded either way: this IS that resume (it re-stamps its own counter), or a
|
|
1826
|
+
// human got there first — and then the counter starts over, because the cap only exists to
|
|
1827
|
+
// bound UNATTENDED resumes.
|
|
1828
|
+
this.clearAutoResume(runId);
|
|
1108
1829
|
const continuations = run.steps.filter((s) => s.id.startsWith('continue-')).length;
|
|
1109
1830
|
const stepId = `continue-${continuations + 1}`;
|
|
1110
1831
|
this.store.addStep(runId, { id: stepId, name: 'Continue', kind: 'agent' });
|
|
1111
|
-
|
|
1832
|
+
const prompt = opts.text?.trim() || 'Continue.';
|
|
1833
|
+
const images = opts.images ?? [];
|
|
1834
|
+
if (deferForCapacity) {
|
|
1835
|
+
this.pendingContinuations.set(runId, {
|
|
1836
|
+
stepId,
|
|
1837
|
+
sessionId: resume ? sessionStep.sessionId : undefined,
|
|
1838
|
+
backend: targetRunner,
|
|
1839
|
+
prompt,
|
|
1840
|
+
images,
|
|
1841
|
+
});
|
|
1842
|
+
this.queue.push(runId);
|
|
1843
|
+
this.store.updateRun(runId, {
|
|
1844
|
+
status: 'queued',
|
|
1845
|
+
error: undefined,
|
|
1846
|
+
finishedAt: undefined,
|
|
1847
|
+
currentStepId: undefined,
|
|
1848
|
+
});
|
|
1849
|
+
return { ok: true };
|
|
1850
|
+
}
|
|
1851
|
+
void this.runContinuation(runId, stepId, resume ? sessionStep.sessionId : undefined, targetRunner, prompt, images).catch((err) => {
|
|
1112
1852
|
const message = err instanceof Error ? err.message : String(err);
|
|
1113
1853
|
this.store.updateRun(runId, {
|
|
1114
1854
|
status: 'failed',
|
|
@@ -1123,7 +1863,11 @@ export class RunManager {
|
|
|
1123
1863
|
/** Screenshots pasted into the follow-up composer — delivered with the
|
|
1124
1864
|
* reopened session's opening message, exactly like a live-session
|
|
1125
1865
|
* message's attachments. */
|
|
1126
|
-
images = []
|
|
1866
|
+
images = [],
|
|
1867
|
+
/** Queued-message screenshots were persisted when they were enqueued and
|
|
1868
|
+
* reconstructed at dequeue. Keep them separate from fresh `images` so
|
|
1869
|
+
* opening a recovered continuation does not persist duplicate files. */
|
|
1870
|
+
persistedImages = [], persistedAttachments = []) {
|
|
1127
1871
|
// Continuation runs in the task's worktree when it still exists (spec
|
|
1128
1872
|
// 006) — the resumed session sees exactly what the original run left.
|
|
1129
1873
|
// Retention (#483) may have reclaimed this run's worktree directory while
|
|
@@ -1141,25 +1885,39 @@ export class RunManager {
|
|
|
1141
1885
|
: this.repoRoot;
|
|
1142
1886
|
const state = { cancelled: false, interrupt: () => undefined, cwd };
|
|
1143
1887
|
this.active.set(runId, state);
|
|
1888
|
+
this.starting.delete(runId);
|
|
1144
1889
|
if (state.cwd === this.repoRoot) {
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
if (!(await this.acquireRepoRoot(runId, state))) {
|
|
1150
|
-
this.store.updateRun(runId, {
|
|
1151
|
-
status: 'cancelled',
|
|
1152
|
-
finishedAt: new Date().toISOString(),
|
|
1153
|
-
currentStepId: undefined,
|
|
1890
|
+
if (repositoryRootLockDisabled()) {
|
|
1891
|
+
this.store.appendEvent(runId, {
|
|
1892
|
+
type: 'note',
|
|
1893
|
+
message: REPOSITORY_ROOT_LOCK_DISABLED_NOTE,
|
|
1154
1894
|
});
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1895
|
+
}
|
|
1896
|
+
else {
|
|
1897
|
+
this.store.appendEvent(runId, {
|
|
1898
|
+
type: 'note',
|
|
1899
|
+
message: 'waiting for exclusive access to the repository working tree',
|
|
1900
|
+
});
|
|
1901
|
+
if (!(await this.acquireRepoRoot(runId, state))) {
|
|
1902
|
+
this.store.updateRun(runId, {
|
|
1903
|
+
status: 'cancelled',
|
|
1904
|
+
finishedAt: new Date().toISOString(),
|
|
1905
|
+
currentStepId: undefined,
|
|
1906
|
+
});
|
|
1907
|
+
this.store.appendEvent(runId, { type: 'lifecycle', message: 'run cancelled' });
|
|
1908
|
+
this.dropActive(runId);
|
|
1909
|
+
return;
|
|
1910
|
+
}
|
|
1158
1911
|
}
|
|
1159
1912
|
}
|
|
1160
1913
|
this.armAutosave(state);
|
|
1161
1914
|
if (record)
|
|
1162
1915
|
seedHandoffFile(this.dataDir, record); // idempotent — normally already there
|
|
1916
|
+
// Registry snapshot for `/skill` expansion. `execute` loads this for the workflow's own
|
|
1917
|
+
// sessions; a continuation builds its OWN ActiveRun, and without this the resumed session
|
|
1918
|
+
// expanded against an empty registry and leaked `/om-...` verbatim to the backend, which
|
|
1919
|
+
// answered "Unknown skill" (#811). Best-effort — discovery must never break Continue.
|
|
1920
|
+
state.skills = await discoverSkills(this.repoRoot).catch(() => []);
|
|
1163
1921
|
this.store.updateRun(runId, {
|
|
1164
1922
|
status: 'running',
|
|
1165
1923
|
error: undefined,
|
|
@@ -1180,19 +1938,22 @@ export class RunManager {
|
|
|
1180
1938
|
// images rather than a bare count, and handed to the agent BOTH as base64 blocks (so it can
|
|
1181
1939
|
// view them) and as absolute paths appended to the prompt (so it can operate on them — and
|
|
1182
1940
|
// because codex/opencode drop image blocks before they reach the model).
|
|
1183
|
-
const
|
|
1941
|
+
const freshAttachments = images
|
|
1184
1942
|
.filter((b) => b.type === 'image')
|
|
1185
1943
|
.map((b) => this.persistImage(runId, b.source.media_type, b.source.data, 'pasted'))
|
|
1186
1944
|
.filter((saved) => saved !== null);
|
|
1945
|
+
const openingImages = [...images, ...persistedImages];
|
|
1946
|
+
const attachments = [...freshAttachments, ...persistedAttachments];
|
|
1187
1947
|
this.store.appendEvent(runId, {
|
|
1188
1948
|
type: 'user-message',
|
|
1189
1949
|
stepId,
|
|
1190
1950
|
text: prompt,
|
|
1191
|
-
imageCount:
|
|
1951
|
+
imageCount: openingImages.filter((b) => b.type === 'image').length,
|
|
1192
1952
|
...(attachments.length ? { images: attachments.map((saved) => saved.url) } : {}),
|
|
1193
1953
|
});
|
|
1194
1954
|
let stepCost = 0;
|
|
1195
1955
|
let turnText = '';
|
|
1956
|
+
let sessionError;
|
|
1196
1957
|
const sink = this.makeUiSink(runId, stepId);
|
|
1197
1958
|
const onEvent = (event) => {
|
|
1198
1959
|
if (event.type === 'image') {
|
|
@@ -1209,6 +1970,13 @@ export class RunManager {
|
|
|
1209
1970
|
return;
|
|
1210
1971
|
}
|
|
1211
1972
|
this.store.appendEvent(runId, { ...event, stepId });
|
|
1973
|
+
if (event.type === 'error') {
|
|
1974
|
+
sessionError ??= event.message;
|
|
1975
|
+
state.session?.interrupt();
|
|
1976
|
+
return;
|
|
1977
|
+
}
|
|
1978
|
+
if (sessionError)
|
|
1979
|
+
return;
|
|
1212
1980
|
if (event.type === 'session') {
|
|
1213
1981
|
this.store.updateStep(runId, stepId, { sessionId: event.sessionId, backend });
|
|
1214
1982
|
}
|
|
@@ -1229,9 +1997,13 @@ export class RunManager {
|
|
|
1229
1997
|
const done = sessionOpen && DONE_MARKER_RE.test(turnText.trimEnd());
|
|
1230
1998
|
// `CEZ:ASK` → the user is genuinely blocked; wins over `CEZ:MONITORING`
|
|
1231
1999
|
// (a pending question is always attention), loses to `CEZ:DONE` (#473).
|
|
1232
|
-
const
|
|
2000
|
+
const askResult = sessionOpen && !done ? parseAskMarkerResult(turnText) : undefined;
|
|
2001
|
+
const ask = askResult?.kind === 'valid' ? askResult.request : null;
|
|
2002
|
+
const askRejection = askResult ? askMarkerRejection(askResult) : undefined;
|
|
1233
2003
|
const monitoring = sessionOpen && !done && !ask && MONITORING_MARKER_RE.test(turnText.trimEnd());
|
|
1234
2004
|
turnText = '';
|
|
2005
|
+
if (askRejection)
|
|
2006
|
+
this.store.appendEvent(runId, { type: 'note', message: askRejection, stepId });
|
|
1235
2007
|
if (done) {
|
|
1236
2008
|
// Goal achieved (agent contract, #347) — same as in runAgentStep.
|
|
1237
2009
|
this.store.appendEvent(runId, { type: 'lifecycle', message: 'goal achieved — session closed' });
|
|
@@ -1267,31 +2039,67 @@ export class RunManager {
|
|
|
1267
2039
|
if (monitoring) {
|
|
1268
2040
|
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
1269
2041
|
this.store.updateStep(runId, stepId, { status: 'running' });
|
|
2042
|
+
this.monitoring.add(runId);
|
|
2043
|
+
this.clearIdleTimer(state);
|
|
2044
|
+
this.armMonitoringWakeTimer(runId, state);
|
|
1270
2045
|
}
|
|
1271
2046
|
else {
|
|
1272
2047
|
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
1273
2048
|
this.store.updateStep(runId, stepId, { status: 'waiting' });
|
|
2049
|
+
this.monitoring.delete(runId);
|
|
2050
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1274
2051
|
}
|
|
1275
2052
|
this.waiting.add(runId);
|
|
1276
|
-
|
|
2053
|
+
if (!monitoring)
|
|
2054
|
+
this.armIdleTimer(runId, state);
|
|
1277
2055
|
this.releaseSlot();
|
|
1278
2056
|
}
|
|
1279
2057
|
}
|
|
2058
|
+
// A turn that completed is the ONLY evidence the provider's window actually reopened, so
|
|
2059
|
+
// it is what retires the consecutive-resume counter — which in turn releases the account
|
|
2060
|
+
// hold for every other task queued behind it (spec
|
|
2061
|
+
// 2026-08-03-auto-resume-after-usage-limit). `settleSuccess` does the same for a run that
|
|
2062
|
+
// finishes outright; this covers the far more common "parked for the user" ending.
|
|
2063
|
+
if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
|
|
2064
|
+
this.store.updateRun(runId, { autoResumeAttempts: undefined });
|
|
2065
|
+
}
|
|
1280
2066
|
appendHandoffHeartbeat(this.dataDir, runId, `turn complete — status=${monitoring ? 'monitoring' : sessionOpen ? 'waiting' : 'running'}`);
|
|
1281
2067
|
}
|
|
1282
2068
|
};
|
|
1283
2069
|
// Backend + model come off the record: the run's current backend by default, or the
|
|
1284
2070
|
// follow-up override that `continueRun` persisted before scheduling (#401).
|
|
1285
2071
|
const continueBackend = backend;
|
|
2072
|
+
/** Settle this turn as a failure before anything is spawned — the shape both
|
|
2073
|
+
* pre-spawn gates below need (model identity, #405; temp directory, #785). */
|
|
2074
|
+
const failBeforeSpawn = (message) => {
|
|
2075
|
+
const failedAt = new Date().toISOString();
|
|
2076
|
+
sink.sessionEnded('error', message);
|
|
2077
|
+
this.store.updateStep(runId, stepId, {
|
|
2078
|
+
status: 'failed',
|
|
2079
|
+
error: message,
|
|
2080
|
+
finishedAt: failedAt,
|
|
2081
|
+
});
|
|
2082
|
+
this.store.updateRun(runId, {
|
|
2083
|
+
status: 'failed',
|
|
2084
|
+
error: `continue failed: ${message}`,
|
|
2085
|
+
finishedAt: failedAt,
|
|
2086
|
+
currentStepId: undefined,
|
|
2087
|
+
});
|
|
2088
|
+
this.store.appendEvent(runId, {
|
|
2089
|
+
type: 'lifecycle',
|
|
2090
|
+
message: `continue failed — ${message}`,
|
|
2091
|
+
});
|
|
2092
|
+
this.dropActive(runId);
|
|
2093
|
+
};
|
|
1286
2094
|
// Apply the SAME canonical-identity gate the first spawn applies (#405, review M1).
|
|
1287
2095
|
// A follow-up may switch both runner and model (#401), so without this the record keeps
|
|
1288
2096
|
// asserting the identity the run STARTED with while a different model serves the turn —
|
|
1289
|
-
// the exact defect
|
|
2097
|
+
// the exact defect that PR existed to remove — and the raw record string reaches the CLI
|
|
1290
2098
|
// in the un-normalised wire form the first step already converted away (`anthropic/opus`
|
|
1291
2099
|
// instead of `opus`). Fail loud here too rather than let the backend pick a default.
|
|
1292
2100
|
let continueModel;
|
|
1293
2101
|
try {
|
|
1294
|
-
const normalized = normalizeModelForBackend(continueBackend, record?.model);
|
|
2102
|
+
const normalized = normalizeModelForBackend(continueBackend, agentModelsLocked(this.repoRoot) ? undefined : record?.model, { configuredProvider: await configuredModelProvider(continueBackend, state.cwd) });
|
|
1295
2103
|
continueModel = normalized?.backendModel;
|
|
1296
2104
|
this.store.updateRun(runId, {
|
|
1297
2105
|
modelIdentity: normalized ? formatModelIdentity(normalized.identity) : undefined,
|
|
@@ -1300,38 +2108,53 @@ export class RunManager {
|
|
|
1300
2108
|
catch (err) {
|
|
1301
2109
|
if (!(err instanceof ModelIdentityError))
|
|
1302
2110
|
throw err;
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
2111
|
+
failBeforeSpawn(err.message);
|
|
2112
|
+
return;
|
|
2113
|
+
}
|
|
2114
|
+
// Resuming reattaches to a session that lives inside ONE account's config dir, so the
|
|
2115
|
+
// continuation must run under the account that created it — not whatever the project has
|
|
2116
|
+
// been switched to since. The owning step is the one carrying this session id.
|
|
2117
|
+
const resumedProfileId = sessionId === undefined
|
|
2118
|
+
? undefined
|
|
2119
|
+
: record?.steps.find((s) => s.sessionId === sessionId)?.profileId;
|
|
2120
|
+
// The temp-directory preflight (#785) rides along with the account resolution: a resumed
|
|
2121
|
+
// turn hits the same broken `/tmp` a fresh one would, and an agent whose shell silently
|
|
2122
|
+
// returns nothing is worse than a turn that refuses to start and says why.
|
|
2123
|
+
let continueProfile;
|
|
2124
|
+
try {
|
|
2125
|
+
continueProfile = await this.agentEnvForStep(runId, continueBackend, {
|
|
2126
|
+
generateFollowups,
|
|
2127
|
+
recordedProfileId: resumedProfileId,
|
|
1319
2128
|
});
|
|
1320
|
-
|
|
2129
|
+
}
|
|
2130
|
+
catch (err) {
|
|
2131
|
+
if (!(err instanceof AgentTempDirError))
|
|
2132
|
+
throw err;
|
|
2133
|
+
failBeforeSpawn(err.message);
|
|
1321
2134
|
return;
|
|
1322
2135
|
}
|
|
2136
|
+
this.store.updateStep(runId, stepId, { profileId: continueProfile.profileId });
|
|
1323
2137
|
const runner = createRunner(continueBackend);
|
|
2138
|
+
state.currentStepId = stepId;
|
|
2139
|
+
this.beginUsageInvocation(runId, state, stepId);
|
|
2140
|
+
// A continuation's opening message becomes the session's `userPrompt` and never passes
|
|
2141
|
+
// through `deliverMessage`, so it needs the SAME delivery-only `/skill` rewrite the
|
|
2142
|
+
// live path applies (#811). Delivery-only: the `user-message` event above already
|
|
2143
|
+
// persisted the user's original text, and the transcript must keep showing that.
|
|
2144
|
+
const openingPrompt = expandRegistrySlashSkillText(prompt, state.skills ?? []);
|
|
1324
2145
|
const session = runner.startSession({
|
|
1325
2146
|
// The Continue step is a fresh agent session on the same run — the
|
|
1326
2147
|
// run's extra system prompt (already resolved at execute time and
|
|
1327
2148
|
// echoed on the record) rides along with the handoff contract.
|
|
1328
2149
|
systemPrompt: composeSystemPrompt(record?.systemPrompt, generateFollowups ? HANDOFF_INSTRUCTIONS : HANDOFF_ONLY_INSTRUCTIONS),
|
|
1329
|
-
userPrompt: attachments.length
|
|
1330
|
-
|
|
2150
|
+
userPrompt: attachments.length
|
|
2151
|
+
? `${openingPrompt}\n\n${pastedAttachmentsText(attachments)}`
|
|
2152
|
+
: openingPrompt,
|
|
2153
|
+
...(openingImages.length ? { images: openingImages } : {}),
|
|
1331
2154
|
cwd: state.cwd,
|
|
1332
2155
|
allowedTools: DEFAULT_ALLOWED_TOOLS,
|
|
1333
|
-
additionalDirectories:
|
|
1334
|
-
env:
|
|
2156
|
+
additionalDirectories: agentDirectories(join(this.dataDir, 'runs'), continueProfile.env),
|
|
2157
|
+
env: continueProfile.env,
|
|
1335
2158
|
model: continueModel,
|
|
1336
2159
|
sessionId,
|
|
1337
2160
|
resume: sessionId !== undefined,
|
|
@@ -1340,13 +2163,14 @@ export class RunManager {
|
|
|
1340
2163
|
state.session = session;
|
|
1341
2164
|
state.sessionEverOpened = true;
|
|
1342
2165
|
this.flushDeferred(runId);
|
|
1343
|
-
state.currentStepId = stepId;
|
|
1344
2166
|
state.interrupt = () => session.interrupt();
|
|
1345
2167
|
if (session.pid !== undefined)
|
|
1346
2168
|
registerRunProcess(runId, session.pid);
|
|
1347
2169
|
const finishedAt = () => new Date().toISOString();
|
|
1348
2170
|
try {
|
|
1349
2171
|
await session.result;
|
|
2172
|
+
if (sessionError)
|
|
2173
|
+
throw new Error(sessionError);
|
|
1350
2174
|
sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
|
|
1351
2175
|
if (state.cancelled) {
|
|
1352
2176
|
this.store.updateStep(runId, stepId, { status: 'cancelled', finishedAt: finishedAt() });
|
|
@@ -1399,6 +2223,12 @@ export class RunManager {
|
|
|
1399
2223
|
// the config default. Per-step `runner` can still override it below.
|
|
1400
2224
|
const config = await loadConfig(this.repoRoot);
|
|
1401
2225
|
const taskBackend = input.runner ?? config.defaultRunner;
|
|
2226
|
+
// The account may have gone into a usage-limit hold since this run was dequeued — the queue
|
|
2227
|
+
// gate cannot be the only one, because dequeue is not the moment of no return. Nothing has
|
|
2228
|
+
// happened yet here, so the run goes back to the queue untouched (spec
|
|
2229
|
+
// 2026-08-03-auto-resume-after-usage-limit).
|
|
2230
|
+
if (this.requeueWhileHeld(runId, workflow, input, taskBackend))
|
|
2231
|
+
return;
|
|
1402
2232
|
// Extra system prompt (R2 2.3): POST override > config default; echoed on
|
|
1403
2233
|
// the record so the UI/API can show what the run actually used.
|
|
1404
2234
|
const extraSystemPrompt = resolveExtraSystemPrompt(input.systemPrompt, config.systemPrompt);
|
|
@@ -1408,7 +2238,7 @@ export class RunManager {
|
|
|
1408
2238
|
// can still override below); the authoritative fail-loud gate is at spawn.
|
|
1409
2239
|
let modelIdentity;
|
|
1410
2240
|
try {
|
|
1411
|
-
const normalized = normalizeModelForBackend(taskBackend, input.model);
|
|
2241
|
+
const normalized = normalizeModelForBackend(taskBackend, agentModelsLocked(this.repoRoot) ? undefined : input.model, { configuredProvider: await configuredModelProvider(taskBackend, this.repoRoot) });
|
|
1412
2242
|
modelIdentity = normalized ? formatModelIdentity(normalized.identity) : undefined;
|
|
1413
2243
|
}
|
|
1414
2244
|
catch {
|
|
@@ -1429,11 +2259,21 @@ export class RunManager {
|
|
|
1429
2259
|
// established; only explicit opt-out and non-Git modes run in place.
|
|
1430
2260
|
const repo = await getRepoInfo(this.repoRoot);
|
|
1431
2261
|
if (repo && input.worktree === false) {
|
|
1432
|
-
// Composer opt-out: run in the repo working tree, no branch/worktree
|
|
1433
|
-
//
|
|
2262
|
+
// Composer opt-out: run in the repo working tree, no branch/worktree. The
|
|
2263
|
+
// repository-root lease serializes these runs by default; the explicit
|
|
2264
|
+
// CEZ_DISABLE_REPO_LOCK=1 escape hatch allows unsafe overlap.
|
|
2265
|
+
// Pin the starting commit: the session's Changes and Commits views use it
|
|
2266
|
+
// as their stable lower bound while reading the current working copy.
|
|
2267
|
+
const startingCommit = await getHeadCommit(repo.root);
|
|
2268
|
+
if (startingCommit)
|
|
2269
|
+
this.store.updateRun(runId, { baseBranch: startingCommit });
|
|
1434
2270
|
emit({ type: 'note', message: 'worktree off — running in the repo working tree' });
|
|
1435
2271
|
}
|
|
1436
2272
|
else if (repo) {
|
|
2273
|
+
emit({
|
|
2274
|
+
type: 'note',
|
|
2275
|
+
message: `worktree on — using an isolated task worktree (${input.worktree === true ? 'explicit request' : 'default'})`,
|
|
2276
|
+
});
|
|
1437
2277
|
// Fork from the configured base branch (config.json `baseBranch`, e.g.
|
|
1438
2278
|
// `develop`) — also the target of the eventual draft PR. Unresolvable
|
|
1439
2279
|
// (typo, not fetched) → note + the currently checked-out branch.
|
|
@@ -1493,14 +2333,31 @@ export class RunManager {
|
|
|
1493
2333
|
emit({ type: 'note', message: 'not a git repository — running in place, one task at a time' });
|
|
1494
2334
|
}
|
|
1495
2335
|
if (state.cwd === this.repoRoot) {
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
2336
|
+
if (repositoryRootLockDisabled()) {
|
|
2337
|
+
emit({
|
|
2338
|
+
type: 'note',
|
|
2339
|
+
message: REPOSITORY_ROOT_LOCK_DISABLED_NOTE,
|
|
2340
|
+
});
|
|
2341
|
+
}
|
|
2342
|
+
else {
|
|
2343
|
+
emit({
|
|
2344
|
+
type: 'note',
|
|
2345
|
+
message: 'waiting for exclusive access to the repository working tree',
|
|
2346
|
+
});
|
|
2347
|
+
// A cancel during the wait leaves the lease ungranted; the step loop
|
|
2348
|
+
// below breaks on `cancelled` before touching the tree and settles the
|
|
2349
|
+
// run through the usual path.
|
|
2350
|
+
await this.acquireRepoRoot(runId, state);
|
|
2351
|
+
}
|
|
2352
|
+
// THE window that matters for an in-place run. Waiting for the exclusive tree can take
|
|
2353
|
+
// minutes, and a run parked on that lease holds no slot (#347) — so the queue keeps
|
|
2354
|
+
// advancing behind it and the dequeue-time gate is long past. Measured with five in-place
|
|
2355
|
+
// tasks and `maxParallel: 2`: four of them started. Re-ask here, where the very next thing
|
|
2356
|
+
// is a spawn, and hand the run back to the queue if the account closed meanwhile. This
|
|
2357
|
+
// check also covers the explicit lock-bypass path, where the account may close while the
|
|
2358
|
+
// run is preparing its first step.
|
|
2359
|
+
if (this.requeueWhileHeld(runId, workflow, input, taskBackend, state))
|
|
2360
|
+
return;
|
|
1504
2361
|
}
|
|
1505
2362
|
// Handoff journal (spec 007) — seeded after the worktree exists so the
|
|
1506
2363
|
// header can name the branch. Idempotent: an existing file stays as-is.
|
|
@@ -1508,6 +2365,9 @@ export class RunManager {
|
|
|
1508
2365
|
if (seeded)
|
|
1509
2366
|
seedHandoffFile(this.dataDir, seeded);
|
|
1510
2367
|
const skills = await discoverSkills(this.repoRoot);
|
|
2368
|
+
// Every ActiveRun construction site must carry the registry — `runContinuation` builds
|
|
2369
|
+
// its own, and the one that skipped this leaked raw `/skill` text to the backend (#811).
|
|
2370
|
+
state.skills = skills;
|
|
1511
2371
|
const retriesUsed = new Map();
|
|
1512
2372
|
let checkFailure = null;
|
|
1513
2373
|
let runError = null;
|
|
@@ -1687,6 +2547,7 @@ export class RunManager {
|
|
|
1687
2547
|
const startTokens = stepRecord?.tokensUsed ?? 0;
|
|
1688
2548
|
let stepCost = stepRecord?.costUsd ?? 0;
|
|
1689
2549
|
let turnText = '';
|
|
2550
|
+
let sessionError;
|
|
1690
2551
|
const sink = this.makeUiSink(runId, step.id);
|
|
1691
2552
|
const onEvent = (event) => {
|
|
1692
2553
|
if (event.type === 'image') {
|
|
@@ -1703,6 +2564,13 @@ export class RunManager {
|
|
|
1703
2564
|
return;
|
|
1704
2565
|
}
|
|
1705
2566
|
emit({ ...event, stepId: step.id });
|
|
2567
|
+
if (event.type === 'error') {
|
|
2568
|
+
sessionError ??= event.message;
|
|
2569
|
+
state.session?.interrupt();
|
|
2570
|
+
return;
|
|
2571
|
+
}
|
|
2572
|
+
if (sessionError)
|
|
2573
|
+
return;
|
|
1706
2574
|
if (event.type === 'session') {
|
|
1707
2575
|
// Codex/OpenCode mint their own session id — persist it so resume works.
|
|
1708
2576
|
this.store.updateStep(runId, step.id, { sessionId: event.sessionId, backend });
|
|
@@ -1723,13 +2591,17 @@ export class RunManager {
|
|
|
1723
2591
|
const done = interactive && sessionOpen && DONE_MARKER_RE.test(turnText.trimEnd());
|
|
1724
2592
|
// `CEZ:ASK` → the user is blocked; wins over `CEZ:MONITORING`, loses to
|
|
1725
2593
|
// `CEZ:DONE` (#473).
|
|
1726
|
-
const
|
|
2594
|
+
const askResult = interactive && sessionOpen && !done ? parseAskMarkerResult(turnText) : undefined;
|
|
2595
|
+
const ask = askResult?.kind === 'valid' ? askResult.request : null;
|
|
2596
|
+
const askRejection = askResult ? askMarkerRejection(askResult) : undefined;
|
|
1727
2597
|
const monitoring = interactive &&
|
|
1728
2598
|
sessionOpen &&
|
|
1729
2599
|
!done &&
|
|
1730
2600
|
!ask &&
|
|
1731
2601
|
MONITORING_MARKER_RE.test(turnText.trimEnd());
|
|
1732
2602
|
turnText = '';
|
|
2603
|
+
if (askRejection)
|
|
2604
|
+
emit({ type: 'note', stepId: step.id, message: askRejection });
|
|
1733
2605
|
if (done) {
|
|
1734
2606
|
// Goal achieved (agent contract, #347): close the session instead
|
|
1735
2607
|
// of parking at `waiting` — the run completes and frees its slot.
|
|
@@ -1752,15 +2624,25 @@ export class RunManager {
|
|
|
1752
2624
|
if (monitoring) {
|
|
1753
2625
|
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
1754
2626
|
this.store.updateStep(runId, step.id, { status: 'running' });
|
|
2627
|
+
this.monitoring.add(runId);
|
|
2628
|
+
this.clearIdleTimer(state);
|
|
2629
|
+
this.armMonitoringWakeTimer(runId, state);
|
|
1755
2630
|
}
|
|
1756
2631
|
else {
|
|
1757
2632
|
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
1758
2633
|
this.store.updateStep(runId, step.id, { status: 'waiting' });
|
|
2634
|
+
this.monitoring.delete(runId);
|
|
2635
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1759
2636
|
}
|
|
1760
2637
|
this.waiting.add(runId);
|
|
1761
|
-
|
|
2638
|
+
if (!monitoring)
|
|
2639
|
+
this.armIdleTimer(runId, state);
|
|
1762
2640
|
this.releaseSlot(); // the freed slot can start a queued run right away — in any project
|
|
1763
2641
|
}
|
|
2642
|
+
// The window is proven open — see the twin in `runContinuation`.
|
|
2643
|
+
if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
|
|
2644
|
+
this.store.updateRun(runId, { autoResumeAttempts: undefined });
|
|
2645
|
+
}
|
|
1764
2646
|
// Cez's own heartbeat — the handoff stays current even when the
|
|
1765
2647
|
// agent forgets to write (spec 007).
|
|
1766
2648
|
appendHandoffHeartbeat(this.dataDir, runId, `turn complete — status=${monitoring ? 'monitoring' : waiting ? 'waiting' : 'running'}`);
|
|
@@ -1773,7 +2655,7 @@ export class RunManager {
|
|
|
1773
2655
|
// instead of letting the backend silently substitute its default.
|
|
1774
2656
|
let backendModel;
|
|
1775
2657
|
try {
|
|
1776
|
-
const normalized = normalizeModelForBackend(stepBackend, step.model ?? input.model);
|
|
2658
|
+
const normalized = normalizeModelForBackend(stepBackend, agentModelsLocked(this.repoRoot) ? undefined : step.model ?? input.model, { configuredProvider: await configuredModelProvider(stepBackend, state.cwd) });
|
|
1777
2659
|
backendModel = normalized?.backendModel;
|
|
1778
2660
|
// Persist the identity of what ACTUALLY runs (#405, review M1). The run-start echo
|
|
1779
2661
|
// (line ~993) is best-effort from `taskBackend`/`input.model`; a per-step `runner`/`model`
|
|
@@ -1788,8 +2670,27 @@ export class RunManager {
|
|
|
1788
2670
|
return err.message;
|
|
1789
2671
|
throw err;
|
|
1790
2672
|
}
|
|
2673
|
+
// Which agent account this step spawns under, and — recorded on the step before the spawn —
|
|
2674
|
+
// which one its session belongs to. `sessionId` and `profileId` are a pair: a resume that
|
|
2675
|
+
// reads the wrong account's config dir finds no session and silently starts a fresh one.
|
|
2676
|
+
// Resolved together with the temp-directory preflight (#785): the step fails with a named,
|
|
2677
|
+
// actionable error instead of spawning a backend whose shell would return empty output.
|
|
2678
|
+
let stepProfile;
|
|
2679
|
+
try {
|
|
2680
|
+
stepProfile = await this.agentEnvForStep(runId, stepBackend, {
|
|
2681
|
+
generateFollowups: followupsEnabled() && input.generateFollowups !== false,
|
|
2682
|
+
});
|
|
2683
|
+
}
|
|
2684
|
+
catch (err) {
|
|
2685
|
+
if (err instanceof AgentTempDirError)
|
|
2686
|
+
return err.message;
|
|
2687
|
+
throw err;
|
|
2688
|
+
}
|
|
2689
|
+
this.store.updateStep(runId, step.id, { profileId: stepProfile.profileId });
|
|
1791
2690
|
const runner = createRunner(stepBackend);
|
|
1792
2691
|
let session;
|
|
2692
|
+
state.currentStepId = step.id;
|
|
2693
|
+
this.beginUsageInvocation(runId, state, step.id);
|
|
1793
2694
|
try {
|
|
1794
2695
|
session = runner.startSession({
|
|
1795
2696
|
// Skill body, then the run's extra prompt (POST override or config
|
|
@@ -1803,8 +2704,8 @@ export class RunManager {
|
|
|
1803
2704
|
allowedTools: step.allowedTools ?? DEFAULT_ALLOWED_TOOLS,
|
|
1804
2705
|
bashAllowlist: step.bashAllowlist,
|
|
1805
2706
|
// The handoff file lives outside the worktree — grant access.
|
|
1806
|
-
additionalDirectories:
|
|
1807
|
-
env:
|
|
2707
|
+
additionalDirectories: agentDirectories(join(this.dataDir, 'runs'), stepProfile.env),
|
|
2708
|
+
env: stepProfile.env,
|
|
1808
2709
|
model: backendModel,
|
|
1809
2710
|
sessionId,
|
|
1810
2711
|
// Interactive sessions have no wall clock — the idle timer rules.
|
|
@@ -1815,6 +2716,7 @@ export class RunManager {
|
|
|
1815
2716
|
});
|
|
1816
2717
|
}
|
|
1817
2718
|
catch (err) {
|
|
2719
|
+
state.currentStepId = undefined;
|
|
1818
2720
|
return err instanceof Error ? err.message : String(err);
|
|
1819
2721
|
}
|
|
1820
2722
|
state.session = session;
|
|
@@ -1826,6 +2728,10 @@ export class RunManager {
|
|
|
1826
2728
|
registerRunProcess(runId, session.pid);
|
|
1827
2729
|
try {
|
|
1828
2730
|
const result = await session.result;
|
|
2731
|
+
if (sessionError) {
|
|
2732
|
+
sink.sessionEnded('error', sessionError);
|
|
2733
|
+
return sessionError;
|
|
2734
|
+
}
|
|
1829
2735
|
// v2 counterpart of v1's `done` (spec: the mappers leave session-close
|
|
1830
2736
|
// events to the RunManager — only it knows how the session settled).
|
|
1831
2737
|
sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
|
|
@@ -1840,6 +2746,9 @@ export class RunManager {
|
|
|
1840
2746
|
finally {
|
|
1841
2747
|
this.recordUsagePeaks(runId);
|
|
1842
2748
|
this.clearIdleTimer(state);
|
|
2749
|
+
this.monitoring.delete(runId);
|
|
2750
|
+
this.waiting.delete(runId);
|
|
2751
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1843
2752
|
state.session = undefined;
|
|
1844
2753
|
state.currentStepId = undefined;
|
|
1845
2754
|
state.interrupt = () => undefined;
|
|
@@ -1863,16 +2772,90 @@ export class RunManager {
|
|
|
1863
2772
|
/** Native backend asks arrive before turn-end. Persist and park immediately
|
|
1864
2773
|
* so the cockpit shows attention and the run releases its workspace slot. */
|
|
1865
2774
|
handleRunnerUiEvent(runId, state, sink, event) {
|
|
2775
|
+
this.recordUsageUiEvent(runId, state, event);
|
|
1866
2776
|
sink.handle(event);
|
|
1867
2777
|
if (event.type !== 'ask.requested' || state.cancelled)
|
|
1868
2778
|
return;
|
|
1869
2779
|
this.clearIdleTimer(state);
|
|
2780
|
+
this.monitoring.delete(runId);
|
|
2781
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1870
2782
|
this.waiting.add(runId);
|
|
1871
2783
|
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
1872
2784
|
if (state.currentStepId)
|
|
1873
2785
|
this.store.updateStep(runId, state.currentStepId, { status: 'waiting' });
|
|
1874
2786
|
this.releaseSlot();
|
|
1875
2787
|
}
|
|
2788
|
+
/** Persist the invocation checkpoint before launching a runner. A throw or
|
|
2789
|
+
* process exit before `turn.started` therefore leaves a durable mismatch. */
|
|
2790
|
+
beginUsageInvocation(runId, state, stepId) {
|
|
2791
|
+
const step = this.store.getRun(runId)?.steps.find((candidate) => candidate.id === stepId);
|
|
2792
|
+
if (!step)
|
|
2793
|
+
return;
|
|
2794
|
+
const epoch = (step.usageInvocationEpoch ?? 0) + 1;
|
|
2795
|
+
this.persistUsageCheckpoint(runId, stepId, {
|
|
2796
|
+
usageInvocationEpoch: epoch,
|
|
2797
|
+
usageInvocationsStarted: (step.usageInvocationsStarted ?? 0) + 1,
|
|
2798
|
+
});
|
|
2799
|
+
state.usageInvocation = {
|
|
2800
|
+
stepId,
|
|
2801
|
+
epoch,
|
|
2802
|
+
observed: false,
|
|
2803
|
+
startedTurns: new Set(),
|
|
2804
|
+
recordedTurns: new Set(),
|
|
2805
|
+
};
|
|
2806
|
+
}
|
|
2807
|
+
/** Fold backend-neutral completed-turn usage into the current step exactly
|
|
2808
|
+
* once. Invocation/turn counters are written before the event reaches the
|
|
2809
|
+
* NDJSON sink so crashes cannot preserve a falsely complete subtotal. */
|
|
2810
|
+
recordUsageUiEvent(runId, state, event) {
|
|
2811
|
+
const invocation = state.usageInvocation;
|
|
2812
|
+
if (!invocation)
|
|
2813
|
+
return;
|
|
2814
|
+
const step = this.store.getRun(runId)?.steps.find((candidate) => candidate.id === invocation.stepId);
|
|
2815
|
+
if (!step)
|
|
2816
|
+
return;
|
|
2817
|
+
if (event.type === 'turn.started') {
|
|
2818
|
+
if (invocation.startedTurns.has(event.turnId))
|
|
2819
|
+
return;
|
|
2820
|
+
invocation.startedTurns.add(event.turnId);
|
|
2821
|
+
const firstObservedTurn = !invocation.observed;
|
|
2822
|
+
invocation.observed = true;
|
|
2823
|
+
this.persistUsageCheckpoint(runId, invocation.stepId, {
|
|
2824
|
+
usageTurnsStarted: (step.usageTurnsStarted ?? 0) + 1,
|
|
2825
|
+
...(firstObservedTurn
|
|
2826
|
+
? { usageInvocationsObserved: (step.usageInvocationsObserved ?? 0) + 1 }
|
|
2827
|
+
: {}),
|
|
2828
|
+
});
|
|
2829
|
+
return;
|
|
2830
|
+
}
|
|
2831
|
+
if (event.type !== 'turn.completed')
|
|
2832
|
+
return;
|
|
2833
|
+
if (!invocation.startedTurns.has(event.turnId) || invocation.recordedTurns.has(event.turnId))
|
|
2834
|
+
return;
|
|
2835
|
+
const input = event.usage?.input;
|
|
2836
|
+
const output = event.usage?.output;
|
|
2837
|
+
if (typeof input !== 'number' ||
|
|
2838
|
+
!Number.isFinite(input) ||
|
|
2839
|
+
input < 0 ||
|
|
2840
|
+
typeof output !== 'number' ||
|
|
2841
|
+
!Number.isFinite(output) ||
|
|
2842
|
+
output < 0) {
|
|
2843
|
+
return;
|
|
2844
|
+
}
|
|
2845
|
+
invocation.recordedTurns.add(event.turnId);
|
|
2846
|
+
this.persistUsageCheckpoint(runId, invocation.stepId, {
|
|
2847
|
+
inputTokens: (step.inputTokens ?? 0) + input,
|
|
2848
|
+
outputTokens: (step.outputTokens ?? 0) + output,
|
|
2849
|
+
usageTurnsRecorded: (step.usageTurnsRecorded ?? 0) + 1,
|
|
2850
|
+
});
|
|
2851
|
+
}
|
|
2852
|
+
/** Usage completeness is a crash boundary, unlike high-frequency token
|
|
2853
|
+
* snapshots: the checkpoint must reach `runs.json` before the runner starts
|
|
2854
|
+
* or the matching UI event is persisted and forwarded. */
|
|
2855
|
+
persistUsageCheckpoint(runId, stepId, patch) {
|
|
2856
|
+
this.store.updateStep(runId, stepId, patch);
|
|
2857
|
+
this.store.flush();
|
|
2858
|
+
}
|
|
1876
2859
|
/**
|
|
1877
2860
|
* Turn-end bookkeeping (#389), shared by `runAgentStep` and
|
|
1878
2861
|
* `runContinuation` — called (fire-and-forget) from every `turn-end` event:
|
|
@@ -1939,7 +2922,14 @@ export class RunManager {
|
|
|
1939
2922
|
// deliberately NEVER a title source; see maybeRefreshTitle below. The
|
|
1940
2923
|
// one exception is an explicit CEZ:TITLE declaration (applied above).
|
|
1941
2924
|
if (run.worktreePath && existsSync(run.worktreePath)) {
|
|
1942
|
-
|
|
2925
|
+
// `taskBranch` + `runStartedAt` are what keep this number *this task's* (#751): a
|
|
2926
|
+
// review/QA run repoints the worktree onto the branch under review, and without the
|
|
2927
|
+
// branch to compare HEAD against and the moment it was checked out, the stat would
|
|
2928
|
+
// claim that whole branch's diff.
|
|
2929
|
+
const stat = await worktreeShortstat(run.worktreePath, run.baseBranch ?? 'HEAD', {
|
|
2930
|
+
taskBranch: run.branch,
|
|
2931
|
+
runStartedAt: run.startedAt,
|
|
2932
|
+
});
|
|
1943
2933
|
if (stat)
|
|
1944
2934
|
this.store.updateRun(runId, { diffStat: stat });
|
|
1945
2935
|
else
|
|
@@ -2044,6 +3034,10 @@ export class RunManager {
|
|
|
2044
3034
|
status: review ? 'review' : 'done',
|
|
2045
3035
|
finishedAt: new Date().toISOString(),
|
|
2046
3036
|
currentStepId: undefined,
|
|
3037
|
+
// A run that got all the way to a settled turn is not in a limit loop, so the resume
|
|
3038
|
+
// counter starts over — otherwise a task that legitimately met the limit once a week would
|
|
3039
|
+
// creep toward the cap forever and stop resuming for no reason anyone could see.
|
|
3040
|
+
autoResumeAttempts: undefined,
|
|
2047
3041
|
});
|
|
2048
3042
|
this.store.appendEvent(runId, {
|
|
2049
3043
|
type: 'lifecycle',
|
|
@@ -2096,7 +3090,10 @@ export class RunManager {
|
|
|
2096
3090
|
throw err;
|
|
2097
3091
|
}
|
|
2098
3092
|
this.queuedImageSeq.set(runId, seq);
|
|
2099
|
-
|
|
3093
|
+
// Versioned, because that is the only surface served now. The cockpit still upgrades
|
|
3094
|
+
// the unversioned URLs sitting in OLD transcripts when it renders them
|
|
3095
|
+
// (`resolveApiUrl`), but a URL minted today must be fetchable as written.
|
|
3096
|
+
return { name, url: `/api/v1/runs/${runId}/images/${name}`, path };
|
|
2100
3097
|
}
|
|
2101
3098
|
return null;
|
|
2102
3099
|
}
|
|
@@ -2123,6 +3120,68 @@ export class RunManager {
|
|
|
2123
3120
|
state.idleTimer = undefined;
|
|
2124
3121
|
}
|
|
2125
3122
|
}
|
|
3123
|
+
reconcileMonitoringWakeTimers() {
|
|
3124
|
+
for (const runId of this.monitoring) {
|
|
3125
|
+
const state = this.active.get(runId);
|
|
3126
|
+
if (state)
|
|
3127
|
+
this.armMonitoringWakeTimer(runId, state);
|
|
3128
|
+
}
|
|
3129
|
+
}
|
|
3130
|
+
armMonitoringWakeTimer(runId, state) {
|
|
3131
|
+
const minutes = this.semaphore.monitoringWakeIntervalMinutes();
|
|
3132
|
+
if (minutes === null) {
|
|
3133
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
3134
|
+
return;
|
|
3135
|
+
}
|
|
3136
|
+
if ((state.monitoringWakeups ?? 0) >= MAX_AUTO_CONTINUES) {
|
|
3137
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
3138
|
+
if (!this.store.getRun(runId)?.monitoringWakeCapReached) {
|
|
3139
|
+
this.store.updateRun(runId, { monitoringWakeCapReached: true });
|
|
3140
|
+
this.store.appendEvent(runId, {
|
|
3141
|
+
type: 'note',
|
|
3142
|
+
message: `automatic monitoring wake-up cap reached (${MAX_AUTO_CONTINUES}); session remains parked`,
|
|
3143
|
+
});
|
|
3144
|
+
}
|
|
3145
|
+
return;
|
|
3146
|
+
}
|
|
3147
|
+
if (state.monitoringWakeTimer && state.monitoringWakeIntervalMinutes === minutes)
|
|
3148
|
+
return;
|
|
3149
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
3150
|
+
state.monitoringWakeIntervalMinutes = minutes;
|
|
3151
|
+
this.store.updateRun(runId, { monitoringWakeCapReached: undefined });
|
|
3152
|
+
const deadline = Date.now() + minutes * 60_000;
|
|
3153
|
+
this.store.updateRun(runId, { monitoringWakeAt: new Date(deadline).toISOString() });
|
|
3154
|
+
state.monitoringWakeTimer = setTimeout(() => {
|
|
3155
|
+
state.monitoringWakeTimer = undefined;
|
|
3156
|
+
this.store.updateRun(runId, { monitoringWakeAt: undefined });
|
|
3157
|
+
if (!this.monitoring.has(runId) || !state.session?.open || state.cancelled)
|
|
3158
|
+
return;
|
|
3159
|
+
const wakeups = state.monitoringWakeups ?? 0;
|
|
3160
|
+
if (wakeups >= MAX_AUTO_CONTINUES) {
|
|
3161
|
+
this.store.updateRun(runId, { monitoringWakeCapReached: true });
|
|
3162
|
+
this.store.appendEvent(runId, {
|
|
3163
|
+
type: 'note',
|
|
3164
|
+
message: `automatic monitoring wake-up cap reached (${MAX_AUTO_CONTINUES}); session remains parked`,
|
|
3165
|
+
});
|
|
3166
|
+
return;
|
|
3167
|
+
}
|
|
3168
|
+
state.monitoringWakeups = wakeups + 1;
|
|
3169
|
+
this.store.appendEvent(runId, {
|
|
3170
|
+
type: 'note',
|
|
3171
|
+
message: `automatic monitoring wake-up (${state.monitoringWakeups}/${MAX_AUTO_CONTINUES})`,
|
|
3172
|
+
});
|
|
3173
|
+
this.deliverMessage(runId, [{ type: 'text', text: MONITORING_WAKE_NUDGE }], false);
|
|
3174
|
+
}, Math.max(0, deadline - Date.now()));
|
|
3175
|
+
state.monitoringWakeTimer.unref?.();
|
|
3176
|
+
}
|
|
3177
|
+
clearMonitoringWakeTimer(state, runId) {
|
|
3178
|
+
if (state.monitoringWakeTimer)
|
|
3179
|
+
clearTimeout(state.monitoringWakeTimer);
|
|
3180
|
+
state.monitoringWakeTimer = undefined;
|
|
3181
|
+
state.monitoringWakeIntervalMinutes = undefined;
|
|
3182
|
+
if (runId)
|
|
3183
|
+
this.store.updateRun(runId, { monitoringWakeAt: undefined });
|
|
3184
|
+
}
|
|
2126
3185
|
/** Autosave-commit the worktree every 90 s while the run lives (spec 006).
|
|
2127
3186
|
* Opt-in via CEZ_AUTOSAVE=1 (#471) — see periodicAutosaveEnabled. */
|
|
2128
3187
|
armAutosave(state) {
|
|
@@ -2244,4 +3303,47 @@ export function skillSystemPrompt(skill) {
|
|
|
2244
3303
|
lines.push('', 'Skill instructions:', skill.body.trim());
|
|
2245
3304
|
return lines.join('\n');
|
|
2246
3305
|
}
|
|
3306
|
+
/**
|
|
3307
|
+
* Expand a registry-backed slash skill in one prompt string before it reaches a
|
|
3308
|
+
* backend. Claude otherwise intercepts an unknown leading slash command, and
|
|
3309
|
+
* Codex/OpenCode have no native slash-skill lookup at all (#676).
|
|
3310
|
+
*
|
|
3311
|
+
* Only a match at character zero counts, and unknown commands pass through
|
|
3312
|
+
* byte-for-byte — a backend's OWN slash commands must keep working. The caller
|
|
3313
|
+
* persists the original user text before applying this delivery-only rewrite.
|
|
3314
|
+
*
|
|
3315
|
+
* Both delivery seams route through here: live-session messages via
|
|
3316
|
+
* `expandRegistrySlashSkill`, and a continuation's opening prompt, which becomes
|
|
3317
|
+
* the session's `userPrompt` and never passes through `deliverMessage` at all
|
|
3318
|
+
* (#811).
|
|
3319
|
+
*/
|
|
3320
|
+
export function expandRegistrySlashSkillText(text, skills) {
|
|
3321
|
+
const match = /^\/([A-Za-z0-9][A-Za-z0-9._-]*)(?=\s|$)/.exec(text);
|
|
3322
|
+
if (!match)
|
|
3323
|
+
return text;
|
|
3324
|
+
const skill = skills.find((candidate) => candidate.name === match[1]);
|
|
3325
|
+
if (!skill)
|
|
3326
|
+
return text;
|
|
3327
|
+
const request = text.slice(match[0].length).trim();
|
|
3328
|
+
return request ? `${skillSystemPrompt(skill)}\n\nUser request:\n${request}` : skillSystemPrompt(skill);
|
|
3329
|
+
}
|
|
3330
|
+
/**
|
|
3331
|
+
* `expandRegistrySlashSkillText` over a live chat message: only the first text
|
|
3332
|
+
* block is eligible, and an unchanged block returns the caller's array
|
|
3333
|
+
* identity untouched.
|
|
3334
|
+
*/
|
|
3335
|
+
export function expandRegistrySlashSkill(content, skills) {
|
|
3336
|
+
const textIndex = content.findIndex((block) => block.type === 'text');
|
|
3337
|
+
if (textIndex < 0)
|
|
3338
|
+
return content;
|
|
3339
|
+
const block = content[textIndex];
|
|
3340
|
+
if (!block || block.type !== 'text')
|
|
3341
|
+
return content;
|
|
3342
|
+
const text = expandRegistrySlashSkillText(block.text, skills);
|
|
3343
|
+
if (text === block.text)
|
|
3344
|
+
return content;
|
|
3345
|
+
const expanded = [...content];
|
|
3346
|
+
expanded[textIndex] = { type: 'text', text };
|
|
3347
|
+
return expanded;
|
|
3348
|
+
}
|
|
2247
3349
|
//# sourceMappingURL=run.js.map
|