@open-mercato/cezar 0.9.1 → 0.9.2-pr772.1071
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -17
- package/dist/agent-config/account-identity.d.ts +44 -0
- package/dist/agent-config/account-identity.js +128 -0
- package/dist/agent-config/account-identity.js.map +1 -0
- package/dist/agent-config/catalog.d.ts +9 -1
- package/dist/agent-config/catalog.js +18 -0
- package/dist/agent-config/catalog.js.map +1 -1
- package/dist/agent-config/files.d.ts +1 -1
- package/dist/agent-config/model-settings/claude.d.ts +2 -0
- package/dist/agent-config/model-settings/claude.js +11 -0
- package/dist/agent-config/model-settings/claude.js.map +1 -0
- package/dist/agent-config/model-settings/codex.d.ts +2 -0
- package/dist/agent-config/model-settings/codex.js +16 -0
- package/dist/agent-config/model-settings/codex.js.map +1 -0
- package/dist/agent-config/model-settings/opencode.d.ts +2 -0
- package/dist/agent-config/model-settings/opencode.js +8 -0
- package/dist/agent-config/model-settings/opencode.js.map +1 -0
- package/dist/agent-config/model-settings/shared.d.ts +9 -0
- package/dist/agent-config/model-settings/shared.js +93 -0
- package/dist/agent-config/model-settings/shared.js.map +1 -0
- package/dist/agent-config/model-settings/types.d.ts +9 -0
- package/dist/agent-config/model-settings/types.js +2 -0
- package/dist/agent-config/model-settings/types.js.map +1 -0
- package/dist/agent-config/models.d.ts +9 -0
- package/dist/agent-config/models.js +30 -0
- package/dist/agent-config/models.js.map +1 -0
- package/dist/agent-config/service.d.ts +1 -1
- package/dist/agent-config/service.js +4 -4
- package/dist/agent-config/service.js.map +1 -1
- package/dist/agent-config/validate.d.ts +1 -1
- package/dist/automations/coordinator.d.ts +26 -0
- package/dist/automations/coordinator.js +66 -0
- package/dist/automations/coordinator.js.map +1 -0
- package/dist/automations/github-poller.d.ts +74 -0
- package/dist/automations/github-poller.js +234 -0
- package/dist/automations/github-poller.js.map +1 -0
- package/dist/automations/scheduler.d.ts +42 -0
- package/dist/automations/scheduler.js +192 -0
- package/dist/automations/scheduler.js.map +1 -0
- package/dist/automations/store.d.ts +65 -0
- package/dist/automations/store.js +298 -0
- package/dist/automations/store.js.map +1 -0
- package/dist/automations/task-template.d.ts +18 -0
- package/dist/automations/task-template.js +95 -0
- package/dist/automations/task-template.js.map +1 -0
- package/dist/automations/types.d.ts +251 -0
- package/dist/automations/types.js +158 -0
- package/dist/automations/types.js.map +1 -0
- package/dist/config.d.ts +17 -112
- package/dist/config.js +45 -4
- package/dist/config.js.map +1 -1
- package/dist/contract/agent-config.d.ts +151 -0
- package/dist/contract/agent-profiles.d.ts +332 -0
- package/dist/contract/automations.d.ts +919 -0
- package/dist/contract/events.d.ts +45 -0
- package/dist/contract/github.d.ts +467 -0
- package/dist/contract/health.d.ts +96 -0
- package/dist/contract/index.d.ts +16 -0
- package/dist/contract/index.js +1614 -0
- package/dist/contract/projects.d.ts +185 -0
- package/dist/contract/repo.d.ts +234 -0
- package/dist/contract/runs.d.ts +1261 -0
- package/dist/contract/skills.d.ts +229 -0
- package/dist/contract/workflows.d.ts +212 -0
- package/dist/contract/workspace.d.ts +490 -0
- package/dist/core/agent-env.d.ts +1 -1
- package/dist/core/agent-model-policy.d.ts +10 -0
- package/dist/core/agent-model-policy.js +36 -0
- package/dist/core/agent-model-policy.js.map +1 -0
- package/dist/core/agent-profiles.d.ts +59 -0
- package/dist/core/agent-profiles.js +83 -0
- package/dist/core/agent-profiles.js.map +1 -0
- package/dist/core/agent-runner.d.ts +12 -1
- package/dist/core/agent-runner.js +13 -0
- package/dist/core/agent-runner.js.map +1 -1
- package/dist/core/ask.d.ts +45 -124
- package/dist/core/ask.js +103 -13
- package/dist/core/ask.js.map +1 -1
- package/dist/core/backend-detect.js +7 -2
- package/dist/core/backend-detect.js.map +1 -1
- package/dist/core/claude-cli-runner.d.ts +2 -2
- package/dist/core/claude-cli-runner.js +52 -9
- package/dist/core/claude-cli-runner.js.map +1 -1
- package/dist/core/claude-ui-mapper.d.ts +1 -1
- package/dist/core/codex-app-server-runner.d.ts +1 -1
- package/dist/core/codex-app-server-runner.js +60 -3
- package/dist/core/codex-app-server-runner.js.map +1 -1
- package/dist/core/codex-app-server-transport.d.ts +6 -1
- package/dist/core/codex-app-server-transport.js +12 -3
- package/dist/core/codex-app-server-transport.js.map +1 -1
- package/dist/core/codex-model-catalog.d.ts +1 -1
- package/dist/core/codex-ui-mapper.d.ts +5 -1
- package/dist/core/codex-ui-mapper.js +74 -17
- package/dist/core/codex-ui-mapper.js.map +1 -1
- package/dist/core/model-identity.d.ts +13 -6
- package/dist/core/model-identity.js +32 -17
- package/dist/core/model-identity.js.map +1 -1
- package/dist/core/model-presets.d.ts +2 -2
- package/dist/core/model-presets.js +1 -1
- package/dist/core/opencode-server-runner.d.ts +2 -2
- package/dist/core/opencode-ui-mapper.d.ts +4 -1
- package/dist/core/opencode-ui-mapper.js +45 -7
- package/dist/core/opencode-ui-mapper.js.map +1 -1
- package/dist/core/provider-auth.d.ts +154 -0
- package/dist/core/provider-auth.js +488 -0
- package/dist/core/provider-auth.js.map +1 -0
- package/dist/core/provider-availability.d.ts +3 -0
- package/dist/core/provider-availability.js +14 -0
- package/dist/core/provider-availability.js.map +1 -0
- package/dist/core/runner-factory.d.ts +1 -1
- package/dist/core/runner-model-catalog.d.ts +1 -1
- package/dist/core/shell-env.d.ts +28 -0
- package/dist/core/shell-env.js +56 -0
- package/dist/core/shell-env.js.map +1 -0
- package/dist/core/tool-display.d.ts +1 -1
- package/dist/core/ui-events.d.ts +1 -1
- package/dist/core/ui-events.js +1 -1
- package/dist/core/usage-limit.d.ts +42 -0
- package/dist/core/usage-limit.js +209 -0
- package/dist/core/usage-limit.js.map +1 -0
- package/dist/git-diff-base.d.ts +60 -0
- package/dist/git-diff-base.js +54 -0
- package/dist/git-diff-base.js.map +1 -0
- package/dist/git-worktree.d.ts +37 -6
- package/dist/git-worktree.js +39 -9
- package/dist/git-worktree.js.map +1 -1
- package/dist/handoff.d.ts +2 -2
- package/dist/handoff.js +1 -1
- package/dist/index.js +65 -8
- package/dist/index.js.map +1 -1
- package/dist/paths.d.ts +69 -6
- package/dist/paths.js +86 -9
- package/dist/paths.js.map +1 -1
- package/dist/planner.d.ts +2 -2
- package/dist/planner.js +7 -1
- package/dist/planner.js.map +1 -1
- package/dist/release/manifests.d.ts +81 -0
- package/dist/release/manifests.js +74 -0
- package/dist/release/manifests.js.map +1 -0
- package/dist/release/snapshot.d.ts +9 -23
- package/dist/release/snapshot.js +9 -22
- package/dist/release/snapshot.js.map +1 -1
- package/dist/release/stable.d.ts +15 -29
- package/dist/release/stable.js +14 -27
- package/dist/release/stable.js.map +1 -1
- package/dist/runs/auto-name.d.ts +1 -1
- package/dist/runs/auto-name.js +6 -1
- package/dist/runs/auto-name.js.map +1 -1
- package/dist/runs/retention.d.ts +1 -1
- package/dist/runs/store.d.ts +149 -336
- package/dist/runs/store.js +187 -9
- package/dist/runs/store.js.map +1 -1
- package/dist/runs/ui-event-sink.d.ts +1 -1
- package/dist/server/app-type.d.ts +21 -0
- package/dist/server/app-type.js +2 -0
- package/dist/server/app-type.js.map +1 -0
- package/dist/server/capabilities.d.ts +6 -5
- package/dist/server/capabilities.js +11 -0
- package/dist/server/capabilities.js.map +1 -1
- package/dist/server/forge/github.d.ts +58 -151
- package/dist/server/forge/github.js +530 -6
- package/dist/server/forge/github.js.map +1 -1
- package/dist/server/forge/index.d.ts +10 -3
- package/dist/server/forge/index.js +14 -1
- package/dist/server/forge/index.js.map +1 -1
- package/dist/server/forge/types.d.ts +86 -1
- package/dist/server/git-changes.d.ts +8 -8
- package/dist/server/git-changes.js +14 -17
- package/dist/server/git-changes.js.map +1 -1
- package/dist/server/git.d.ts +2 -0
- package/dist/server/git.js +9 -0
- package/dist/server/git.js.map +1 -1
- package/dist/server/github.d.ts +3 -3
- package/dist/server/github.js +1 -1
- package/dist/server/github.js.map +1 -1
- package/dist/server/open-in-terminal.d.ts +8 -1
- package/dist/server/open-in-terminal.js +16 -8
- package/dist/server/open-in-terminal.js.map +1 -1
- package/dist/server/pr.d.ts +2 -2
- package/dist/server/project-context.d.ts +20 -3
- package/dist/server/project-context.js +30 -1
- package/dist/server/project-context.js.map +1 -1
- package/dist/server/provider-action-gate.d.ts +8 -0
- package/dist/server/provider-action-gate.js +54 -0
- package/dist/server/provider-action-gate.js.map +1 -0
- package/dist/server/provider-auth-runtime.d.ts +21 -0
- package/dist/server/provider-auth-runtime.js +66 -0
- package/dist/server/provider-auth-runtime.js.map +1 -0
- package/dist/server/server.d.ts +12128 -47
- package/dist/server/server.js +2652 -917
- package/dist/server/server.js.map +1 -1
- package/dist/server/validators.d.ts +97 -0
- package/dist/server/validators.js +86 -0
- package/dist/server/validators.js.map +1 -0
- package/dist/server/ws.d.ts +2 -2
- package/dist/server/ws.js +2 -2
- package/dist/server/ws.js.map +1 -1
- package/dist/server-install/engine.d.ts +1 -1
- package/dist/server-install/platforms/macosx-ngrok.d.ts +1 -1
- package/dist/server-install/platforms/ubuntu-vps.d.ts +16 -1
- package/dist/server-install/platforms/ubuntu-vps.js +83 -1
- package/dist/server-install/platforms/ubuntu-vps.js.map +1 -1
- package/dist/server-install/state.d.ts +1 -1
- package/dist/server-install/steps.d.ts +2 -2
- package/dist/server-install/strategies.d.ts +1 -1
- package/dist/server-install/types.d.ts +38 -424
- package/dist/server-install/ui.d.ts +1 -1
- package/dist/skills-banner.d.ts +7 -6
- package/dist/skills-banner.js +7 -6
- package/dist/skills-banner.js.map +1 -1
- package/dist/skills-remote.d.ts +2 -2
- package/dist/skills-update.d.ts +88 -0
- package/dist/skills-update.js +392 -0
- package/dist/skills-update.js.map +1 -0
- package/dist/skills.d.ts +2 -0
- package/dist/skills.js +7 -2
- package/dist/skills.js.map +1 -1
- package/dist/todos.d.ts +2 -28
- package/dist/todos.js +1 -1
- package/dist/ui-state.d.ts +16 -2
- package/dist/ui-state.js +14 -1
- package/dist/ui-state.js.map +1 -1
- package/dist/workflows/load.d.ts +1 -1
- package/dist/workflows/run.d.ts +238 -12
- package/dist/workflows/run.js +1103 -115
- package/dist/workflows/run.js.map +1 -1
- package/dist/workflows/types.d.ts +59 -224
- package/dist/workflows/types.js +14 -0
- package/dist/workflows/types.js.map +1 -1
- package/dist/workspace/agent-accounts.d.ts +149 -0
- package/dist/workspace/agent-accounts.js +303 -0
- package/dist/workspace/agent-accounts.js.map +1 -0
- package/dist/workspace/agent-profiles.d.ts +78 -0
- package/dist/workspace/agent-profiles.js +115 -0
- package/dist/workspace/agent-profiles.js.map +1 -0
- package/dist/workspace/config.d.ts +82 -206
- package/dist/workspace/config.js +183 -14
- package/dist/workspace/config.js.map +1 -1
- package/dist/workspace/migrations.d.ts +1 -1
- package/dist/workspace/migrations.js +19 -4
- package/dist/workspace/migrations.js.map +1 -1
- package/dist/workspace/projects.d.ts +8 -2
- package/dist/workspace/projects.js +9 -3
- package/dist/workspace/projects.js.map +1 -1
- package/dist/workspace/semaphore.d.ts +45 -0
- package/dist/workspace/semaphore.js +45 -1
- package/dist/workspace/semaphore.js.map +1 -1
- package/dist/workspace/ui-state.d.ts +13 -3
- package/dist/workspace/ui-state.js +15 -5
- package/dist/workspace/ui-state.js.map +1 -1
- package/package.json +17 -39
- package/scripts/inline-contract.mjs +112 -0
- package/scripts/mock-claude.mjs +59 -1
- package/scripts/sync-readme.mjs +20 -0
- package/web/dist/assets/alert-dialog-LMDsWZ8t.js +1 -0
- package/web/dist/assets/arrow-down-CDJD8f8N.js +1 -0
- package/web/dist/assets/arrow-left-DUirK4gC.js +1 -0
- package/web/dist/assets/bundle-mjs-BT31bpU6.js +1 -0
- package/web/dist/assets/centered-state-CX0EhxaQ.js +43 -0
- package/web/dist/assets/chevron-right-B70aoBYh.js +1 -0
- package/web/dist/assets/{chunk-BO2N2NFS-DtrdTCWa.js → chunk-BO2N2NFS-DE6qKn3r.js} +8 -8
- package/web/dist/assets/circle-check-C6IhUojk.js +1 -0
- package/web/dist/assets/circle-x-hV9f6Ypp.js +1 -0
- package/web/dist/assets/collapsible-D2igywSh.js +1 -0
- package/web/dist/assets/commit-list-ocsM8XZs.js +1 -0
- package/web/dist/assets/compare-variants-CYdYsQBv.js +1 -0
- package/web/dist/assets/{core-DePtBHcl.js → core-BCsw8oQw.js} +1 -1
- package/web/dist/assets/diff-3y-HnHEH.js +3 -0
- package/web/dist/assets/diff-stat-OeTa1MCD.js +1 -0
- package/web/dist/assets/{diff-view-DCeSegKk.js → diff-view-D8VBKGxl.js} +4 -4
- package/web/dist/assets/dropdown-menu-B9UHiJFv.js +1 -0
- package/web/dist/assets/editable-title-C9aK-GCt.js +1 -0
- package/web/dist/assets/{ellipsis-vertical-CNnSLn5a.js → ellipsis-vertical-BVp0AyW4.js} +1 -1
- package/web/dist/assets/{file-D4GBh1ao.js → file-D1v6Cb9Q.js} +1 -1
- package/web/dist/assets/folder-HNaOtP70.js +1 -0
- package/web/dist/assets/{git-pull-request-DnR8Xh4_.js → git-pull-request-C6KuSF45.js} +1 -1
- package/web/dist/assets/git-toolbar-DpRPWLaM.js +1 -0
- package/web/dist/assets/github-UumAmuAM.js +1 -0
- package/web/dist/assets/highlighted-body-OFNGDK62-BL-MfFgl.js +1 -0
- package/web/dist/assets/highlighter-CdHqIwFr.js +3 -0
- package/web/dist/assets/image-preview-M1y8lBCU.js +1 -0
- package/web/dist/assets/index-BNgqhk51.js +6 -0
- package/web/dist/assets/index-De7KdVHL.css +2 -0
- package/web/dist/assets/lib-BQXq3kEf.js +1 -0
- package/web/dist/assets/lib-BxQXEXDF.js +1 -0
- package/web/dist/assets/markdown-DfljSy6W.js +2 -0
- package/web/dist/assets/mermaid-GHXKKRXX-C4jxqUUb.js +1 -0
- package/web/dist/assets/new-task-form-CkgsW8GC.js +1 -0
- package/web/dist/assets/pill-DWy7Dbv3.js +1 -0
- package/web/dist/assets/project-router-BJR_8pfk.js +1 -0
- package/web/dist/assets/prompt-templates-CKbMbly-.js +15 -0
- package/web/dist/assets/react-runtime-CCIEwYL0.js +9 -0
- package/web/dist/assets/{refresh-cw-BT8E96Tm.js → refresh-cw-Bs4wU0JM.js} +1 -1
- package/web/dist/assets/repo-git-CBbpvexr.js +1 -0
- package/web/dist/assets/rolldown-runtime-QTnfLwEv.js +1 -0
- package/web/dist/assets/run-diff-CzrO-R7P.js +3 -0
- package/web/dist/assets/run-header-BQDPFKUM.js +1 -0
- package/web/dist/assets/{search-x-BDjaMrzG.js → search-x-sxnYXz0H.js} +1 -1
- package/web/dist/assets/skill-empty-hint-CMfCEn9L.js +1 -0
- package/web/dist/assets/skills-BiWfHmCR.js +1 -0
- package/web/dist/assets/skills-DrG0KcAJ.js +1 -0
- package/web/dist/assets/sparkles-CL1cbhb_.js +1 -0
- package/web/dist/assets/{square-terminal-Ckjjn0vb.js → square-terminal-5JEPtGG-.js} +1 -1
- package/web/dist/assets/tab-link-DVT33Q2m.js +1 -0
- package/web/dist/assets/task-changes-BuuNjQ_C.js +1 -0
- package/web/dist/assets/task-commits-CVjZWXuP.js +1 -0
- package/web/dist/assets/task-files-f7USaQD1.js +2 -0
- package/web/dist/assets/task-thread-Cc7rMyer.js +9 -0
- package/web/dist/assets/textarea-BieGhkR-.js +1 -0
- package/web/dist/assets/thread-loading-CwfY_3r7.js +1 -0
- package/web/dist/assets/{trash-2-DAyZ0VSi.js → trash-2-Cmbuh3Q9.js} +1 -1
- package/web/dist/assets/{triangle-alert-Uy8rokUh.js → triangle-alert-dswtPk4F.js} +1 -1
- package/web/dist/assets/{upload-CDdyPPo_.js → upload-BY1qMAxW.js} +1 -1
- package/web/dist/assets/use-desktop-DrxT9b31.js +1 -0
- package/web/dist/assets/use-submit-shortcut-CULpfw9y.js +1 -0
- package/web/dist/assets/utils-C5bzdO9U.js +64 -0
- package/web/dist/assets/workflows-D6GsX0A9.js +11 -0
- package/web/dist/assets/zoomable-image-CEjRqsuW.js +1 -0
- package/web/dist/index.html +33 -13
- package/scripts/dev.mjs +0 -74
- package/scripts/release-snapshot.mjs +0 -130
- package/scripts/release.mjs +0 -119
- package/web/dist/assets/arrow-left-CnSdp92h.js +0 -1
- package/web/dist/assets/bundle-mjs-BegPhL5c.js +0 -1
- package/web/dist/assets/centered-state-DOsTcJZF.js +0 -43
- package/web/dist/assets/commit-list-BrxBEzRK.js +0 -1
- package/web/dist/assets/compare-variants-Bv2c9ORF.js +0 -1
- package/web/dist/assets/dist-B6XQOhgM.js +0 -1
- package/web/dist/assets/git-toolbar-pIC1ubPM.js +0 -1
- package/web/dist/assets/github-DTTSG0ht.js +0 -1
- package/web/dist/assets/highlighted-body-OFNGDK62-BuFJwunK.js +0 -1
- package/web/dist/assets/image-preview-DEhW0TFz.js +0 -1
- package/web/dist/assets/index-Cijmlz7q.js +0 -30
- package/web/dist/assets/index-DzroxqBG.css +0 -2
- package/web/dist/assets/lib-DPEZDBUN.js +0 -1
- package/web/dist/assets/lib-uHpeUPzK.js +0 -1
- package/web/dist/assets/mermaid-GHXKKRXX-CN2zQue0.js +0 -1
- package/web/dist/assets/open-mercato-toBr6SOa.svg +0 -11
- package/web/dist/assets/project-router-BMx4e8m-.js +0 -1
- package/web/dist/assets/repo-git-BusYfa3t.js +0 -1
- package/web/dist/assets/run-diff-wOJ3odA-.js +0 -3
- package/web/dist/assets/run-header-Cg5incil.js +0 -1
- package/web/dist/assets/skill-empty-hint-DTUSp6Vl.js +0 -1
- package/web/dist/assets/skills-PpP2owbM.js +0 -1
- package/web/dist/assets/tab-link-F3uQrdm2.js +0 -1
- package/web/dist/assets/task-changes-iD9kS79I.js +0 -1
- package/web/dist/assets/task-commits-COkLaUri.js +0 -1
- package/web/dist/assets/task-files-Ckttz7XH.js +0 -2
- package/web/dist/assets/task-thread-CQD91VPt.js +0 -9
- package/web/dist/assets/textarea-Bj_LCreg.js +0 -1
- package/web/dist/assets/use-desktop-DUh0STzq.js +0 -3
- package/web/dist/assets/utils-B8s5qIcK.js +0 -1
- package/web/dist/assets/workflows-BFk8C0co.js +0 -11
- package/web/dist/assets/zoomable-image-PEDOqyLQ.js +0 -1
- /package/web/{open-mercato.svg → dist/open-mercato.svg} +0 -0
package/dist/workflows/run.js
CHANGED
|
@@ -2,29 +2,36 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { spawn } from 'node:child_process';
|
|
3
3
|
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
|
-
import {
|
|
5
|
+
import { parseAskMarkerResult, stripAskMarker, } from '../core/ask.js';
|
|
6
6
|
import { onUsage, registerRunProcess, unregisterRunProcess } from '../core/process-usage.js';
|
|
7
|
+
import { parseUsageLimit } from '../core/usage-limit.js';
|
|
7
8
|
import { createRunner } from '../core/runner-factory.js';
|
|
8
9
|
import { modelConflictsWithRunner } from '../core/model-presets.js';
|
|
10
|
+
import { AGENT_MODELS_LOCKED_ERROR, agentModelsLocked } from '../core/agent-model-policy.js';
|
|
9
11
|
import { ModelIdentityError, formatModelIdentity, normalizeModelForBackend, } from '../core/model-identity.js';
|
|
10
12
|
import { HANDOFF_ONLY_INSTRUCTIONS, HANDOFF_INSTRUCTIONS, appendHandoffHeartbeat, followupsEnabled, handoffPath, seedHandoffFile, } from '../handoff.js';
|
|
11
13
|
import { todosPath } from '../todos.js';
|
|
12
14
|
import { discoverSkills } from '../skills.js';
|
|
13
15
|
import { materializeSkillDir } from '../skills-remote.js';
|
|
14
16
|
import { seedAgentConfigLocalLayer } from '../agent-config/seed.js';
|
|
17
|
+
import { readAgentModelProvider } from '../agent-config/models.js';
|
|
15
18
|
import { loadConfig, resolveWorktreeRetention } from '../config.js';
|
|
16
19
|
import { autosaveCommit, createWorktree, resolveBaseRef, worktreeDiff, worktreeShortstat } from '../git-worktree.js';
|
|
17
|
-
import { getRepoInfo } from '../server/git.js';
|
|
20
|
+
import { getHeadCommit, getRepoInfo } from '../server/git.js';
|
|
18
21
|
import { loadWorkflows } from './load.js';
|
|
19
22
|
import { reclaimWorktrees, rematerializeReclaimedWorktree } from '../runs/retention.js';
|
|
20
23
|
import { extractTaskRefs, refineTaskRefs, titleRefNumber } from '../runs/task-refs.js';
|
|
21
24
|
import { parseTaskMarkers, stripTaskMarkers } from '../runs/task-markers.js';
|
|
22
25
|
import { autoNamingActive, generateRunName, liveTitleUpdatesEnabled, postValidateTitle } from '../runs/auto-name.js';
|
|
23
26
|
import { reviewGateEnabled } from '../runs/review-gate.js';
|
|
27
|
+
import { resolveProfileEnvForRoot } from '../workspace/agent-profiles.js';
|
|
24
28
|
import { WorkspaceSemaphore } from '../workspace/semaphore.js';
|
|
25
29
|
import { UiEventSink } from '../runs/ui-event-sink.js';
|
|
26
30
|
import { chainStepNote, DEFAULT_ALLOWED_TOOLS, stepKind } from './types.js';
|
|
27
31
|
const CHECK_OUTPUT_CAP = 20_000;
|
|
32
|
+
async function configuredModelProvider(backend, repoRoot) {
|
|
33
|
+
return readAgentModelProvider(backend, repoRoot).catch(() => undefined);
|
|
34
|
+
}
|
|
28
35
|
/** An interactive session that hears nothing from the user closes itself. */
|
|
29
36
|
export const IDLE_TIMEOUT_MS = 15 * 60_000;
|
|
30
37
|
/**
|
|
@@ -77,6 +84,18 @@ function emitAskRequested(sink, ask) {
|
|
|
77
84
|
sink.handle({ type: 'ask.requested', requestId, questions: ask.questions });
|
|
78
85
|
return requestId;
|
|
79
86
|
}
|
|
87
|
+
/** A persisted, non-fatal explanation for protocol-shaped text that could not
|
|
88
|
+
* become an ask card. Never include the raw payload in this diagnostic. */
|
|
89
|
+
function askMarkerRejection(result) {
|
|
90
|
+
if (result.kind === 'invalid-json') {
|
|
91
|
+
return 'structured question ignored — CEZ:ASK payload is not valid JSON';
|
|
92
|
+
}
|
|
93
|
+
if (result.kind !== 'invalid-structure')
|
|
94
|
+
return undefined;
|
|
95
|
+
const issue = result.issues[0];
|
|
96
|
+
const location = issue?.path.length ? ` at ${issue.path.join('.')}` : '';
|
|
97
|
+
return `structured question ignored — CEZ:ASK payload failed validation${location}${issue ? `: ${issue.message}` : ''}`;
|
|
98
|
+
}
|
|
80
99
|
/** Periodic "cezar autosave" commit in the task worktree (spec 006). */
|
|
81
100
|
export const AUTOSAVE_INTERVAL_MS = 90_000;
|
|
82
101
|
/** The periodic autosave timer is opt-in (#471): off, a task branch carries only the
|
|
@@ -89,6 +108,96 @@ export function periodicAutosaveEnabled(env = process.env) {
|
|
|
89
108
|
/** Safety cap on autonomous auto-continues per run — stops a stuck agent from nudging forever. */
|
|
90
109
|
const MAX_AUTO_CONTINUES = 40;
|
|
91
110
|
const AUTONOMOUS_NUDGE = 'Continue working autonomously until the task is fully complete. Do not ask me for confirmation or clarification — make reasonable assumptions and proceed. When everything is done, end the session with your done signal.';
|
|
111
|
+
const MONITORING_WAKE_NUDGE = 'Re-check the downstream work you were monitoring. Continue toward the task goal; emit CEZ:MONITORING again only if it is still pending.';
|
|
112
|
+
/**
|
|
113
|
+
* Auto-resume after a provider usage limit (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
114
|
+
*
|
|
115
|
+
* The wait is the provider's own reset instant plus this grace: resuming AT the boundary races the
|
|
116
|
+
* provider's clock (and its rounding), and one failed resume costs the whole window over again.
|
|
117
|
+
* Thirty seconds is cheap next to five hours and long enough to be past any sane skew.
|
|
118
|
+
*/
|
|
119
|
+
export const AUTO_RESUME_GRACE_MS = 30_000;
|
|
120
|
+
/**
|
|
121
|
+
* Consecutive automatic resumes allowed without a human turn. A resume can only fire after a real
|
|
122
|
+
* reset instant, so this is not a throttle — it is the backstop for the pathological case (a
|
|
123
|
+
* provider that answers "limit reached, retry now" in a loop), and it is deliberately generous
|
|
124
|
+
* enough to sit through a couple of days of five-hour windows.
|
|
125
|
+
*/
|
|
126
|
+
export const MAX_AUTO_RESUMES = 12;
|
|
127
|
+
/**
|
|
128
|
+
* How long a missed deadline stays worth acting on. The promise is "we pick this up when the
|
|
129
|
+
* window reopens" — kept across a restart or an overnight close, which is the case the feature
|
|
130
|
+
* exists for. A day later it is no longer that promise: the user has moved on, and a task
|
|
131
|
+
* springing back to life is a surprise rather than a service. Such a deadline is retired with a
|
|
132
|
+
* note instead of fired, so the only tasks a sweep can revive are ones someone is still waiting on.
|
|
133
|
+
*/
|
|
134
|
+
export const AUTO_RESUME_MISSED_WINDOW_MS = 24 * 60 * 60_000;
|
|
135
|
+
/**
|
|
136
|
+
* How often the queue checks that it is not wedged.
|
|
137
|
+
*
|
|
138
|
+
* A hold is the only thing in the engine that can make an idle queue CORRECT, so it is also the
|
|
139
|
+
* only thing that can make a wedged one look correct. This tick is the way out: cheap (a few
|
|
140
|
+
* in-memory checks), unref'd, and it only ever acts when idling has no justification left.
|
|
141
|
+
*/
|
|
142
|
+
export const QUEUE_WATCHDOG_MS = 60_000;
|
|
143
|
+
/** Shared empty holds for the common "nothing is held" pump — avoids allocating per sweep. */
|
|
144
|
+
const NO_HOLDS = { deadline: new Set(), inFlight: new Set() };
|
|
145
|
+
/**
|
|
146
|
+
* May this run start, given what its account is holding?
|
|
147
|
+
*
|
|
148
|
+
* The two kinds of hold bind different work, and getting that wrong has produced a bug in each
|
|
149
|
+
* direction (spec 2026-08-03-auto-resume-after-usage-limit):
|
|
150
|
+
*
|
|
151
|
+
* - a `deadline` hold means the window is KNOWN shut until an instant, so it blocks everything
|
|
152
|
+
* on that account — resumes included. Exempting them let four resumes fire at once and
|
|
153
|
+
* re-limit one after another, which is the stampede wearing a different hat.
|
|
154
|
+
* - an `inFlight` hold means a resume is testing the window right now and nothing is proven, so
|
|
155
|
+
* it blocks fresh work but not other resumes. Blocking those deadlocked a live workspace.
|
|
156
|
+
*/
|
|
157
|
+
function accountHeldFor(run, holds, fallbackRunner) {
|
|
158
|
+
const key = runAccountKey(run, fallbackRunner);
|
|
159
|
+
if (holds.deadline.has(key))
|
|
160
|
+
return true;
|
|
161
|
+
return holds.inFlight.has(key) && !resumeInFlight(run);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Which agent ACCOUNT a run's work runs on — the thing a provider usage limit actually closes
|
|
165
|
+
* (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
166
|
+
*
|
|
167
|
+
* Backend plus agent account, because those are the two axes a limit is scoped to: a Claude
|
|
168
|
+
* limit must never stall a Codex task, and a second Claude login is a second budget. A record
|
|
169
|
+
* that names no runner has not started yet and will take the configured default, which is what
|
|
170
|
+
* `fallbackRunner` carries; a run that HAS started always carries its resolved runner (execute
|
|
171
|
+
* persists it), and only started runs can be holding.
|
|
172
|
+
*/
|
|
173
|
+
export function runAccountKey(run, fallbackRunner) {
|
|
174
|
+
return `${run.runner ?? fallbackRunner}:${run.agentProfile ?? 'default'}`;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Is this run an automatic resume that has not completed a turn yet?
|
|
178
|
+
*
|
|
179
|
+
* Such a run is the work the reopened window is FOR, so the hold must never apply to it — not
|
|
180
|
+
* its own, and not another resume's. Two resumes that hold each other is a deadlock the queue
|
|
181
|
+
* cannot recover from: both sit `queued` with a counter and no deadline, each waiting for the
|
|
182
|
+
* other to prove a window neither will ever get to test. That is the shape a live run produced
|
|
183
|
+
* — two scheduled tasks fired, both went `queued`, and nothing in the workspace moved again.
|
|
184
|
+
*
|
|
185
|
+
* The hold exists to stop NEW work walking into a closed window. A resume is not new work.
|
|
186
|
+
*/
|
|
187
|
+
function resumeInFlight(run) {
|
|
188
|
+
return (run.autoResumeAttempts !== undefined && (run.status === 'queued' || run.status === 'running'));
|
|
189
|
+
}
|
|
190
|
+
const AUTO_RESUME_PROMPT = 'The provider usage limit that interrupted this task has reset. Read the handoff file (CEZ_HANDOFF_FILE) to recover context, then continue the task from where you left off.';
|
|
191
|
+
/**
|
|
192
|
+
* The wake instant as a human reads it — local, to the SECOND, with the zone named. The
|
|
193
|
+
* transcript line is what someone scanning a stalled task actually reads, and "18:41" is not
|
|
194
|
+
* enough to tell a wait that is nearly over from one that just started; the machine-readable ISO
|
|
195
|
+
* copy lives on `RunRecord.autoResumeAt`. Server-side formatting is honest here because cezar is
|
|
196
|
+
* local-first: the process and the browser reading it are the same machine.
|
|
197
|
+
*/
|
|
198
|
+
function formatWakeInstant(at) {
|
|
199
|
+
return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'long' }).format(at);
|
|
200
|
+
}
|
|
92
201
|
/**
|
|
93
202
|
* The effective "extra" system prompt for a run (spec §protocol v2, R2 2.3):
|
|
94
203
|
* the per-run override (`POST /api/runs` `systemPrompt`) REPLACES the
|
|
@@ -171,6 +280,7 @@ const VARIANT_HINTS = {
|
|
|
171
280
|
B: 'Approach hint: prefer the minimal, surgical change.',
|
|
172
281
|
C: 'Approach hint: prefer a thorough, structural approach.',
|
|
173
282
|
};
|
|
283
|
+
const RESTART_CONTINUATION_PROMPT = 'The cezar process restarted while you were working on this task. Read the handoff file (CEZ_HANDOFF_FILE) to recover context, then continue the task from where you left off.';
|
|
174
284
|
/**
|
|
175
285
|
* The mini workflow engine: executes a `WorkflowDef` against a repo, one step
|
|
176
286
|
* at a time, persisting every event to the RunStore (which the SSE endpoints
|
|
@@ -200,7 +310,13 @@ export class RunManager {
|
|
|
200
310
|
// timeout already bounds how long a session can sit open. Invariant:
|
|
201
311
|
// `waiting ⊆ active` — always cleared together via dropActive().
|
|
202
312
|
waiting = new Set();
|
|
313
|
+
/** Durable monitoring subset. Only the configured number receives the waiting-slot exemption. */
|
|
314
|
+
monitoring = new Set();
|
|
203
315
|
pendingJobs = new Map();
|
|
316
|
+
/** Interrupted agent turns recovered after a process restart. Unlike an
|
|
317
|
+
* explicit user Continue, these are bulk scheduler work and must re-enter
|
|
318
|
+
* through `pump()` so both workspace and per-project caps are honored. */
|
|
319
|
+
pendingContinuations = new Map();
|
|
204
320
|
/** Per-run image counter behind `pasted-<n>` / `screenshot-<n>` (#472). Lives on
|
|
205
321
|
* the manager rather than the `ActiveRun` so a *queued* run — which has no
|
|
206
322
|
* `ActiveRun` at all — can persist attachments. Seeded lazily from disk. */
|
|
@@ -208,6 +324,12 @@ export class RunManager {
|
|
|
208
324
|
/** Messages that landed in the dequeue → session-open gap (#472), flushed as
|
|
209
325
|
* ordinary follow-up turns the moment the session opens. In-memory only. */
|
|
210
326
|
deferredMessages = new Map();
|
|
327
|
+
/** Armed usage-limit resumes, keyed by run id (spec
|
|
328
|
+
* 2026-08-03-auto-resume-after-usage-limit). The DEADLINE itself lives on the record
|
|
329
|
+
* (`autoResumeAt`) — this map holds only the process-local timer, so a restart rebuilds it
|
|
330
|
+
* from the record rather than losing the wait. Runs here are `failed` and therefore NOT in
|
|
331
|
+
* `active`, which is why the timer cannot live on an `ActiveRun` like the monitoring one. */
|
|
332
|
+
autoResumeTimers = new Map();
|
|
211
333
|
pumping = false;
|
|
212
334
|
/** A pump that arrived while one was in flight — replayed by `pump()`'s own
|
|
213
335
|
* loop so a slot freed mid-sweep is never a lost wakeup. */
|
|
@@ -226,6 +348,13 @@ export class RunManager {
|
|
|
226
348
|
/** Unsubscribe handle for the constructor's `onUsage` subscription — released
|
|
227
349
|
* by dispose() so a torn-down manager stops receiving sampler ticks. */
|
|
228
350
|
offUsage;
|
|
351
|
+
/** The stalled-queue watchdog (see `rescueStalledQueue`). */
|
|
352
|
+
queueWatchdog;
|
|
353
|
+
/** Set by the watchdog for exactly one sweep: ignore the usage-limit hold and make progress. */
|
|
354
|
+
forceNextPump = false;
|
|
355
|
+
/** Runs the watchdog started despite the hold. The spawn-time gate (`requeueWhileHeld`) would
|
|
356
|
+
* otherwise hand them straight back and the rescue would undo itself in a millisecond. */
|
|
357
|
+
forceStarted = new Set();
|
|
229
358
|
/** The workspace-wide parallel-cap semaphore + cached resource config
|
|
230
359
|
* (spec 2026-07-20, step 2.5). Boot constructs ONE and every manager shares
|
|
231
360
|
* it; the private fallback keeps single-manager callers and tests working. */
|
|
@@ -242,10 +371,13 @@ export class RunManager {
|
|
|
242
371
|
busySlots: () => this.busySlots(),
|
|
243
372
|
pump: () => this.pump(),
|
|
244
373
|
oldestQueuedAt: () => this.oldestQueuedAt(),
|
|
374
|
+
accountHolds: () => this.accountHolds(),
|
|
245
375
|
});
|
|
246
376
|
// Memory guard (#memory-guard): the shared process-tree sampler already ticks ~every 2 s for
|
|
247
377
|
// the runs table; piggyback on it to enforce the per-task memory ceiling.
|
|
248
378
|
this.offUsage = onUsage((snapshot) => void this.enforceMemoryLimit(snapshot));
|
|
379
|
+
this.queueWatchdog = setInterval(() => void this.rescueStalledQueue(), QUEUE_WATCHDOG_MS);
|
|
380
|
+
this.queueWatchdog.unref?.();
|
|
249
381
|
}
|
|
250
382
|
/**
|
|
251
383
|
* Release everything this manager owns without touching run records
|
|
@@ -260,17 +392,23 @@ export class RunManager {
|
|
|
260
392
|
dispose() {
|
|
261
393
|
this.offUsage();
|
|
262
394
|
this.offSemaphore();
|
|
263
|
-
|
|
395
|
+
clearInterval(this.queueWatchdog);
|
|
396
|
+
for (const [runId, state] of this.active) {
|
|
264
397
|
this.clearIdleTimer(state);
|
|
398
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
265
399
|
this.clearAutosaveTimer(state);
|
|
266
400
|
state.releaseRepoRoot?.();
|
|
267
401
|
state.releaseRepoRoot = undefined;
|
|
268
402
|
}
|
|
403
|
+
for (const timer of this.autoResumeTimers.values())
|
|
404
|
+
clearTimeout(timer);
|
|
405
|
+
this.autoResumeTimers.clear();
|
|
269
406
|
this.active.clear();
|
|
270
407
|
this.waiting.clear();
|
|
271
408
|
this.starting.clear();
|
|
272
409
|
this.queue.length = 0;
|
|
273
410
|
this.pendingJobs.clear();
|
|
411
|
+
this.pendingContinuations.clear();
|
|
274
412
|
this.memoryPausing.clear();
|
|
275
413
|
this.lastNamerKey.clear();
|
|
276
414
|
}
|
|
@@ -338,13 +476,49 @@ export class RunManager {
|
|
|
338
476
|
CEZ_TODOS_FILE: generateFollowups ? todosPath(this.dataDir) : '',
|
|
339
477
|
};
|
|
340
478
|
}
|
|
479
|
+
/**
|
|
480
|
+
* `agentEnv` plus the agent-account variable for the profile this STEP runs under (spec
|
|
481
|
+
* 2026-07-29-agent-profiles), and the id it resolved to so the caller can record it.
|
|
482
|
+
*
|
|
483
|
+
* Resolved per step, not per run, because a workflow can mix backends: an override naming a
|
|
484
|
+
* Claude account says nothing about which Codex account a codex step should use. Resolution
|
|
485
|
+
* order, most specific first:
|
|
486
|
+
*
|
|
487
|
+
* 1. the step's ALREADY-RECORDED `profileId` — a resume or Continue must reattach to the
|
|
488
|
+
* account that created the session, whatever the project has since been switched to;
|
|
489
|
+
* 2. the run's composer override, but only for steps on the run's own runner;
|
|
490
|
+
* 3. the project's stored selection, and failing that the discovered default.
|
|
491
|
+
*
|
|
492
|
+
* Read fresh every time. `~/.cezar/config.json` is shared by every cezar process on this
|
|
493
|
+
* machine, so a cached snapshot is a staleness bug, and one small JSON read is free next to
|
|
494
|
+
* spawning a CLI. Never throws: an unreadable home degrades to the default profile, which is
|
|
495
|
+
* exactly the behaviour that predates profiles.
|
|
496
|
+
*/
|
|
497
|
+
async agentEnvForStep(runId, backend, options = {}) {
|
|
498
|
+
const run = this.store.getRun(runId);
|
|
499
|
+
const profileId = options.recordedProfileId
|
|
500
|
+
?? (backend === (run?.runner ?? 'claude') ? run?.agentProfile : undefined);
|
|
501
|
+
const resolved = await resolveProfileEnvForRoot(this.repoRoot, backend, profileId);
|
|
502
|
+
return {
|
|
503
|
+
env: { ...this.agentEnv(runId, options.generateFollowups), ...resolved.env },
|
|
504
|
+
profileId: resolved.profile.id,
|
|
505
|
+
};
|
|
506
|
+
}
|
|
341
507
|
startRun(workflow, input, group) {
|
|
508
|
+
// Sanitize at the manager boundary so CLI runs, workflows, variants, and
|
|
509
|
+
// direct callers cannot bypass the HTTP policy.
|
|
510
|
+
const effectiveInput = agentModelsLocked(this.repoRoot)
|
|
511
|
+
? { ...input, model: undefined }
|
|
512
|
+
: input;
|
|
342
513
|
const run = this.store.createRun({
|
|
343
514
|
title: makeRunTitle(input.task, workflow) + (group ? ` (${group.variant})` : ''),
|
|
344
515
|
workflow: workflow.name,
|
|
345
516
|
task: input.task,
|
|
346
|
-
model:
|
|
517
|
+
model: effectiveInput.model,
|
|
347
518
|
runner: input.runner,
|
|
519
|
+
// The composer's per-task account (spec 2026-07-29-agent-profiles). Persisted at creation
|
|
520
|
+
// so a queued run picks it up at dequeue and every later resume reads the same answer.
|
|
521
|
+
agentProfile: input.agentProfile,
|
|
348
522
|
// The global inbox is the ceiling on the per-run flag (#471). Enforced here rather than
|
|
349
523
|
// at the HTTP route because `cezar run`, the inbox's own "▶ Run" and variants all reach
|
|
350
524
|
// startRun directly — a route-level gate would leave those writing todos.json.
|
|
@@ -354,6 +528,9 @@ export class RunManager {
|
|
|
354
528
|
// auto-nudge reads `input.autonomous` (`execute`), but the record is the
|
|
355
529
|
// only source those after-the-fact consumers have.
|
|
356
530
|
autonomous: input.autonomous === true,
|
|
531
|
+
// Persist the explicit opt-out so queued-run restart recovery and the
|
|
532
|
+
// session Git routes can distinguish it from a removed isolated worktree.
|
|
533
|
+
worktree: !group && input.worktree === false ? false : undefined,
|
|
357
534
|
groupId: group?.groupId,
|
|
358
535
|
variant: group?.variant,
|
|
359
536
|
steps: workflow.steps.map((s) => ({ id: s.id, name: s.name ?? s.id, kind: stepKind(s) })),
|
|
@@ -388,7 +565,7 @@ export class RunManager {
|
|
|
388
565
|
// above shows instantly; the namer's short title replaces it when (and if)
|
|
389
566
|
// the model answers. Never awaited, never fails the run.
|
|
390
567
|
void this.autoNameRun(run.id, skillHint, input.task);
|
|
391
|
-
this.pendingJobs.set(run.id, { workflow, input });
|
|
568
|
+
this.pendingJobs.set(run.id, { workflow, input: effectiveInput });
|
|
392
569
|
this.queue.push(run.id);
|
|
393
570
|
void this.pump();
|
|
394
571
|
return run;
|
|
@@ -405,7 +582,7 @@ export class RunManager {
|
|
|
405
582
|
return VARIANT_LETTERS.slice(0, Math.min(Math.max(count, 1), VARIANT_LETTERS.length)).map((variant) => {
|
|
406
583
|
const hint = VARIANT_HINTS[variant];
|
|
407
584
|
const task = hint ? `${input.task}\n\n${hint}` : input.task;
|
|
408
|
-
return this.startRun(workflow, { ...input, task }, { groupId, variant });
|
|
585
|
+
return this.startRun(workflow, { ...input, task, worktree: undefined }, { groupId, variant });
|
|
409
586
|
});
|
|
410
587
|
}
|
|
411
588
|
/**
|
|
@@ -418,7 +595,9 @@ export class RunManager {
|
|
|
418
595
|
* `maxParallel`, including when other projects saturate the cap.
|
|
419
596
|
*/
|
|
420
597
|
busySlots() {
|
|
421
|
-
|
|
598
|
+
const ordinaryWaiting = this.waiting.size - this.monitoring.size;
|
|
599
|
+
const exemptMonitoring = Math.min(this.monitoring.size, this.semaphore.maxMonitoringSessions());
|
|
600
|
+
return this.active.size + this.starting.size - ordinaryWaiting - exemptMonitoring;
|
|
422
601
|
}
|
|
423
602
|
/** Epoch ms of this manager's oldest queued run (the semaphore's fairness
|
|
424
603
|
* key when a freed slot is broadcast), or null when nothing is queued.
|
|
@@ -454,6 +633,8 @@ export class RunManager {
|
|
|
454
633
|
* root (spec 006 degradation rule), which is always the tighter bound.
|
|
455
634
|
*/
|
|
456
635
|
async pump() {
|
|
636
|
+
this.reconcileMonitoringWakeTimers();
|
|
637
|
+
this.reconcileAutoResumes();
|
|
457
638
|
// A pump requested while one is in flight can't just be dropped: the
|
|
458
639
|
// in-flight pass may already have read capacity (it awaits `getRepoInfo`
|
|
459
640
|
// before the first check), so a slot freed in that window would be lost
|
|
@@ -478,15 +659,69 @@ export class RunManager {
|
|
|
478
659
|
const capacity = () => this.semaphore.busy() < maxParallel &&
|
|
479
660
|
this.busySlots() < projectMax &&
|
|
480
661
|
(repo !== null || this.busySlots() < 1);
|
|
662
|
+
// The usage-limit hold (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
663
|
+
//
|
|
664
|
+
// A limit closes an ACCOUNT, not a run — so starting the next queued task walks it into
|
|
665
|
+
// the same wall. Measured before this gate existed: eight tasks under `maxParallel: 2`
|
|
666
|
+
// all failed within 517 ms, each spawning a CLI (and, outside worktree-opt-out mode, a
|
|
667
|
+
// worktree and a branch) only to be marked `scheduled`. The cap was respected at every
|
|
668
|
+
// instant and was no brake at all, because a doomed run lives ~200 ms.
|
|
669
|
+
//
|
|
670
|
+
// So: while any run on an account is waiting out a limit, nothing new starts on THAT
|
|
671
|
+
// account. Other accounts (a second login, a different backend) keep running — the hold
|
|
672
|
+
// is keyed, not global. The set is derived from the durable records rather than tracked
|
|
673
|
+
// separately, which is what makes it survive a restart, expire on its own, and lift the
|
|
674
|
+
// instant a user cancels a resume.
|
|
675
|
+
// The watchdog's one-shot override — read and cleared here, so a forced sweep never
|
|
676
|
+
// leaks into the next ordinary one.
|
|
677
|
+
const forced = this.forceNextPump;
|
|
678
|
+
this.forceNextPump = false;
|
|
679
|
+
const holds = this.queue.length > 0 && !forced ? this.semaphore.accountHolds() : NO_HOLDS;
|
|
680
|
+
const anyHold = holds.deadline.size > 0 || holds.inFlight.size > 0;
|
|
681
|
+
// Only pay for the config read when something is actually held: a queued record may name
|
|
682
|
+
// no runner, and then the account it would use is the configured default.
|
|
683
|
+
const defaultRunner = anyHold ? (await loadConfig(this.repoRoot)).defaultRunner : undefined;
|
|
481
684
|
while (this.queue.length > 0 && capacity()) {
|
|
482
|
-
|
|
685
|
+
// FIFO among the runs that CAN start; a held one keeps its place in the queue rather
|
|
686
|
+
// than being dequeued and re-queued (which would churn its position and its record).
|
|
687
|
+
const next = !anyHold
|
|
688
|
+
? 0
|
|
689
|
+
: this.queue.findIndex((id) => {
|
|
690
|
+
const queued = this.store.getRun(id);
|
|
691
|
+
return !queued || !accountHeldFor(queued, holds, defaultRunner ?? 'claude');
|
|
692
|
+
});
|
|
693
|
+
if (next === -1)
|
|
694
|
+
break; // everything queued is waiting on a held account
|
|
695
|
+
const runId = this.queue.splice(next, 1)[0];
|
|
483
696
|
if (!runId)
|
|
484
697
|
break;
|
|
698
|
+
// A forced sweep has to reach the spawn: the gate inside `execute` asks the same
|
|
699
|
+
// question and would send this run straight back to the queue.
|
|
700
|
+
if (forced)
|
|
701
|
+
this.forceStarted.add(runId);
|
|
485
702
|
const job = this.pendingJobs.get(runId);
|
|
703
|
+
const continuation = this.pendingContinuations.get(runId);
|
|
486
704
|
this.pendingJobs.delete(runId);
|
|
487
|
-
|
|
705
|
+
this.pendingContinuations.delete(runId);
|
|
706
|
+
if (!job && !continuation)
|
|
488
707
|
continue;
|
|
489
708
|
this.starting.add(runId);
|
|
709
|
+
if (continuation) {
|
|
710
|
+
const hydrated = this.hydrateQueuedContinuation(runId, continuation);
|
|
711
|
+
void this.runContinuation(runId, hydrated.stepId, hydrated.sessionId, hydrated.backend, hydrated.prompt, hydrated.images, hydrated.persistedImages, hydrated.persistedAttachments).catch((err) => {
|
|
712
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
713
|
+
this.store.updateRun(runId, {
|
|
714
|
+
status: 'failed',
|
|
715
|
+
error: `continue crashed: ${message}`,
|
|
716
|
+
finishedAt: new Date().toISOString(),
|
|
717
|
+
});
|
|
718
|
+
this.starting.delete(runId);
|
|
719
|
+
this.dropActive(runId);
|
|
720
|
+
});
|
|
721
|
+
continue;
|
|
722
|
+
}
|
|
723
|
+
if (!job)
|
|
724
|
+
continue;
|
|
490
725
|
// Rebuild the prompt from the store at the last instant (#472), so an edit
|
|
491
726
|
// or a stacked message that landed while the run waited is honored. Entered
|
|
492
727
|
// in the same synchronous tick as the `pendingJobs.delete` above, so no
|
|
@@ -514,6 +749,87 @@ export class RunManager {
|
|
|
514
749
|
this.pumping = false;
|
|
515
750
|
}
|
|
516
751
|
}
|
|
752
|
+
/**
|
|
753
|
+
* Make one `queued` RECORD executable again — the engine half a queued run needs but does not
|
|
754
|
+
* persist (`pendingJobs` / `pendingContinuations` are process-local, the record is not).
|
|
755
|
+
*
|
|
756
|
+
* Two callers, one path: boot recovery re-adopts everything the previous process was holding,
|
|
757
|
+
* and the queue watchdog re-adopts anything the running process has somehow lost. A queued
|
|
758
|
+
* record with no work item behind it is invisible to `pump()` and would sit there for good,
|
|
759
|
+
* which is the worst failure this engine has — the task is neither running nor failed, just
|
|
760
|
+
* silently never going to happen.
|
|
761
|
+
*
|
|
762
|
+
* A continuation is reconstructed first: its executable details are gone, but the pending
|
|
763
|
+
* `continue-N` step and the session before it are durable, which is enough. Otherwise the
|
|
764
|
+
* workflow is revived from the record. A run that can be neither is failed loudly rather than
|
|
765
|
+
* left in the queue as a ghost.
|
|
766
|
+
*/
|
|
767
|
+
async reviveQueuedRun(run, reason) {
|
|
768
|
+
const queuedContinuation = [...run.steps]
|
|
769
|
+
.reverse()
|
|
770
|
+
.find((step) => step.status === 'pending' && step.id.startsWith('continue-'));
|
|
771
|
+
const sessionStep = queuedContinuation
|
|
772
|
+
? [...run.steps].reverse().find((step) => step.id !== queuedContinuation.id && step.sessionId)
|
|
773
|
+
: undefined;
|
|
774
|
+
if (queuedContinuation && sessionStep?.sessionId) {
|
|
775
|
+
const backend = run.runner ?? 'claude';
|
|
776
|
+
const sessionBackend = sessionStep.backend ?? backend;
|
|
777
|
+
this.pendingContinuations.set(run.id, {
|
|
778
|
+
stepId: queuedContinuation.id,
|
|
779
|
+
sessionId: sessionBackend === backend ? sessionStep.sessionId : undefined,
|
|
780
|
+
backend,
|
|
781
|
+
prompt: RESTART_CONTINUATION_PROMPT,
|
|
782
|
+
images: [],
|
|
783
|
+
});
|
|
784
|
+
this.queue.push(run.id);
|
|
785
|
+
this.store.appendEvent(run.id, {
|
|
786
|
+
type: 'lifecycle',
|
|
787
|
+
message: `${reason} — interrupted continuation re-queued`,
|
|
788
|
+
});
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
const workflow = await this.reviveWorkflow(run);
|
|
792
|
+
if (!workflow) {
|
|
793
|
+
this.store.updateRun(run.id, {
|
|
794
|
+
status: 'failed',
|
|
795
|
+
error: 'interrupted — workflow definition not recoverable after a restart',
|
|
796
|
+
finishedAt: new Date().toISOString(),
|
|
797
|
+
});
|
|
798
|
+
this.store.appendEvent(run.id, {
|
|
799
|
+
type: 'lifecycle',
|
|
800
|
+
message: `${reason} — workflow definition not recoverable, task failed`,
|
|
801
|
+
});
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
804
|
+
// Re-apply the inbox ceiling (#471). `execute()` gates again at spawn time, so the agent is
|
|
805
|
+
// safe either way — but a run queued while the inbox was on and recovered after it was
|
|
806
|
+
// switched off would otherwise keep echoing `generateFollowups: true` on a run that
|
|
807
|
+
// demonstrably produced none. Normalize the record, the way startRun does.
|
|
808
|
+
const generateFollowups = followupsEnabled() ? run.generateFollowups : false;
|
|
809
|
+
if (generateFollowups !== run.generateFollowups) {
|
|
810
|
+
this.store.updateRun(run.id, { generateFollowups });
|
|
811
|
+
}
|
|
812
|
+
this.pendingJobs.set(run.id, {
|
|
813
|
+
workflow,
|
|
814
|
+
// Folded through the same helper `pump()` uses (#472) so a restart carries the stack.
|
|
815
|
+
// Idempotent: hydration always composes from `run.task` + the stack, never from an
|
|
816
|
+
// already-folded `input.task`, so re-hydrating at dequeue yields the same string.
|
|
817
|
+
input: this.hydrateQueuedInput(run.id, {
|
|
818
|
+
task: run.task,
|
|
819
|
+
model: run.model,
|
|
820
|
+
runner: run.runner,
|
|
821
|
+
generateFollowups,
|
|
822
|
+
// Re-thread autonomy (#489): the rebuilt input feeds `execute`, whose mid-run auto-nudge
|
|
823
|
+
// reads `input.autonomous`. Without this a recovered autonomous run would run
|
|
824
|
+
// non-autonomously and later wrongly park at `review`.
|
|
825
|
+
autonomous: run.autonomous,
|
|
826
|
+
// Preserve an explicit worktree opt-out across a queued restart.
|
|
827
|
+
worktree: run.worktree,
|
|
828
|
+
}),
|
|
829
|
+
});
|
|
830
|
+
this.queue.push(run.id);
|
|
831
|
+
this.store.appendEvent(run.id, { type: 'lifecycle', message: `${reason} — task re-queued` });
|
|
832
|
+
}
|
|
517
833
|
/**
|
|
518
834
|
* Startup recovery (#367) — re-adopt runs that were live when the previous
|
|
519
835
|
* cezar process exited (requires the store opened with `keepLive`):
|
|
@@ -532,49 +848,7 @@ export class RunManager {
|
|
|
532
848
|
.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
533
849
|
for (const run of live) {
|
|
534
850
|
if (run.status === 'queued') {
|
|
535
|
-
|
|
536
|
-
if (workflow) {
|
|
537
|
-
// Re-apply the inbox ceiling (#471). `execute()` gates again at spawn time, so the
|
|
538
|
-
// agent is safe either way — but a run queued while the inbox was on and recovered
|
|
539
|
-
// after it was switched off would otherwise keep echoing `generateFollowups: true`
|
|
540
|
-
// on a run that demonstrably produced none. Normalize the record, the way startRun
|
|
541
|
-
// does, so the stored answer matches what actually happens.
|
|
542
|
-
const generateFollowups = followupsEnabled() ? run.generateFollowups : false;
|
|
543
|
-
if (generateFollowups !== run.generateFollowups) {
|
|
544
|
-
this.store.updateRun(run.id, { generateFollowups });
|
|
545
|
-
}
|
|
546
|
-
this.pendingJobs.set(run.id, {
|
|
547
|
-
workflow,
|
|
548
|
-
// Folded through the same helper `pump()` uses (#472) so a restart
|
|
549
|
-
// carries the stack. Idempotent: hydration always composes from
|
|
550
|
-
// `run.task` + the stack, never from an already-folded `input.task`,
|
|
551
|
-
// so re-hydrating at dequeue yields the same string, not a doubled one.
|
|
552
|
-
input: this.hydrateQueuedInput(run.id, {
|
|
553
|
-
task: run.task,
|
|
554
|
-
model: run.model,
|
|
555
|
-
runner: run.runner,
|
|
556
|
-
generateFollowups,
|
|
557
|
-
// Re-thread autonomy (#489): the rebuilt input feeds `execute`,
|
|
558
|
-
// whose mid-run auto-nudge reads `input.autonomous`. Without this a
|
|
559
|
-
// recovered queued autonomous run would run non-autonomously (no
|
|
560
|
-
// auto-nudge) and later wrongly park at `review`.
|
|
561
|
-
autonomous: run.autonomous,
|
|
562
|
-
}),
|
|
563
|
-
});
|
|
564
|
-
this.queue.push(run.id);
|
|
565
|
-
this.store.appendEvent(run.id, { type: 'lifecycle', message: 'cezar restarted — task re-queued' });
|
|
566
|
-
}
|
|
567
|
-
else {
|
|
568
|
-
this.store.updateRun(run.id, {
|
|
569
|
-
status: 'failed',
|
|
570
|
-
error: 'interrupted — workflow definition not recoverable after a restart',
|
|
571
|
-
finishedAt: new Date().toISOString(),
|
|
572
|
-
});
|
|
573
|
-
this.store.appendEvent(run.id, {
|
|
574
|
-
type: 'lifecycle',
|
|
575
|
-
message: 'cezar restarted — workflow definition not recoverable, task failed',
|
|
576
|
-
});
|
|
577
|
-
}
|
|
851
|
+
await this.reviveQueuedRun(run, 'cezar restarted');
|
|
578
852
|
continue;
|
|
579
853
|
}
|
|
580
854
|
if (run.status === 'waiting') {
|
|
@@ -605,8 +879,8 @@ export class RunManager {
|
|
|
605
879
|
currentStepId: undefined,
|
|
606
880
|
});
|
|
607
881
|
const resumed = this.continueRun(run.id, {
|
|
608
|
-
text:
|
|
609
|
-
});
|
|
882
|
+
text: RESTART_CONTINUATION_PROMPT,
|
|
883
|
+
}, true);
|
|
610
884
|
this.store.appendEvent(run.id, {
|
|
611
885
|
type: 'lifecycle',
|
|
612
886
|
message: resumed.ok
|
|
@@ -614,14 +888,21 @@ export class RunManager {
|
|
|
614
888
|
: `cezar restarted — could not resume the interrupted task (${resumed.error ?? 'unknown'})`,
|
|
615
889
|
});
|
|
616
890
|
}
|
|
891
|
+
// Re-arm usage-limit resumes (spec 2026-08-03-auto-resume-after-usage-limit): the wait is
|
|
892
|
+
// routinely longer than a cezar session, so the deadline is durable and the timer is rebuilt
|
|
893
|
+
// from it. `pump()` reconciles again on every sweep, so this is the fast path, not the only
|
|
894
|
+
// one — see `reconcileAutoResumes`.
|
|
895
|
+
this.reconcileAutoResumes();
|
|
617
896
|
void this.pump();
|
|
618
897
|
}
|
|
619
898
|
/** The persisted definition when it looks sane, else the catalog by name. */
|
|
620
899
|
async reviveWorkflow(run) {
|
|
900
|
+
// "Looks sane" is the STORE's job now: it parses `workflowDef` against the definition schema
|
|
901
|
+
// and `.catch`es a def that no longer fits to `undefined`, so anything present here already
|
|
902
|
+
// has the `steps` array the old inline `Array.isArray` check was asking for.
|
|
621
903
|
const def = run.workflowDef;
|
|
622
|
-
if (def
|
|
904
|
+
if (def)
|
|
623
905
|
return def;
|
|
624
|
-
}
|
|
625
906
|
const { workflows } = await loadWorkflows(this.repoRoot);
|
|
626
907
|
return workflows.find((w) => w.name === run.workflow) ?? null;
|
|
627
908
|
}
|
|
@@ -632,12 +913,26 @@ export class RunManager {
|
|
|
632
913
|
if (state)
|
|
633
914
|
state.releaseRepoRoot = undefined;
|
|
634
915
|
this.waiting.delete(runId);
|
|
916
|
+
this.monitoring.delete(runId);
|
|
917
|
+
if (state)
|
|
918
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
635
919
|
this.active.delete(runId);
|
|
636
920
|
this.memoryPausing.delete(runId);
|
|
637
921
|
this.lastNamerKey.delete(runId);
|
|
922
|
+
this.forceStarted.delete(runId);
|
|
638
923
|
// The run's slot is gone from busySlots() as of the deletes above — hand it
|
|
639
924
|
// to the workspace's oldest queued run, in ANY project. Every terminal path
|
|
640
925
|
// funnels through here, so this one call covers them all.
|
|
926
|
+
// Same reasoning as retention below — every terminal path funnels through here, so the
|
|
927
|
+
// usage-limit question ("did this run stop because the account is out of window, and when
|
|
928
|
+
// does that window reopen?") is asked once, in one place, off the record the failing path
|
|
929
|
+
// has already written. Nothing to do for any other outcome.
|
|
930
|
+
//
|
|
931
|
+
// BEFORE releasing the slot, and that order is the whole point: `releaseSlot` pumps every
|
|
932
|
+
// manager, and a pump reads the hold off the records. Publishing the schedule afterwards
|
|
933
|
+
// left a window — measured as exactly one extra task — where the queue saw a free slot and
|
|
934
|
+
// an account that looked healthy, and started work that was already doomed.
|
|
935
|
+
this.scheduleAutoResumeIfLimited(runId);
|
|
641
936
|
this.releaseSlot();
|
|
642
937
|
// A run leaving the active registry is a terminal transition (done/review/
|
|
643
938
|
// failed/cancelled) — the one moment the finished-worktree count can grow.
|
|
@@ -646,6 +941,333 @@ export class RunManager {
|
|
|
646
941
|
// the lifecycle.
|
|
647
942
|
void this.enforceRetention();
|
|
648
943
|
}
|
|
944
|
+
// ---- usage-limit auto-resume (spec 2026-08-03-auto-resume-after-usage-limit) --------------
|
|
945
|
+
/**
|
|
946
|
+
* A run just failed: if the provider said "usage limit, back at T", promise to resume it at
|
|
947
|
+
* `T + AUTO_RESUME_GRACE_MS` instead of leaving the task dead until someone notices.
|
|
948
|
+
*
|
|
949
|
+
* Every refusal below is silent-but-honest — the run stays `failed` with its Continue button,
|
|
950
|
+
* which is exactly the pre-feature behavior — except the safety cap, which says so on the
|
|
951
|
+
* transcript, because a run that stops resuming itself needs to explain why.
|
|
952
|
+
*/
|
|
953
|
+
scheduleAutoResumeIfLimited(runId) {
|
|
954
|
+
if (this.autoResumeTimers.has(runId))
|
|
955
|
+
return; // already promised
|
|
956
|
+
const run = this.store.getRun(runId);
|
|
957
|
+
if (!run || run.status !== 'failed')
|
|
958
|
+
return;
|
|
959
|
+
// Archiving IS resigning from a task. Reviving one because a window happened to reopen would
|
|
960
|
+
// be the feature working against the clearest signal the user can give it.
|
|
961
|
+
if (run.archived)
|
|
962
|
+
return;
|
|
963
|
+
const limit = parseUsageLimit(run.error);
|
|
964
|
+
if (!limit)
|
|
965
|
+
return;
|
|
966
|
+
if (!this.semaphore.autoResumeOnUsageLimit())
|
|
967
|
+
return;
|
|
968
|
+
// No session to resume = nothing this feature can do; `continueRun` would refuse anyway.
|
|
969
|
+
if (!run.steps.some((step) => step.sessionId))
|
|
970
|
+
return;
|
|
971
|
+
const attempts = run.autoResumeAttempts ?? 0;
|
|
972
|
+
if (attempts >= MAX_AUTO_RESUMES) {
|
|
973
|
+
this.store.appendEvent(runId, {
|
|
974
|
+
type: 'note',
|
|
975
|
+
message: `automatic resume cap reached (${MAX_AUTO_RESUMES}) — continue this task manually`,
|
|
976
|
+
});
|
|
977
|
+
return;
|
|
978
|
+
}
|
|
979
|
+
const wakeAt = new Date(limit.resetAt.getTime() + AUTO_RESUME_GRACE_MS);
|
|
980
|
+
this.armAutoResume(runId, wakeAt.getTime());
|
|
981
|
+
this.store.appendEvent(runId, {
|
|
982
|
+
type: 'lifecycle',
|
|
983
|
+
message: `usage limit reached — resuming automatically at ${formatWakeInstant(wakeAt)}`,
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
/** Publish the deadline on the record (the cockpit's only source) and arm the timer for it. */
|
|
987
|
+
armAutoResume(runId, deadline) {
|
|
988
|
+
this.store.updateRun(runId, { autoResumeAt: new Date(deadline).toISOString() });
|
|
989
|
+
const timer = setTimeout(() => this.fireAutoResume(runId), Math.max(0, deadline - Date.now()));
|
|
990
|
+
timer.unref?.();
|
|
991
|
+
this.autoResumeTimers.set(runId, timer);
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* The window has reopened. Re-check the record synchronously — hours may have passed, and the
|
|
995
|
+
* user may have continued, deleted or cancelled the run in them — then hand the resume to the
|
|
996
|
+
* ordinary queued-continuation path so it obeys both concurrency caps like any other work.
|
|
997
|
+
*/
|
|
998
|
+
fireAutoResume(runId) {
|
|
999
|
+
this.autoResumeTimers.delete(runId);
|
|
1000
|
+
const run = this.store.getRun(runId);
|
|
1001
|
+
if (!run || run.status !== 'failed' || !run.autoResumeAt)
|
|
1002
|
+
return;
|
|
1003
|
+
// Belt and braces against the one gap `reconcileAutoResumes` cannot close: the setting going
|
|
1004
|
+
// off in the window between the last pump and this tick.
|
|
1005
|
+
if (!this.semaphore.autoResumeOnUsageLimit()) {
|
|
1006
|
+
this.clearAutoResume(runId);
|
|
1007
|
+
return;
|
|
1008
|
+
}
|
|
1009
|
+
const attempts = (run.autoResumeAttempts ?? 0) + 1;
|
|
1010
|
+
// `continueRun` retires the pending resume (timer + record fields) on the way in — this is a
|
|
1011
|
+
// resume, not a user turn, so the counter is put back straight after.
|
|
1012
|
+
const resumed = this.continueRun(runId, { text: AUTO_RESUME_PROMPT }, true);
|
|
1013
|
+
if (!resumed.ok) {
|
|
1014
|
+
// Refusals happen before `continueRun` retires anything, so the deadline is still on the
|
|
1015
|
+
// record — and a deadline in the past is a promise the cockpit keeps displaying and the
|
|
1016
|
+
// engine will never keep. Retire it here instead, and say why.
|
|
1017
|
+
this.clearAutoResume(runId);
|
|
1018
|
+
this.store.appendEvent(runId, {
|
|
1019
|
+
type: 'note',
|
|
1020
|
+
message: `automatic resume could not start — ${resumed.error ?? 'unknown'}`,
|
|
1021
|
+
});
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1024
|
+
this.store.updateRun(runId, { autoResumeAttempts: attempts });
|
|
1025
|
+
this.store.appendEvent(runId, {
|
|
1026
|
+
type: 'lifecycle',
|
|
1027
|
+
message: `usage limit reset — resuming automatically (${attempts}/${MAX_AUTO_RESUMES})`,
|
|
1028
|
+
});
|
|
1029
|
+
// A deferred continuation only ENQUEUES itself; the queue moves when something pumps it, and
|
|
1030
|
+
// `recover()` — the other deferring caller — pumps once after its whole bulk sweep. A timer
|
|
1031
|
+
// firing on its own has no such follow-up, so without this the resumed run sits at `queued`
|
|
1032
|
+
// until some unrelated run happens to finish. This is the pump for it.
|
|
1033
|
+
void this.pump();
|
|
1034
|
+
}
|
|
1035
|
+
/**
|
|
1036
|
+
* Make the armed timers agree with the records and the current setting. Runs on every `pump()`
|
|
1037
|
+
* — which is where a settings change lands (a config PUT refreshes the shared semaphore, which
|
|
1038
|
+
* pumps every manager) — and once from `recover()`.
|
|
1039
|
+
*
|
|
1040
|
+
* It is a RECONCILE rather than a one-shot restore because the deadline is durable state and
|
|
1041
|
+
* the timer is not: a restart, a rebuilt project context, a manager disposed mid-wait, or a
|
|
1042
|
+
* refusal all leave a record promising a resume that no timer is holding. Rebuilding from the
|
|
1043
|
+
* record covers every one of those at once — the alternative is a hint counting down to a time
|
|
1044
|
+
* that has already passed, which is exactly the failure this method exists to make impossible.
|
|
1045
|
+
*
|
|
1046
|
+
* Cheap: an in-memory scan, and arming is skipped for every run already held.
|
|
1047
|
+
*/
|
|
1048
|
+
reconcileAutoResumes() {
|
|
1049
|
+
if (!this.semaphore.autoResumeOnUsageLimit()) {
|
|
1050
|
+
// Sweep the RECORDS, not the timer map. A record promising a resume that no timer is
|
|
1051
|
+
// holding is the exact population this method exists for, and it is also the one the
|
|
1052
|
+
// setting can be switched off in front of: cezar restarted while it was off, the config
|
|
1053
|
+
// was hand-edited, or the project context was disposed mid-wait. Retiring only the armed
|
|
1054
|
+
// timers leaves such a record with a live `autoResumeAt`, which `accountHolds()` reads as
|
|
1055
|
+
// a deadline hold — so nothing new starts on that account, `rescueStalledQueue` treats the
|
|
1056
|
+
// phantom appointment as a legitimate reason to sit still, and the cockpit shows a
|
|
1057
|
+
// `scheduled` row for a resume that will never come. `clearAutoResume` covers the armed
|
|
1058
|
+
// ones too, so this one loop is the whole cancellation.
|
|
1059
|
+
const pending = new Set([
|
|
1060
|
+
...this.autoResumeTimers.keys(),
|
|
1061
|
+
...this.store.listRuns().filter((run) => run.autoResumeAt !== undefined).map((run) => run.id),
|
|
1062
|
+
]);
|
|
1063
|
+
for (const runId of pending) {
|
|
1064
|
+
this.clearAutoResume(runId);
|
|
1065
|
+
this.store.appendEvent(runId, {
|
|
1066
|
+
type: 'note',
|
|
1067
|
+
message: 'automatic resume cancelled — auto-resume is switched off',
|
|
1068
|
+
});
|
|
1069
|
+
}
|
|
1070
|
+
return;
|
|
1071
|
+
}
|
|
1072
|
+
for (const run of this.store.listRuns()) {
|
|
1073
|
+
if (run.status !== 'failed' || !run.autoResumeAt)
|
|
1074
|
+
continue;
|
|
1075
|
+
if (this.autoResumeTimers.has(run.id))
|
|
1076
|
+
continue;
|
|
1077
|
+
const deadline = Date.parse(run.autoResumeAt);
|
|
1078
|
+
// A deadline that is unreadable, belongs to a run that has spent its cap, or belongs to a
|
|
1079
|
+
// task the user has archived is retired rather than re-armed: it can only mislead. One
|
|
1080
|
+
// that has just passed arms at zero — the window is open, which is the point.
|
|
1081
|
+
if (run.archived
|
|
1082
|
+
|| !Number.isFinite(deadline)
|
|
1083
|
+
|| (run.autoResumeAttempts ?? 0) >= MAX_AUTO_RESUMES) {
|
|
1084
|
+
this.store.updateRun(run.id, { autoResumeAt: undefined });
|
|
1085
|
+
continue;
|
|
1086
|
+
}
|
|
1087
|
+
// …and one missed by more than a day is retired loudly: reviving a task from another era
|
|
1088
|
+
// is a surprise, not a service, and this is what keeps a sweep from resurrecting every
|
|
1089
|
+
// limit-stopped task a user has long since walked away from.
|
|
1090
|
+
if (Date.now() - deadline > AUTO_RESUME_MISSED_WINDOW_MS) {
|
|
1091
|
+
this.store.updateRun(run.id, { autoResumeAt: undefined });
|
|
1092
|
+
this.store.appendEvent(run.id, {
|
|
1093
|
+
type: 'note',
|
|
1094
|
+
message: 'automatic resume expired — its window reopened over a day ago; continue this task manually',
|
|
1095
|
+
});
|
|
1096
|
+
continue;
|
|
1097
|
+
}
|
|
1098
|
+
this.armAutoResume(run.id, deadline);
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
/**
|
|
1102
|
+
* Hand a run that has not spawned anything back to the queue, when the account it would run on
|
|
1103
|
+
* went into a usage-limit hold (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
1104
|
+
*
|
|
1105
|
+
* The dequeue-time gate in `pump()` cannot be the only one: a run can sit between dequeue and
|
|
1106
|
+
* spawn for a long time — an in-place run waiting for the exclusive repo-root lease is the
|
|
1107
|
+
* measured case — and the account can close in that gap. This is the last honest moment to
|
|
1108
|
+
* refuse, because everything after it costs a real agent turn.
|
|
1109
|
+
*
|
|
1110
|
+
* "Untouched" is the contract: the run has created no session and no worktree, so it goes back
|
|
1111
|
+
* as plain `queued` with its `startedAt` cleared, and `pump()` will pick it up when the window
|
|
1112
|
+
* reopens. Returns true when the caller must abandon the run.
|
|
1113
|
+
*/
|
|
1114
|
+
requeueWhileHeld(runId, workflow, input, runner, state) {
|
|
1115
|
+
const run = this.store.getRun(runId);
|
|
1116
|
+
if (!run || run.status === 'cancelled' || state?.cancelled)
|
|
1117
|
+
return false;
|
|
1118
|
+
// The watchdog sent this one through. Checked, never consumed: the spawn path asks this
|
|
1119
|
+
// question TWICE — here at the top of `execute`, and again after the exclusive repo-root
|
|
1120
|
+
// lease is granted — so a one-shot flag would clear at the first gate and let the second one
|
|
1121
|
+
// hand an in-place run straight back, re-wedging the queue the rescue had just freed.
|
|
1122
|
+
// `dropActive` retires the entry on every terminal path, so the set still cleans itself up.
|
|
1123
|
+
if (this.forceStarted.has(runId))
|
|
1124
|
+
return false;
|
|
1125
|
+
if (!accountHeldFor({ ...run, runner }, this.semaphore.accountHolds(), runner))
|
|
1126
|
+
return false;
|
|
1127
|
+
state?.releaseRepoRoot?.();
|
|
1128
|
+
if (state)
|
|
1129
|
+
state.releaseRepoRoot = undefined;
|
|
1130
|
+
this.pendingJobs.set(runId, { workflow, input });
|
|
1131
|
+
this.queue.push(runId);
|
|
1132
|
+
this.store.updateRun(runId, { status: 'queued', startedAt: undefined, currentStepId: undefined });
|
|
1133
|
+
this.store.appendEvent(runId, {
|
|
1134
|
+
type: 'note',
|
|
1135
|
+
message: 'held in the queue — this agent account is waiting out a usage limit',
|
|
1136
|
+
});
|
|
1137
|
+
this.dropActive(runId);
|
|
1138
|
+
return true;
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* The failsafe: a queue must never be able to wedge.
|
|
1142
|
+
*
|
|
1143
|
+
* Everything else in this file makes an idle queue CORRECT under some condition — a slot cap, a
|
|
1144
|
+
* repo-root lease, and now a usage-limit hold. That is also what makes a wedged queue look
|
|
1145
|
+
* correct, and the hold has already produced one in the field: two resumes fired together, each
|
|
1146
|
+
* holding the account the other was waiting on, and the whole workspace stopped with every task
|
|
1147
|
+
* `queued`. That specific bug is fixed and tested, but "the queue stopped and nothing will ever
|
|
1148
|
+
* restart it" is too expensive a failure mode to leave resting on any single fix being right.
|
|
1149
|
+
*
|
|
1150
|
+
* The test is deliberately about JUSTIFICATION rather than about any particular bug: idling is
|
|
1151
|
+
* legitimate while work is running (here or in another project), or while a real appointment is
|
|
1152
|
+
* still ahead — a scheduled resume that will fire and pump on its own. Anything else is a
|
|
1153
|
+
* queue with work in it, nothing running anywhere, and no event coming to wake it. That gets one
|
|
1154
|
+
* forced sweep, which starts work under the ordinary caps and lets the account's real state
|
|
1155
|
+
* re-assert itself: if the window truly is shut, that task meets the limit and re-establishes an
|
|
1156
|
+
* honest hold, with a real deadline behind it this time.
|
|
1157
|
+
*
|
|
1158
|
+
* Public so a test can drive the wedge directly instead of waiting out the interval.
|
|
1159
|
+
*/
|
|
1160
|
+
async rescueStalledQueue(now = Date.now()) {
|
|
1161
|
+
// First, the worst shape: a record that says `queued` while the engine holds no job, no
|
|
1162
|
+
// continuation and no queue entry for it. `pump()` cannot see such a run — it iterates the
|
|
1163
|
+
// queue, and this one is not in it — so nothing will ever start it. Re-adopt it through the
|
|
1164
|
+
// same path boot recovery uses.
|
|
1165
|
+
for (const run of this.store.listRuns()) {
|
|
1166
|
+
if (run.status !== 'queued')
|
|
1167
|
+
continue;
|
|
1168
|
+
if (this.active.has(run.id) || this.starting.has(run.id))
|
|
1169
|
+
continue;
|
|
1170
|
+
if (this.pendingJobs.has(run.id) || this.pendingContinuations.has(run.id))
|
|
1171
|
+
continue;
|
|
1172
|
+
if (this.queue.includes(run.id))
|
|
1173
|
+
continue;
|
|
1174
|
+
console.warn(`[cez] queue watchdog: re-adopting queued run ${run.id} the engine had lost`);
|
|
1175
|
+
await this.reviveQueuedRun(run, 'queue watchdog');
|
|
1176
|
+
}
|
|
1177
|
+
if (this.queue.length === 0)
|
|
1178
|
+
return;
|
|
1179
|
+
if (this.busySlots() > 0 || this.starting.size > 0)
|
|
1180
|
+
return;
|
|
1181
|
+
if (this.semaphore.busy() > 0)
|
|
1182
|
+
return;
|
|
1183
|
+
// A future deadline is a real reason to sit still: that timer will fire and pump.
|
|
1184
|
+
for (const run of this.store.listRuns()) {
|
|
1185
|
+
if (run.status !== 'failed' || !run.autoResumeAt)
|
|
1186
|
+
continue;
|
|
1187
|
+
const deadline = Date.parse(run.autoResumeAt);
|
|
1188
|
+
if (Number.isFinite(deadline) && deadline > now)
|
|
1189
|
+
return;
|
|
1190
|
+
}
|
|
1191
|
+
if (this.semaphore.accountHolds().inFlight.size === 0) {
|
|
1192
|
+
// Not the hold, then — some other wakeup went missing. An ordinary pump is the whole fix,
|
|
1193
|
+
// and it is idempotent, so this stays quiet.
|
|
1194
|
+
void this.pump();
|
|
1195
|
+
return;
|
|
1196
|
+
}
|
|
1197
|
+
console.warn('[cez] queue watchdog: work is queued, nothing is running, and the usage-limit hold has no'
|
|
1198
|
+
+ ' deadline behind it — starting the next task anyway');
|
|
1199
|
+
this.forceNextPump = true;
|
|
1200
|
+
void this.pump();
|
|
1201
|
+
}
|
|
1202
|
+
/**
|
|
1203
|
+
* The accounts this project is currently holding: one key per run parked on a usage-limit
|
|
1204
|
+
* resume that has not come due yet (spec 2026-08-03-auto-resume-after-usage-limit).
|
|
1205
|
+
*
|
|
1206
|
+
* Published to the shared semaphore so the hold spans PROJECTS — one Claude account can be
|
|
1207
|
+
* driving tasks in three repos, and a limit closes it for all of them. Derived from the
|
|
1208
|
+
* records on every ask rather than tracked as state: a deadline that passes, a resume that
|
|
1209
|
+
* fires, a cancel, an archive and a delete all lift the hold with no bookkeeping.
|
|
1210
|
+
*
|
|
1211
|
+
* Deliberately excludes a deadline that has already passed — that run is about to resume, and
|
|
1212
|
+
* holding the queue for it would only stall the very work the window reopened for.
|
|
1213
|
+
*/
|
|
1214
|
+
accountHolds(now = Date.now()) {
|
|
1215
|
+
const deadline = new Set();
|
|
1216
|
+
const inFlight = new Set();
|
|
1217
|
+
for (const run of this.store.listRuns()) {
|
|
1218
|
+
// A holding run always carries the runner it actually ran on, so the fallback is unused
|
|
1219
|
+
// here — it is spelled out rather than `!` so a future record shape degrades, not throws.
|
|
1220
|
+
const key = () => runAccountKey(run, run.runner ?? 'claude');
|
|
1221
|
+
if (run.status === 'failed' && run.autoResumeAt) {
|
|
1222
|
+
const at = Date.parse(run.autoResumeAt);
|
|
1223
|
+
if (Number.isFinite(at) && at > now)
|
|
1224
|
+
deadline.add(key());
|
|
1225
|
+
}
|
|
1226
|
+
else if (resumeInFlight(run)) {
|
|
1227
|
+
inFlight.add(key());
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
return { deadline, inFlight };
|
|
1231
|
+
}
|
|
1232
|
+
/**
|
|
1233
|
+
* The PER-TASK off switch (`DELETE /api/v1/runs/:id/auto-resume`, and the archive route):
|
|
1234
|
+
* stop resuming THIS task, without touching the workspace setting or any other task.
|
|
1235
|
+
*
|
|
1236
|
+
* Idempotent — a run with nothing pending answers the same way, because "this task will not
|
|
1237
|
+
* resume itself" is equally true either way. Returns false only when the run does not exist,
|
|
1238
|
+
* which is the route's 404.
|
|
1239
|
+
*/
|
|
1240
|
+
cancelAutoResume(runId) {
|
|
1241
|
+
const run = this.store.getRun(runId);
|
|
1242
|
+
if (!run)
|
|
1243
|
+
return false;
|
|
1244
|
+
const pending = run.autoResumeAt !== undefined || this.autoResumeTimers.has(runId);
|
|
1245
|
+
this.clearAutoResume(runId);
|
|
1246
|
+
if (pending) {
|
|
1247
|
+
this.store.appendEvent(runId, {
|
|
1248
|
+
type: 'note',
|
|
1249
|
+
message: 'automatic resume cancelled for this task',
|
|
1250
|
+
});
|
|
1251
|
+
// This run may have been the last thing holding its account's queue — nothing else will
|
|
1252
|
+
// notice, since the hold is derived and its release is not an event.
|
|
1253
|
+
void this.pump();
|
|
1254
|
+
}
|
|
1255
|
+
return true;
|
|
1256
|
+
}
|
|
1257
|
+
/** Retire a pending resume — timer, deadline and counter. The counter goes too because every
|
|
1258
|
+
* caller is a fresh epoch: a human Continue, or a resume that re-stamps its own count. */
|
|
1259
|
+
clearAutoResume(runId) {
|
|
1260
|
+
const timer = this.autoResumeTimers.get(runId);
|
|
1261
|
+
if (timer)
|
|
1262
|
+
clearTimeout(timer);
|
|
1263
|
+
this.autoResumeTimers.delete(runId);
|
|
1264
|
+
const run = this.store.getRun(runId);
|
|
1265
|
+
if (!run)
|
|
1266
|
+
return;
|
|
1267
|
+
if (run.autoResumeAt !== undefined || run.autoResumeAttempts !== undefined) {
|
|
1268
|
+
this.store.updateRun(runId, { autoResumeAt: undefined, autoResumeAttempts: undefined });
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
649
1271
|
/** Reclaim finished worktrees beyond the keep-limit (#483) — directory only,
|
|
650
1272
|
* `cez/<id8>` branch kept. Best-effort; a failure never affects run
|
|
651
1273
|
* lifecycle. `review`/live runs are excluded by the selector. */
|
|
@@ -723,6 +1345,7 @@ export class RunManager {
|
|
|
723
1345
|
if (queuedAt >= 0) {
|
|
724
1346
|
this.queue.splice(queuedAt, 1);
|
|
725
1347
|
this.pendingJobs.delete(runId);
|
|
1348
|
+
this.pendingContinuations.delete(runId);
|
|
726
1349
|
this.store.updateRun(runId, { status: 'cancelled', finishedAt: new Date().toISOString() });
|
|
727
1350
|
this.store.appendEvent(runId, { type: 'lifecycle', message: 'cancelled while queued' });
|
|
728
1351
|
return true;
|
|
@@ -763,35 +1386,13 @@ export class RunManager {
|
|
|
763
1386
|
.map((part) => part.trim())
|
|
764
1387
|
.filter((part) => part.length > 0)
|
|
765
1388
|
.join('\n\n');
|
|
766
|
-
const readImages = (urls, kind) => {
|
|
767
|
-
const images = [];
|
|
768
|
-
for (const url of urls) {
|
|
769
|
-
const name = url.split('/').pop();
|
|
770
|
-
if (!name || name.includes('..') || name.includes('/') || name.includes('\\'))
|
|
771
|
-
continue;
|
|
772
|
-
try {
|
|
773
|
-
const data = readFileSync(join(this.dataDir, 'runs', `${runId}-images`, name));
|
|
774
|
-
images.push({
|
|
775
|
-
type: 'image',
|
|
776
|
-
source: { type: 'base64', media_type: mediaTypeFor(name), data: data.toString('base64') },
|
|
777
|
-
});
|
|
778
|
-
}
|
|
779
|
-
catch {
|
|
780
|
-
// Degrade, never fail the boot (AGENTS.md): the user deleted `.ai/cezar/`
|
|
781
|
-
// or the file is unreadable — start with the text and say which image went.
|
|
782
|
-
this.store.appendEvent(runId, {
|
|
783
|
-
type: 'note',
|
|
784
|
-
message: `${kind} attachment ${name} could not be read — starting without it`,
|
|
785
|
-
});
|
|
786
|
-
}
|
|
787
|
-
}
|
|
788
|
-
return images;
|
|
789
|
-
};
|
|
790
1389
|
// Keep the original in-memory blocks for a live process (including the
|
|
791
1390
|
// best-effort case where persistence failed). Recovery has no such copy,
|
|
792
1391
|
// so rebuild it from the durable task-image URLs.
|
|
793
|
-
const images = input.images?.length
|
|
794
|
-
|
|
1392
|
+
const images = input.images?.length
|
|
1393
|
+
? input.images
|
|
1394
|
+
: this.readPersistedImages(runId, run.taskImages ?? [], 'task').blocks;
|
|
1395
|
+
const stackedImages = this.readPersistedImages(runId, stack.flatMap((m) => m.images ?? []), 'queued').blocks;
|
|
795
1396
|
return {
|
|
796
1397
|
...input,
|
|
797
1398
|
task,
|
|
@@ -799,15 +1400,67 @@ export class RunManager {
|
|
|
799
1400
|
...(stackedImages.length ? { stackedImages } : { stackedImages: undefined }),
|
|
800
1401
|
};
|
|
801
1402
|
}
|
|
1403
|
+
/** Apply edits and messages made while a restart continuation waits for
|
|
1404
|
+
* capacity. The durable record remains the source of truth, just as it is for
|
|
1405
|
+
* an ordinary queued workflow (#472), so a second restart reconstructs and
|
|
1406
|
+
* hydrates the same amendments instead of dropping them. */
|
|
1407
|
+
hydrateQueuedContinuation(runId, continuation) {
|
|
1408
|
+
const run = this.store.getRun(runId);
|
|
1409
|
+
if (!run) {
|
|
1410
|
+
return { ...continuation, persistedImages: [], persistedAttachments: [] };
|
|
1411
|
+
}
|
|
1412
|
+
const stack = run.queuedMessages ?? [];
|
|
1413
|
+
const amendedTask = [run.task, ...stack.map((message) => message.text)]
|
|
1414
|
+
.map((part) => part.trim())
|
|
1415
|
+
.filter((part) => part.length > 0)
|
|
1416
|
+
.join('\n\n');
|
|
1417
|
+
const prompt = amendedTask
|
|
1418
|
+
? `${continuation.prompt}\n\nCurrent task and queued updates:\n\n${amendedTask}`
|
|
1419
|
+
: continuation.prompt;
|
|
1420
|
+
const persisted = this.readPersistedImages(runId, stack.flatMap((message) => message.images ?? []), 'queued');
|
|
1421
|
+
return {
|
|
1422
|
+
...continuation,
|
|
1423
|
+
prompt,
|
|
1424
|
+
persistedImages: persisted.blocks,
|
|
1425
|
+
persistedAttachments: persisted.attachments,
|
|
1426
|
+
};
|
|
1427
|
+
}
|
|
1428
|
+
readPersistedImages(runId, urls, kind) {
|
|
1429
|
+
const blocks = [];
|
|
1430
|
+
const attachments = [];
|
|
1431
|
+
for (const url of urls) {
|
|
1432
|
+
const name = url.split('/').pop();
|
|
1433
|
+
if (!name || name.includes('..') || name.includes('/') || name.includes('\\'))
|
|
1434
|
+
continue;
|
|
1435
|
+
const path = join(this.dataDir, 'runs', `${runId}-images`, name);
|
|
1436
|
+
try {
|
|
1437
|
+
const data = readFileSync(path);
|
|
1438
|
+
blocks.push({
|
|
1439
|
+
type: 'image',
|
|
1440
|
+
source: { type: 'base64', media_type: mediaTypeFor(name), data: data.toString('base64') },
|
|
1441
|
+
});
|
|
1442
|
+
attachments.push({ name, url, path });
|
|
1443
|
+
}
|
|
1444
|
+
catch {
|
|
1445
|
+
// Degrade, never fail the boot (AGENTS.md): the user deleted `.ai/cezar/`
|
|
1446
|
+
// or the file is unreadable — start with the text and say which image went.
|
|
1447
|
+
this.store.appendEvent(runId, {
|
|
1448
|
+
type: 'note',
|
|
1449
|
+
message: `${kind} attachment ${name} could not be read — starting without it`,
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
return { blocks, attachments };
|
|
1454
|
+
}
|
|
802
1455
|
/**
|
|
803
1456
|
* Still waiting for a slot? Checked against the engine's own queue rather than
|
|
804
1457
|
* the record's `status` (#472): the record is written by `execute()` a tick
|
|
805
1458
|
* after `pump()` dequeues, so a status read can see `queued` for a run that has
|
|
806
|
-
* already started.
|
|
807
|
-
* the authoritative answer for "can this prompt still be amended".
|
|
1459
|
+
* already started. The pending maps are deleted synchronously at dequeue, so
|
|
1460
|
+
* they are the authoritative answer for "can this prompt still be amended".
|
|
808
1461
|
*/
|
|
809
1462
|
isQueued(runId) {
|
|
810
|
-
return this.pendingJobs.has(runId);
|
|
1463
|
+
return this.pendingJobs.has(runId) || this.pendingContinuations.has(runId);
|
|
811
1464
|
}
|
|
812
1465
|
/** Split `ContentBlock[]` into the persisted shape a stacked message holds. */
|
|
813
1466
|
toQueuedMessage(runId, content) {
|
|
@@ -987,6 +1640,18 @@ export class RunManager {
|
|
|
987
1640
|
* then offers "Continue" instead.
|
|
988
1641
|
*/
|
|
989
1642
|
sendMessage(runId, content) {
|
|
1643
|
+
const delivered = this.deliverMessage(runId, content, true);
|
|
1644
|
+
if (delivered) {
|
|
1645
|
+
const state = this.active.get(runId);
|
|
1646
|
+
if (state)
|
|
1647
|
+
state.monitoringWakeups = 0;
|
|
1648
|
+
this.store.updateRun(runId, { monitoringWakeCapReached: undefined });
|
|
1649
|
+
}
|
|
1650
|
+
return delivered;
|
|
1651
|
+
}
|
|
1652
|
+
/** Shared live-session delivery. Synthetic scheduler prompts reuse lifecycle
|
|
1653
|
+
* bookkeeping without masquerading as user-authored transcript messages. */
|
|
1654
|
+
deliverMessage(runId, content, userAuthored) {
|
|
990
1655
|
const state = this.active.get(runId);
|
|
991
1656
|
if (!state?.session?.open || state.cancelled)
|
|
992
1657
|
return false;
|
|
@@ -997,27 +1662,32 @@ export class RunManager {
|
|
|
997
1662
|
// Persist the attached images so the thread can render them (not just count them) — the same
|
|
998
1663
|
// on-disk store + `/images/` route the agent's own screenshots use. `pasted` prefix marks
|
|
999
1664
|
// these as user attachments (vs. agent tool screenshots) on disk (#357).
|
|
1000
|
-
const persisted = content
|
|
1665
|
+
const persisted = userAuthored ? content
|
|
1001
1666
|
.filter((b) => b.type === 'image')
|
|
1002
1667
|
.map((b) => this.persistImage(runId, b.source.media_type, b.source.data, 'pasted'))
|
|
1003
|
-
.filter((saved) => saved !== null);
|
|
1668
|
+
.filter((saved) => saved !== null) : [];
|
|
1004
1669
|
const images = persisted.map((saved) => saved.url);
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1670
|
+
if (userAuthored) {
|
|
1671
|
+
this.store.appendEvent(runId, {
|
|
1672
|
+
type: 'user-message',
|
|
1673
|
+
stepId: state.currentStepId,
|
|
1674
|
+
text,
|
|
1675
|
+
imageCount: content.filter((b) => b.type === 'image').length,
|
|
1676
|
+
images,
|
|
1677
|
+
});
|
|
1678
|
+
}
|
|
1012
1679
|
// Tell the agent where the pasted files live on disk (#357): the base64 blocks below still
|
|
1013
1680
|
// ride along so the model can *view* them, but a real path is what lets it *operate* on them
|
|
1014
1681
|
// (save, `cp`, attach to a GitHub issue/PR) — and it's the only usable reference on backends
|
|
1015
1682
|
// (codex, opencode) that drop image blocks entirely before reaching the model.
|
|
1016
|
-
const
|
|
1683
|
+
const expanded = userAuthored ? expandRegistrySlashSkill(content, state.skills ?? []) : content;
|
|
1684
|
+
const deliverable = persisted.length ? [...expanded, pastedAttachmentsNote(persisted)] : expanded;
|
|
1017
1685
|
const delivered = state.session.sendMessage(deliverable);
|
|
1018
1686
|
if (delivered) {
|
|
1019
1687
|
this.clearIdleTimer(state);
|
|
1688
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1020
1689
|
this.waiting.delete(runId); // resumed — the run counts against slots again
|
|
1690
|
+
this.monitoring.delete(runId);
|
|
1021
1691
|
// Clear any `monitoring` activity — the agent is actively working again
|
|
1022
1692
|
// (spec 2026-07-18-subagent-monitoring-status, #490).
|
|
1023
1693
|
this.store.updateRun(runId, { status: 'running', activity: undefined });
|
|
@@ -1054,7 +1724,13 @@ export class RunManager {
|
|
|
1054
1724
|
* behaves exactly like an interactive step: `waiting` after each turn,
|
|
1055
1725
|
* messages via sendMessage, closed by finish/idle/cancel.
|
|
1056
1726
|
*/
|
|
1057
|
-
continueRun(runId, opts = {}
|
|
1727
|
+
continueRun(runId, opts = {},
|
|
1728
|
+
/** Restart recovery may discover several interrupted tasks at once. Those
|
|
1729
|
+
* continuations are queued; an explicit user Continue remains immediate. */
|
|
1730
|
+
deferForCapacity = false) {
|
|
1731
|
+
if (agentModelsLocked(this.repoRoot) && opts.model?.trim()) {
|
|
1732
|
+
return { ok: false, error: AGENT_MODELS_LOCKED_ERROR };
|
|
1733
|
+
}
|
|
1058
1734
|
if (this.active.has(runId))
|
|
1059
1735
|
return { ok: false, error: 'run is still active' };
|
|
1060
1736
|
const run = this.store.getRun(runId);
|
|
@@ -1105,10 +1781,34 @@ export class RunManager {
|
|
|
1105
1781
|
: {}),
|
|
1106
1782
|
});
|
|
1107
1783
|
}
|
|
1784
|
+
// Everything that could refuse this continuation has now passed, so a pending usage-limit
|
|
1785
|
+
// resume is superseded either way: this IS that resume (it re-stamps its own counter), or a
|
|
1786
|
+
// human got there first — and then the counter starts over, because the cap only exists to
|
|
1787
|
+
// bound UNATTENDED resumes.
|
|
1788
|
+
this.clearAutoResume(runId);
|
|
1108
1789
|
const continuations = run.steps.filter((s) => s.id.startsWith('continue-')).length;
|
|
1109
1790
|
const stepId = `continue-${continuations + 1}`;
|
|
1110
1791
|
this.store.addStep(runId, { id: stepId, name: 'Continue', kind: 'agent' });
|
|
1111
|
-
|
|
1792
|
+
const prompt = opts.text?.trim() || 'Continue.';
|
|
1793
|
+
const images = opts.images ?? [];
|
|
1794
|
+
if (deferForCapacity) {
|
|
1795
|
+
this.pendingContinuations.set(runId, {
|
|
1796
|
+
stepId,
|
|
1797
|
+
sessionId: resume ? sessionStep.sessionId : undefined,
|
|
1798
|
+
backend: targetRunner,
|
|
1799
|
+
prompt,
|
|
1800
|
+
images,
|
|
1801
|
+
});
|
|
1802
|
+
this.queue.push(runId);
|
|
1803
|
+
this.store.updateRun(runId, {
|
|
1804
|
+
status: 'queued',
|
|
1805
|
+
error: undefined,
|
|
1806
|
+
finishedAt: undefined,
|
|
1807
|
+
currentStepId: undefined,
|
|
1808
|
+
});
|
|
1809
|
+
return { ok: true };
|
|
1810
|
+
}
|
|
1811
|
+
void this.runContinuation(runId, stepId, resume ? sessionStep.sessionId : undefined, targetRunner, prompt, images).catch((err) => {
|
|
1112
1812
|
const message = err instanceof Error ? err.message : String(err);
|
|
1113
1813
|
this.store.updateRun(runId, {
|
|
1114
1814
|
status: 'failed',
|
|
@@ -1123,7 +1823,11 @@ export class RunManager {
|
|
|
1123
1823
|
/** Screenshots pasted into the follow-up composer — delivered with the
|
|
1124
1824
|
* reopened session's opening message, exactly like a live-session
|
|
1125
1825
|
* message's attachments. */
|
|
1126
|
-
images = []
|
|
1826
|
+
images = [],
|
|
1827
|
+
/** Queued-message screenshots were persisted when they were enqueued and
|
|
1828
|
+
* reconstructed at dequeue. Keep them separate from fresh `images` so
|
|
1829
|
+
* opening a recovered continuation does not persist duplicate files. */
|
|
1830
|
+
persistedImages = [], persistedAttachments = []) {
|
|
1127
1831
|
// Continuation runs in the task's worktree when it still exists (spec
|
|
1128
1832
|
// 006) — the resumed session sees exactly what the original run left.
|
|
1129
1833
|
// Retention (#483) may have reclaimed this run's worktree directory while
|
|
@@ -1141,6 +1845,7 @@ export class RunManager {
|
|
|
1141
1845
|
: this.repoRoot;
|
|
1142
1846
|
const state = { cancelled: false, interrupt: () => undefined, cwd };
|
|
1143
1847
|
this.active.set(runId, state);
|
|
1848
|
+
this.starting.delete(runId);
|
|
1144
1849
|
if (state.cwd === this.repoRoot) {
|
|
1145
1850
|
this.store.appendEvent(runId, {
|
|
1146
1851
|
type: 'note',
|
|
@@ -1180,19 +1885,22 @@ export class RunManager {
|
|
|
1180
1885
|
// images rather than a bare count, and handed to the agent BOTH as base64 blocks (so it can
|
|
1181
1886
|
// view them) and as absolute paths appended to the prompt (so it can operate on them — and
|
|
1182
1887
|
// because codex/opencode drop image blocks before they reach the model).
|
|
1183
|
-
const
|
|
1888
|
+
const freshAttachments = images
|
|
1184
1889
|
.filter((b) => b.type === 'image')
|
|
1185
1890
|
.map((b) => this.persistImage(runId, b.source.media_type, b.source.data, 'pasted'))
|
|
1186
1891
|
.filter((saved) => saved !== null);
|
|
1892
|
+
const openingImages = [...images, ...persistedImages];
|
|
1893
|
+
const attachments = [...freshAttachments, ...persistedAttachments];
|
|
1187
1894
|
this.store.appendEvent(runId, {
|
|
1188
1895
|
type: 'user-message',
|
|
1189
1896
|
stepId,
|
|
1190
1897
|
text: prompt,
|
|
1191
|
-
imageCount:
|
|
1898
|
+
imageCount: openingImages.filter((b) => b.type === 'image').length,
|
|
1192
1899
|
...(attachments.length ? { images: attachments.map((saved) => saved.url) } : {}),
|
|
1193
1900
|
});
|
|
1194
1901
|
let stepCost = 0;
|
|
1195
1902
|
let turnText = '';
|
|
1903
|
+
let sessionError;
|
|
1196
1904
|
const sink = this.makeUiSink(runId, stepId);
|
|
1197
1905
|
const onEvent = (event) => {
|
|
1198
1906
|
if (event.type === 'image') {
|
|
@@ -1209,6 +1917,13 @@ export class RunManager {
|
|
|
1209
1917
|
return;
|
|
1210
1918
|
}
|
|
1211
1919
|
this.store.appendEvent(runId, { ...event, stepId });
|
|
1920
|
+
if (event.type === 'error') {
|
|
1921
|
+
sessionError ??= event.message;
|
|
1922
|
+
state.session?.interrupt();
|
|
1923
|
+
return;
|
|
1924
|
+
}
|
|
1925
|
+
if (sessionError)
|
|
1926
|
+
return;
|
|
1212
1927
|
if (event.type === 'session') {
|
|
1213
1928
|
this.store.updateStep(runId, stepId, { sessionId: event.sessionId, backend });
|
|
1214
1929
|
}
|
|
@@ -1229,9 +1944,13 @@ export class RunManager {
|
|
|
1229
1944
|
const done = sessionOpen && DONE_MARKER_RE.test(turnText.trimEnd());
|
|
1230
1945
|
// `CEZ:ASK` → the user is genuinely blocked; wins over `CEZ:MONITORING`
|
|
1231
1946
|
// (a pending question is always attention), loses to `CEZ:DONE` (#473).
|
|
1232
|
-
const
|
|
1947
|
+
const askResult = sessionOpen && !done ? parseAskMarkerResult(turnText) : undefined;
|
|
1948
|
+
const ask = askResult?.kind === 'valid' ? askResult.request : null;
|
|
1949
|
+
const askRejection = askResult ? askMarkerRejection(askResult) : undefined;
|
|
1233
1950
|
const monitoring = sessionOpen && !done && !ask && MONITORING_MARKER_RE.test(turnText.trimEnd());
|
|
1234
1951
|
turnText = '';
|
|
1952
|
+
if (askRejection)
|
|
1953
|
+
this.store.appendEvent(runId, { type: 'note', message: askRejection, stepId });
|
|
1235
1954
|
if (done) {
|
|
1236
1955
|
// Goal achieved (agent contract, #347) — same as in runAgentStep.
|
|
1237
1956
|
this.store.appendEvent(runId, { type: 'lifecycle', message: 'goal achieved — session closed' });
|
|
@@ -1267,16 +1986,30 @@ export class RunManager {
|
|
|
1267
1986
|
if (monitoring) {
|
|
1268
1987
|
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
1269
1988
|
this.store.updateStep(runId, stepId, { status: 'running' });
|
|
1989
|
+
this.monitoring.add(runId);
|
|
1990
|
+
this.clearIdleTimer(state);
|
|
1991
|
+
this.armMonitoringWakeTimer(runId, state);
|
|
1270
1992
|
}
|
|
1271
1993
|
else {
|
|
1272
1994
|
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
1273
1995
|
this.store.updateStep(runId, stepId, { status: 'waiting' });
|
|
1996
|
+
this.monitoring.delete(runId);
|
|
1997
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1274
1998
|
}
|
|
1275
1999
|
this.waiting.add(runId);
|
|
1276
|
-
|
|
2000
|
+
if (!monitoring)
|
|
2001
|
+
this.armIdleTimer(runId, state);
|
|
1277
2002
|
this.releaseSlot();
|
|
1278
2003
|
}
|
|
1279
2004
|
}
|
|
2005
|
+
// A turn that completed is the ONLY evidence the provider's window actually reopened, so
|
|
2006
|
+
// it is what retires the consecutive-resume counter — which in turn releases the account
|
|
2007
|
+
// hold for every other task queued behind it (spec
|
|
2008
|
+
// 2026-08-03-auto-resume-after-usage-limit). `settleSuccess` does the same for a run that
|
|
2009
|
+
// finishes outright; this covers the far more common "parked for the user" ending.
|
|
2010
|
+
if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
|
|
2011
|
+
this.store.updateRun(runId, { autoResumeAttempts: undefined });
|
|
2012
|
+
}
|
|
1280
2013
|
appendHandoffHeartbeat(this.dataDir, runId, `turn complete — status=${monitoring ? 'monitoring' : sessionOpen ? 'waiting' : 'running'}`);
|
|
1281
2014
|
}
|
|
1282
2015
|
};
|
|
@@ -1291,7 +2024,7 @@ export class RunManager {
|
|
|
1291
2024
|
// instead of `opus`). Fail loud here too rather than let the backend pick a default.
|
|
1292
2025
|
let continueModel;
|
|
1293
2026
|
try {
|
|
1294
|
-
const normalized = normalizeModelForBackend(continueBackend, record?.model);
|
|
2027
|
+
const normalized = normalizeModelForBackend(continueBackend, agentModelsLocked(this.repoRoot) ? undefined : record?.model, { configuredProvider: await configuredModelProvider(continueBackend, state.cwd) });
|
|
1295
2028
|
continueModel = normalized?.backendModel;
|
|
1296
2029
|
this.store.updateRun(runId, {
|
|
1297
2030
|
modelIdentity: normalized ? formatModelIdentity(normalized.identity) : undefined,
|
|
@@ -1320,18 +2053,31 @@ export class RunManager {
|
|
|
1320
2053
|
this.dropActive(runId);
|
|
1321
2054
|
return;
|
|
1322
2055
|
}
|
|
2056
|
+
// Resuming reattaches to a session that lives inside ONE account's config dir, so the
|
|
2057
|
+
// continuation must run under the account that created it — not whatever the project has
|
|
2058
|
+
// been switched to since. The owning step is the one carrying this session id.
|
|
2059
|
+
const resumedProfileId = sessionId === undefined
|
|
2060
|
+
? undefined
|
|
2061
|
+
: record?.steps.find((s) => s.sessionId === sessionId)?.profileId;
|
|
2062
|
+
const continueProfile = await this.agentEnvForStep(runId, continueBackend, {
|
|
2063
|
+
generateFollowups,
|
|
2064
|
+
recordedProfileId: resumedProfileId,
|
|
2065
|
+
});
|
|
2066
|
+
this.store.updateStep(runId, stepId, { profileId: continueProfile.profileId });
|
|
1323
2067
|
const runner = createRunner(continueBackend);
|
|
2068
|
+
state.currentStepId = stepId;
|
|
2069
|
+
this.beginUsageInvocation(runId, state, stepId);
|
|
1324
2070
|
const session = runner.startSession({
|
|
1325
2071
|
// The Continue step is a fresh agent session on the same run — the
|
|
1326
2072
|
// run's extra system prompt (already resolved at execute time and
|
|
1327
2073
|
// echoed on the record) rides along with the handoff contract.
|
|
1328
2074
|
systemPrompt: composeSystemPrompt(record?.systemPrompt, generateFollowups ? HANDOFF_INSTRUCTIONS : HANDOFF_ONLY_INSTRUCTIONS),
|
|
1329
2075
|
userPrompt: attachments.length ? `${prompt}\n\n${pastedAttachmentsText(attachments)}` : prompt,
|
|
1330
|
-
...(
|
|
2076
|
+
...(openingImages.length ? { images: openingImages } : {}),
|
|
1331
2077
|
cwd: state.cwd,
|
|
1332
2078
|
allowedTools: DEFAULT_ALLOWED_TOOLS,
|
|
1333
2079
|
additionalDirectories: [join(this.dataDir, 'runs')],
|
|
1334
|
-
env:
|
|
2080
|
+
env: continueProfile.env,
|
|
1335
2081
|
model: continueModel,
|
|
1336
2082
|
sessionId,
|
|
1337
2083
|
resume: sessionId !== undefined,
|
|
@@ -1340,13 +2086,14 @@ export class RunManager {
|
|
|
1340
2086
|
state.session = session;
|
|
1341
2087
|
state.sessionEverOpened = true;
|
|
1342
2088
|
this.flushDeferred(runId);
|
|
1343
|
-
state.currentStepId = stepId;
|
|
1344
2089
|
state.interrupt = () => session.interrupt();
|
|
1345
2090
|
if (session.pid !== undefined)
|
|
1346
2091
|
registerRunProcess(runId, session.pid);
|
|
1347
2092
|
const finishedAt = () => new Date().toISOString();
|
|
1348
2093
|
try {
|
|
1349
2094
|
await session.result;
|
|
2095
|
+
if (sessionError)
|
|
2096
|
+
throw new Error(sessionError);
|
|
1350
2097
|
sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
|
|
1351
2098
|
if (state.cancelled) {
|
|
1352
2099
|
this.store.updateStep(runId, stepId, { status: 'cancelled', finishedAt: finishedAt() });
|
|
@@ -1399,6 +2146,12 @@ export class RunManager {
|
|
|
1399
2146
|
// the config default. Per-step `runner` can still override it below.
|
|
1400
2147
|
const config = await loadConfig(this.repoRoot);
|
|
1401
2148
|
const taskBackend = input.runner ?? config.defaultRunner;
|
|
2149
|
+
// The account may have gone into a usage-limit hold since this run was dequeued — the queue
|
|
2150
|
+
// gate cannot be the only one, because dequeue is not the moment of no return. Nothing has
|
|
2151
|
+
// happened yet here, so the run goes back to the queue untouched (spec
|
|
2152
|
+
// 2026-08-03-auto-resume-after-usage-limit).
|
|
2153
|
+
if (this.requeueWhileHeld(runId, workflow, input, taskBackend))
|
|
2154
|
+
return;
|
|
1402
2155
|
// Extra system prompt (R2 2.3): POST override > config default; echoed on
|
|
1403
2156
|
// the record so the UI/API can show what the run actually used.
|
|
1404
2157
|
const extraSystemPrompt = resolveExtraSystemPrompt(input.systemPrompt, config.systemPrompt);
|
|
@@ -1408,7 +2161,7 @@ export class RunManager {
|
|
|
1408
2161
|
// can still override below); the authoritative fail-loud gate is at spawn.
|
|
1409
2162
|
let modelIdentity;
|
|
1410
2163
|
try {
|
|
1411
|
-
const normalized = normalizeModelForBackend(taskBackend, input.model);
|
|
2164
|
+
const normalized = normalizeModelForBackend(taskBackend, agentModelsLocked(this.repoRoot) ? undefined : input.model, { configuredProvider: await configuredModelProvider(taskBackend, this.repoRoot) });
|
|
1412
2165
|
modelIdentity = normalized ? formatModelIdentity(normalized.identity) : undefined;
|
|
1413
2166
|
}
|
|
1414
2167
|
catch {
|
|
@@ -1429,11 +2182,20 @@ export class RunManager {
|
|
|
1429
2182
|
// established; only explicit opt-out and non-Git modes run in place.
|
|
1430
2183
|
const repo = await getRepoInfo(this.repoRoot);
|
|
1431
2184
|
if (repo && input.worktree === false) {
|
|
1432
|
-
// Composer opt-out: run in the repo working tree, no branch/worktree
|
|
1433
|
-
//
|
|
2185
|
+
// Composer opt-out: run in the repo working tree, no branch/worktree. The
|
|
2186
|
+
// repository-root lease serializes these runs so workflows cannot overlap.
|
|
2187
|
+
// Pin the starting commit: the session's Changes and Commits views use it
|
|
2188
|
+
// as their stable lower bound while reading the current working copy.
|
|
2189
|
+
const startingCommit = await getHeadCommit(repo.root);
|
|
2190
|
+
if (startingCommit)
|
|
2191
|
+
this.store.updateRun(runId, { baseBranch: startingCommit });
|
|
1434
2192
|
emit({ type: 'note', message: 'worktree off — running in the repo working tree' });
|
|
1435
2193
|
}
|
|
1436
2194
|
else if (repo) {
|
|
2195
|
+
emit({
|
|
2196
|
+
type: 'note',
|
|
2197
|
+
message: `worktree on — using an isolated task worktree (${input.worktree === true ? 'explicit request' : 'default'})`,
|
|
2198
|
+
});
|
|
1437
2199
|
// Fork from the configured base branch (config.json `baseBranch`, e.g.
|
|
1438
2200
|
// `develop`) — also the target of the eventual draft PR. Unresolvable
|
|
1439
2201
|
// (typo, not fetched) → note + the currently checked-out branch.
|
|
@@ -1501,6 +2263,13 @@ export class RunManager {
|
|
|
1501
2263
|
// below breaks on `cancelled` before touching the tree and settles the
|
|
1502
2264
|
// run through the usual path.
|
|
1503
2265
|
await this.acquireRepoRoot(runId, state);
|
|
2266
|
+
// THE window that matters for an in-place run. Waiting for the exclusive tree can take
|
|
2267
|
+
// minutes, and a run parked on that lease holds no slot (#347) — so the queue keeps
|
|
2268
|
+
// advancing behind it and the dequeue-time gate is long past. Measured with five in-place
|
|
2269
|
+
// tasks and `maxParallel: 2`: four of them started. Re-ask here, where the very next thing
|
|
2270
|
+
// is a spawn, and hand the run back to the queue if the account closed meanwhile.
|
|
2271
|
+
if (this.requeueWhileHeld(runId, workflow, input, taskBackend, state))
|
|
2272
|
+
return;
|
|
1504
2273
|
}
|
|
1505
2274
|
// Handoff journal (spec 007) — seeded after the worktree exists so the
|
|
1506
2275
|
// header can name the branch. Idempotent: an existing file stays as-is.
|
|
@@ -1508,6 +2277,7 @@ export class RunManager {
|
|
|
1508
2277
|
if (seeded)
|
|
1509
2278
|
seedHandoffFile(this.dataDir, seeded);
|
|
1510
2279
|
const skills = await discoverSkills(this.repoRoot);
|
|
2280
|
+
state.skills = skills;
|
|
1511
2281
|
const retriesUsed = new Map();
|
|
1512
2282
|
let checkFailure = null;
|
|
1513
2283
|
let runError = null;
|
|
@@ -1687,6 +2457,7 @@ export class RunManager {
|
|
|
1687
2457
|
const startTokens = stepRecord?.tokensUsed ?? 0;
|
|
1688
2458
|
let stepCost = stepRecord?.costUsd ?? 0;
|
|
1689
2459
|
let turnText = '';
|
|
2460
|
+
let sessionError;
|
|
1690
2461
|
const sink = this.makeUiSink(runId, step.id);
|
|
1691
2462
|
const onEvent = (event) => {
|
|
1692
2463
|
if (event.type === 'image') {
|
|
@@ -1703,6 +2474,13 @@ export class RunManager {
|
|
|
1703
2474
|
return;
|
|
1704
2475
|
}
|
|
1705
2476
|
emit({ ...event, stepId: step.id });
|
|
2477
|
+
if (event.type === 'error') {
|
|
2478
|
+
sessionError ??= event.message;
|
|
2479
|
+
state.session?.interrupt();
|
|
2480
|
+
return;
|
|
2481
|
+
}
|
|
2482
|
+
if (sessionError)
|
|
2483
|
+
return;
|
|
1706
2484
|
if (event.type === 'session') {
|
|
1707
2485
|
// Codex/OpenCode mint their own session id — persist it so resume works.
|
|
1708
2486
|
this.store.updateStep(runId, step.id, { sessionId: event.sessionId, backend });
|
|
@@ -1723,13 +2501,17 @@ export class RunManager {
|
|
|
1723
2501
|
const done = interactive && sessionOpen && DONE_MARKER_RE.test(turnText.trimEnd());
|
|
1724
2502
|
// `CEZ:ASK` → the user is blocked; wins over `CEZ:MONITORING`, loses to
|
|
1725
2503
|
// `CEZ:DONE` (#473).
|
|
1726
|
-
const
|
|
2504
|
+
const askResult = interactive && sessionOpen && !done ? parseAskMarkerResult(turnText) : undefined;
|
|
2505
|
+
const ask = askResult?.kind === 'valid' ? askResult.request : null;
|
|
2506
|
+
const askRejection = askResult ? askMarkerRejection(askResult) : undefined;
|
|
1727
2507
|
const monitoring = interactive &&
|
|
1728
2508
|
sessionOpen &&
|
|
1729
2509
|
!done &&
|
|
1730
2510
|
!ask &&
|
|
1731
2511
|
MONITORING_MARKER_RE.test(turnText.trimEnd());
|
|
1732
2512
|
turnText = '';
|
|
2513
|
+
if (askRejection)
|
|
2514
|
+
emit({ type: 'note', stepId: step.id, message: askRejection });
|
|
1733
2515
|
if (done) {
|
|
1734
2516
|
// Goal achieved (agent contract, #347): close the session instead
|
|
1735
2517
|
// of parking at `waiting` — the run completes and frees its slot.
|
|
@@ -1752,15 +2534,25 @@ export class RunManager {
|
|
|
1752
2534
|
if (monitoring) {
|
|
1753
2535
|
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
1754
2536
|
this.store.updateStep(runId, step.id, { status: 'running' });
|
|
2537
|
+
this.monitoring.add(runId);
|
|
2538
|
+
this.clearIdleTimer(state);
|
|
2539
|
+
this.armMonitoringWakeTimer(runId, state);
|
|
1755
2540
|
}
|
|
1756
2541
|
else {
|
|
1757
2542
|
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
1758
2543
|
this.store.updateStep(runId, step.id, { status: 'waiting' });
|
|
2544
|
+
this.monitoring.delete(runId);
|
|
2545
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1759
2546
|
}
|
|
1760
2547
|
this.waiting.add(runId);
|
|
1761
|
-
|
|
2548
|
+
if (!monitoring)
|
|
2549
|
+
this.armIdleTimer(runId, state);
|
|
1762
2550
|
this.releaseSlot(); // the freed slot can start a queued run right away — in any project
|
|
1763
2551
|
}
|
|
2552
|
+
// The window is proven open — see the twin in `runContinuation`.
|
|
2553
|
+
if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
|
|
2554
|
+
this.store.updateRun(runId, { autoResumeAttempts: undefined });
|
|
2555
|
+
}
|
|
1764
2556
|
// Cez's own heartbeat — the handoff stays current even when the
|
|
1765
2557
|
// agent forgets to write (spec 007).
|
|
1766
2558
|
appendHandoffHeartbeat(this.dataDir, runId, `turn complete — status=${monitoring ? 'monitoring' : waiting ? 'waiting' : 'running'}`);
|
|
@@ -1773,7 +2565,7 @@ export class RunManager {
|
|
|
1773
2565
|
// instead of letting the backend silently substitute its default.
|
|
1774
2566
|
let backendModel;
|
|
1775
2567
|
try {
|
|
1776
|
-
const normalized = normalizeModelForBackend(stepBackend, step.model ?? input.model);
|
|
2568
|
+
const normalized = normalizeModelForBackend(stepBackend, agentModelsLocked(this.repoRoot) ? undefined : step.model ?? input.model, { configuredProvider: await configuredModelProvider(stepBackend, state.cwd) });
|
|
1777
2569
|
backendModel = normalized?.backendModel;
|
|
1778
2570
|
// Persist the identity of what ACTUALLY runs (#405, review M1). The run-start echo
|
|
1779
2571
|
// (line ~993) is best-effort from `taskBackend`/`input.model`; a per-step `runner`/`model`
|
|
@@ -1788,8 +2580,17 @@ export class RunManager {
|
|
|
1788
2580
|
return err.message;
|
|
1789
2581
|
throw err;
|
|
1790
2582
|
}
|
|
2583
|
+
// Which agent account this step spawns under, and — recorded on the step before the spawn —
|
|
2584
|
+
// which one its session belongs to. `sessionId` and `profileId` are a pair: a resume that
|
|
2585
|
+
// reads the wrong account's config dir finds no session and silently starts a fresh one.
|
|
2586
|
+
const stepProfile = await this.agentEnvForStep(runId, stepBackend, {
|
|
2587
|
+
generateFollowups: followupsEnabled() && input.generateFollowups !== false,
|
|
2588
|
+
});
|
|
2589
|
+
this.store.updateStep(runId, step.id, { profileId: stepProfile.profileId });
|
|
1791
2590
|
const runner = createRunner(stepBackend);
|
|
1792
2591
|
let session;
|
|
2592
|
+
state.currentStepId = step.id;
|
|
2593
|
+
this.beginUsageInvocation(runId, state, step.id);
|
|
1793
2594
|
try {
|
|
1794
2595
|
session = runner.startSession({
|
|
1795
2596
|
// Skill body, then the run's extra prompt (POST override or config
|
|
@@ -1804,7 +2605,7 @@ export class RunManager {
|
|
|
1804
2605
|
bashAllowlist: step.bashAllowlist,
|
|
1805
2606
|
// The handoff file lives outside the worktree — grant access.
|
|
1806
2607
|
additionalDirectories: [join(this.dataDir, 'runs')],
|
|
1807
|
-
env:
|
|
2608
|
+
env: stepProfile.env,
|
|
1808
2609
|
model: backendModel,
|
|
1809
2610
|
sessionId,
|
|
1810
2611
|
// Interactive sessions have no wall clock — the idle timer rules.
|
|
@@ -1815,6 +2616,7 @@ export class RunManager {
|
|
|
1815
2616
|
});
|
|
1816
2617
|
}
|
|
1817
2618
|
catch (err) {
|
|
2619
|
+
state.currentStepId = undefined;
|
|
1818
2620
|
return err instanceof Error ? err.message : String(err);
|
|
1819
2621
|
}
|
|
1820
2622
|
state.session = session;
|
|
@@ -1826,6 +2628,10 @@ export class RunManager {
|
|
|
1826
2628
|
registerRunProcess(runId, session.pid);
|
|
1827
2629
|
try {
|
|
1828
2630
|
const result = await session.result;
|
|
2631
|
+
if (sessionError) {
|
|
2632
|
+
sink.sessionEnded('error', sessionError);
|
|
2633
|
+
return sessionError;
|
|
2634
|
+
}
|
|
1829
2635
|
// v2 counterpart of v1's `done` (spec: the mappers leave session-close
|
|
1830
2636
|
// events to the RunManager — only it knows how the session settled).
|
|
1831
2637
|
sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
|
|
@@ -1840,6 +2646,9 @@ export class RunManager {
|
|
|
1840
2646
|
finally {
|
|
1841
2647
|
this.recordUsagePeaks(runId);
|
|
1842
2648
|
this.clearIdleTimer(state);
|
|
2649
|
+
this.monitoring.delete(runId);
|
|
2650
|
+
this.waiting.delete(runId);
|
|
2651
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1843
2652
|
state.session = undefined;
|
|
1844
2653
|
state.currentStepId = undefined;
|
|
1845
2654
|
state.interrupt = () => undefined;
|
|
@@ -1863,16 +2672,90 @@ export class RunManager {
|
|
|
1863
2672
|
/** Native backend asks arrive before turn-end. Persist and park immediately
|
|
1864
2673
|
* so the cockpit shows attention and the run releases its workspace slot. */
|
|
1865
2674
|
handleRunnerUiEvent(runId, state, sink, event) {
|
|
2675
|
+
this.recordUsageUiEvent(runId, state, event);
|
|
1866
2676
|
sink.handle(event);
|
|
1867
2677
|
if (event.type !== 'ask.requested' || state.cancelled)
|
|
1868
2678
|
return;
|
|
1869
2679
|
this.clearIdleTimer(state);
|
|
2680
|
+
this.monitoring.delete(runId);
|
|
2681
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
1870
2682
|
this.waiting.add(runId);
|
|
1871
2683
|
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
1872
2684
|
if (state.currentStepId)
|
|
1873
2685
|
this.store.updateStep(runId, state.currentStepId, { status: 'waiting' });
|
|
1874
2686
|
this.releaseSlot();
|
|
1875
2687
|
}
|
|
2688
|
+
/** Persist the invocation checkpoint before launching a runner. A throw or
|
|
2689
|
+
* process exit before `turn.started` therefore leaves a durable mismatch. */
|
|
2690
|
+
beginUsageInvocation(runId, state, stepId) {
|
|
2691
|
+
const step = this.store.getRun(runId)?.steps.find((candidate) => candidate.id === stepId);
|
|
2692
|
+
if (!step)
|
|
2693
|
+
return;
|
|
2694
|
+
const epoch = (step.usageInvocationEpoch ?? 0) + 1;
|
|
2695
|
+
this.persistUsageCheckpoint(runId, stepId, {
|
|
2696
|
+
usageInvocationEpoch: epoch,
|
|
2697
|
+
usageInvocationsStarted: (step.usageInvocationsStarted ?? 0) + 1,
|
|
2698
|
+
});
|
|
2699
|
+
state.usageInvocation = {
|
|
2700
|
+
stepId,
|
|
2701
|
+
epoch,
|
|
2702
|
+
observed: false,
|
|
2703
|
+
startedTurns: new Set(),
|
|
2704
|
+
recordedTurns: new Set(),
|
|
2705
|
+
};
|
|
2706
|
+
}
|
|
2707
|
+
/** Fold backend-neutral completed-turn usage into the current step exactly
|
|
2708
|
+
* once. Invocation/turn counters are written before the event reaches the
|
|
2709
|
+
* NDJSON sink so crashes cannot preserve a falsely complete subtotal. */
|
|
2710
|
+
recordUsageUiEvent(runId, state, event) {
|
|
2711
|
+
const invocation = state.usageInvocation;
|
|
2712
|
+
if (!invocation)
|
|
2713
|
+
return;
|
|
2714
|
+
const step = this.store.getRun(runId)?.steps.find((candidate) => candidate.id === invocation.stepId);
|
|
2715
|
+
if (!step)
|
|
2716
|
+
return;
|
|
2717
|
+
if (event.type === 'turn.started') {
|
|
2718
|
+
if (invocation.startedTurns.has(event.turnId))
|
|
2719
|
+
return;
|
|
2720
|
+
invocation.startedTurns.add(event.turnId);
|
|
2721
|
+
const firstObservedTurn = !invocation.observed;
|
|
2722
|
+
invocation.observed = true;
|
|
2723
|
+
this.persistUsageCheckpoint(runId, invocation.stepId, {
|
|
2724
|
+
usageTurnsStarted: (step.usageTurnsStarted ?? 0) + 1,
|
|
2725
|
+
...(firstObservedTurn
|
|
2726
|
+
? { usageInvocationsObserved: (step.usageInvocationsObserved ?? 0) + 1 }
|
|
2727
|
+
: {}),
|
|
2728
|
+
});
|
|
2729
|
+
return;
|
|
2730
|
+
}
|
|
2731
|
+
if (event.type !== 'turn.completed')
|
|
2732
|
+
return;
|
|
2733
|
+
if (!invocation.startedTurns.has(event.turnId) || invocation.recordedTurns.has(event.turnId))
|
|
2734
|
+
return;
|
|
2735
|
+
const input = event.usage?.input;
|
|
2736
|
+
const output = event.usage?.output;
|
|
2737
|
+
if (typeof input !== 'number' ||
|
|
2738
|
+
!Number.isFinite(input) ||
|
|
2739
|
+
input < 0 ||
|
|
2740
|
+
typeof output !== 'number' ||
|
|
2741
|
+
!Number.isFinite(output) ||
|
|
2742
|
+
output < 0) {
|
|
2743
|
+
return;
|
|
2744
|
+
}
|
|
2745
|
+
invocation.recordedTurns.add(event.turnId);
|
|
2746
|
+
this.persistUsageCheckpoint(runId, invocation.stepId, {
|
|
2747
|
+
inputTokens: (step.inputTokens ?? 0) + input,
|
|
2748
|
+
outputTokens: (step.outputTokens ?? 0) + output,
|
|
2749
|
+
usageTurnsRecorded: (step.usageTurnsRecorded ?? 0) + 1,
|
|
2750
|
+
});
|
|
2751
|
+
}
|
|
2752
|
+
/** Usage completeness is a crash boundary, unlike high-frequency token
|
|
2753
|
+
* snapshots: the checkpoint must reach `runs.json` before the runner starts
|
|
2754
|
+
* or the matching UI event is persisted and forwarded. */
|
|
2755
|
+
persistUsageCheckpoint(runId, stepId, patch) {
|
|
2756
|
+
this.store.updateStep(runId, stepId, patch);
|
|
2757
|
+
this.store.flush();
|
|
2758
|
+
}
|
|
1876
2759
|
/**
|
|
1877
2760
|
* Turn-end bookkeeping (#389), shared by `runAgentStep` and
|
|
1878
2761
|
* `runContinuation` — called (fire-and-forget) from every `turn-end` event:
|
|
@@ -1939,7 +2822,12 @@ export class RunManager {
|
|
|
1939
2822
|
// deliberately NEVER a title source; see maybeRefreshTitle below. The
|
|
1940
2823
|
// one exception is an explicit CEZ:TITLE declaration (applied above).
|
|
1941
2824
|
if (run.worktreePath && existsSync(run.worktreePath)) {
|
|
1942
|
-
|
|
2825
|
+
// `taskBranch` is what keeps this number *this task's* (#751): a review/QA run
|
|
2826
|
+
// repoints the worktree onto the branch under review, and without the branch to
|
|
2827
|
+
// compare HEAD against, the stat would claim that whole branch's diff.
|
|
2828
|
+
const stat = await worktreeShortstat(run.worktreePath, run.baseBranch ?? 'HEAD', {
|
|
2829
|
+
taskBranch: run.branch,
|
|
2830
|
+
});
|
|
1943
2831
|
if (stat)
|
|
1944
2832
|
this.store.updateRun(runId, { diffStat: stat });
|
|
1945
2833
|
else
|
|
@@ -2044,6 +2932,10 @@ export class RunManager {
|
|
|
2044
2932
|
status: review ? 'review' : 'done',
|
|
2045
2933
|
finishedAt: new Date().toISOString(),
|
|
2046
2934
|
currentStepId: undefined,
|
|
2935
|
+
// A run that got all the way to a settled turn is not in a limit loop, so the resume
|
|
2936
|
+
// counter starts over — otherwise a task that legitimately met the limit once a week would
|
|
2937
|
+
// creep toward the cap forever and stop resuming for no reason anyone could see.
|
|
2938
|
+
autoResumeAttempts: undefined,
|
|
2047
2939
|
});
|
|
2048
2940
|
this.store.appendEvent(runId, {
|
|
2049
2941
|
type: 'lifecycle',
|
|
@@ -2096,7 +2988,10 @@ export class RunManager {
|
|
|
2096
2988
|
throw err;
|
|
2097
2989
|
}
|
|
2098
2990
|
this.queuedImageSeq.set(runId, seq);
|
|
2099
|
-
|
|
2991
|
+
// Versioned, because that is the only surface served now. The cockpit still upgrades
|
|
2992
|
+
// the unversioned URLs sitting in OLD transcripts when it renders them
|
|
2993
|
+
// (`resolveApiUrl`), but a URL minted today must be fetchable as written.
|
|
2994
|
+
return { name, url: `/api/v1/runs/${runId}/images/${name}`, path };
|
|
2100
2995
|
}
|
|
2101
2996
|
return null;
|
|
2102
2997
|
}
|
|
@@ -2123,6 +3018,68 @@ export class RunManager {
|
|
|
2123
3018
|
state.idleTimer = undefined;
|
|
2124
3019
|
}
|
|
2125
3020
|
}
|
|
3021
|
+
reconcileMonitoringWakeTimers() {
|
|
3022
|
+
for (const runId of this.monitoring) {
|
|
3023
|
+
const state = this.active.get(runId);
|
|
3024
|
+
if (state)
|
|
3025
|
+
this.armMonitoringWakeTimer(runId, state);
|
|
3026
|
+
}
|
|
3027
|
+
}
|
|
3028
|
+
armMonitoringWakeTimer(runId, state) {
|
|
3029
|
+
const minutes = this.semaphore.monitoringWakeIntervalMinutes();
|
|
3030
|
+
if (minutes === null) {
|
|
3031
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
3032
|
+
return;
|
|
3033
|
+
}
|
|
3034
|
+
if ((state.monitoringWakeups ?? 0) >= MAX_AUTO_CONTINUES) {
|
|
3035
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
3036
|
+
if (!this.store.getRun(runId)?.monitoringWakeCapReached) {
|
|
3037
|
+
this.store.updateRun(runId, { monitoringWakeCapReached: true });
|
|
3038
|
+
this.store.appendEvent(runId, {
|
|
3039
|
+
type: 'note',
|
|
3040
|
+
message: `automatic monitoring wake-up cap reached (${MAX_AUTO_CONTINUES}); session remains parked`,
|
|
3041
|
+
});
|
|
3042
|
+
}
|
|
3043
|
+
return;
|
|
3044
|
+
}
|
|
3045
|
+
if (state.monitoringWakeTimer && state.monitoringWakeIntervalMinutes === minutes)
|
|
3046
|
+
return;
|
|
3047
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
3048
|
+
state.monitoringWakeIntervalMinutes = minutes;
|
|
3049
|
+
this.store.updateRun(runId, { monitoringWakeCapReached: undefined });
|
|
3050
|
+
const deadline = Date.now() + minutes * 60_000;
|
|
3051
|
+
this.store.updateRun(runId, { monitoringWakeAt: new Date(deadline).toISOString() });
|
|
3052
|
+
state.monitoringWakeTimer = setTimeout(() => {
|
|
3053
|
+
state.monitoringWakeTimer = undefined;
|
|
3054
|
+
this.store.updateRun(runId, { monitoringWakeAt: undefined });
|
|
3055
|
+
if (!this.monitoring.has(runId) || !state.session?.open || state.cancelled)
|
|
3056
|
+
return;
|
|
3057
|
+
const wakeups = state.monitoringWakeups ?? 0;
|
|
3058
|
+
if (wakeups >= MAX_AUTO_CONTINUES) {
|
|
3059
|
+
this.store.updateRun(runId, { monitoringWakeCapReached: true });
|
|
3060
|
+
this.store.appendEvent(runId, {
|
|
3061
|
+
type: 'note',
|
|
3062
|
+
message: `automatic monitoring wake-up cap reached (${MAX_AUTO_CONTINUES}); session remains parked`,
|
|
3063
|
+
});
|
|
3064
|
+
return;
|
|
3065
|
+
}
|
|
3066
|
+
state.monitoringWakeups = wakeups + 1;
|
|
3067
|
+
this.store.appendEvent(runId, {
|
|
3068
|
+
type: 'note',
|
|
3069
|
+
message: `automatic monitoring wake-up (${state.monitoringWakeups}/${MAX_AUTO_CONTINUES})`,
|
|
3070
|
+
});
|
|
3071
|
+
this.deliverMessage(runId, [{ type: 'text', text: MONITORING_WAKE_NUDGE }], false);
|
|
3072
|
+
}, Math.max(0, deadline - Date.now()));
|
|
3073
|
+
state.monitoringWakeTimer.unref?.();
|
|
3074
|
+
}
|
|
3075
|
+
clearMonitoringWakeTimer(state, runId) {
|
|
3076
|
+
if (state.monitoringWakeTimer)
|
|
3077
|
+
clearTimeout(state.monitoringWakeTimer);
|
|
3078
|
+
state.monitoringWakeTimer = undefined;
|
|
3079
|
+
state.monitoringWakeIntervalMinutes = undefined;
|
|
3080
|
+
if (runId)
|
|
3081
|
+
this.store.updateRun(runId, { monitoringWakeAt: undefined });
|
|
3082
|
+
}
|
|
2126
3083
|
/** Autosave-commit the worktree every 90 s while the run lives (spec 006).
|
|
2127
3084
|
* Opt-in via CEZ_AUTOSAVE=1 (#471) — see periodicAutosaveEnabled. */
|
|
2128
3085
|
armAutosave(state) {
|
|
@@ -2244,4 +3201,35 @@ export function skillSystemPrompt(skill) {
|
|
|
2244
3201
|
lines.push('', 'Skill instructions:', skill.body.trim());
|
|
2245
3202
|
return lines.join('\n');
|
|
2246
3203
|
}
|
|
3204
|
+
/**
|
|
3205
|
+
* Expand a registry-backed slash skill in a live chat message before it reaches
|
|
3206
|
+
* a backend. Claude otherwise intercepts an unknown leading slash command, and
|
|
3207
|
+
* Codex/OpenCode have no native slash-skill lookup at all (#676).
|
|
3208
|
+
*
|
|
3209
|
+
* Only the first text block is eligible, only at character zero, and unknown
|
|
3210
|
+
* commands pass through byte-for-byte. The caller persists the original user
|
|
3211
|
+
* content before applying this delivery-only rewrite.
|
|
3212
|
+
*/
|
|
3213
|
+
export function expandRegistrySlashSkill(content, skills) {
|
|
3214
|
+
const textIndex = content.findIndex((block) => block.type === 'text');
|
|
3215
|
+
if (textIndex < 0)
|
|
3216
|
+
return content;
|
|
3217
|
+
const block = content[textIndex];
|
|
3218
|
+
if (!block || block.type !== 'text')
|
|
3219
|
+
return content;
|
|
3220
|
+
const match = /^\/([A-Za-z0-9][A-Za-z0-9._-]*)(?=\s|$)/.exec(block.text);
|
|
3221
|
+
if (!match)
|
|
3222
|
+
return content;
|
|
3223
|
+
const skill = skills.find((candidate) => candidate.name === match[1]);
|
|
3224
|
+
if (!skill)
|
|
3225
|
+
return content;
|
|
3226
|
+
const request = block.text.slice(match[0].length).trim();
|
|
3227
|
+
const replacement = {
|
|
3228
|
+
type: 'text',
|
|
3229
|
+
text: request ? `${skillSystemPrompt(skill)}\n\nUser request:\n${request}` : skillSystemPrompt(skill),
|
|
3230
|
+
};
|
|
3231
|
+
const expanded = [...content];
|
|
3232
|
+
expanded[textIndex] = replacement;
|
|
3233
|
+
return expanded;
|
|
3234
|
+
}
|
|
2247
3235
|
//# sourceMappingURL=run.js.map
|