@wjarka/cezarion 0.15.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -46
- package/dist/agent-config/catalog.d.ts +6 -2
- package/dist/agent-config/catalog.js +78 -3
- package/dist/agent-config/catalog.js.map +1 -1
- package/dist/agent-config/model-settings/shared.js +6 -3
- package/dist/agent-config/model-settings/shared.js.map +1 -1
- package/dist/agent-config/models.js +4 -0
- package/dist/agent-config/models.js.map +1 -1
- package/dist/agent-config/validate.js +6 -0
- package/dist/agent-config/validate.js.map +1 -1
- package/dist/artifacts/cli.js +10 -2
- package/dist/artifacts/cli.js.map +1 -1
- package/dist/automations/coordinator.d.ts +7 -0
- package/dist/automations/coordinator.js +22 -2
- package/dist/automations/coordinator.js.map +1 -1
- package/dist/automations/github-poller.d.ts +4 -4
- package/dist/automations/github-poller.js.map +1 -1
- package/dist/automations/schedule-runner.d.ts +130 -0
- package/dist/automations/schedule-runner.js +326 -0
- package/dist/automations/schedule-runner.js.map +1 -0
- package/dist/automations/scheduler.d.ts +38 -7
- package/dist/automations/scheduler.js +90 -22
- package/dist/automations/scheduler.js.map +1 -1
- package/dist/automations/store.d.ts +80 -3
- package/dist/automations/store.js +208 -16
- package/dist/automations/store.js.map +1 -1
- package/dist/automations/task-template.d.ts +42 -6
- package/dist/automations/task-template.js +152 -24
- package/dist/automations/task-template.js.map +1 -1
- package/dist/automations/types.d.ts +142 -6
- package/dist/automations/types.js +64 -6
- package/dist/automations/types.js.map +1 -1
- package/dist/autosave-holders.js +8 -12
- package/dist/autosave-holders.js.map +1 -1
- package/dist/ci-wait/client.d.ts +8 -1
- package/dist/ci-wait/client.js +43 -1
- package/dist/ci-wait/client.js.map +1 -1
- package/dist/ci-wait/controller.d.ts +83 -4
- package/dist/ci-wait/controller.js +76 -11
- package/dist/ci-wait/controller.js.map +1 -1
- package/dist/ci-wait/errors.d.ts +7 -1
- package/dist/ci-wait/errors.js +36 -0
- package/dist/ci-wait/errors.js.map +1 -1
- package/dist/ci-wait/github.d.ts +2 -0
- package/dist/ci-wait/github.js +3 -1
- package/dist/ci-wait/github.js.map +1 -1
- package/dist/ci-wait/mcp.d.ts +2 -98
- package/dist/ci-wait/mcp.js +11 -25
- package/dist/ci-wait/mcp.js.map +1 -1
- package/dist/ci-wait/supervisor.d.ts +2 -1
- package/dist/ci-wait/supervisor.js +2 -1
- package/dist/ci-wait/supervisor.js.map +1 -1
- package/dist/ci-wait/tools.d.ts +307 -0
- package/dist/ci-wait/tools.js +63 -0
- package/dist/ci-wait/tools.js.map +1 -0
- package/dist/config.d.ts +3 -0
- package/dist/config.js +7 -0
- package/dist/config.js.map +1 -1
- package/dist/contract/agent-config.d.ts +5 -0
- package/dist/contract/agent-profiles.d.ts +15 -0
- package/dist/contract/automation-schedule.d.ts +60 -0
- package/dist/contract/automations.d.ts +208 -36
- package/dist/contract/delegation.d.ts +96 -0
- package/dist/contract/discovery.d.ts +7 -0
- package/dist/contract/github.d.ts +1 -0
- package/dist/contract/health.d.ts +6 -0
- package/dist/contract/index.d.ts +3 -0
- package/dist/contract/index.js +1445 -745
- package/dist/contract/preview.d.ts +508 -0
- package/dist/contract/runs.d.ts +716 -15
- package/dist/contract/skills.d.ts +39 -0
- package/dist/contract/workflows.d.ts +6 -0
- package/dist/contract/workspace.d.ts +65 -0
- package/dist/contract/zoned-time.d.ts +35 -0
- package/dist/core/agent-env.d.ts +11 -0
- package/dist/core/agent-env.js +27 -5
- package/dist/core/agent-env.js.map +1 -1
- package/dist/core/agent-profiles.js +7 -0
- package/dist/core/agent-profiles.js.map +1 -1
- package/dist/core/agent-runner.d.ts +49 -1
- package/dist/core/agent-runner.js +2 -1
- package/dist/core/agent-runner.js.map +1 -1
- package/dist/core/backend-detect.d.ts +3 -2
- package/dist/core/backend-detect.js +27 -1
- package/dist/core/backend-detect.js.map +1 -1
- package/dist/core/claude-cli-runner.d.ts +1 -0
- package/dist/core/claude-cli-runner.js +69 -19
- package/dist/core/claude-cli-runner.js.map +1 -1
- package/dist/core/codex-app-server-runner.d.ts +1 -0
- package/dist/core/codex-app-server-runner.js +200 -25
- package/dist/core/codex-app-server-runner.js.map +1 -1
- package/dist/core/codex-app-server-transport.d.ts +7 -3
- package/dist/core/codex-app-server-transport.js +11 -9
- package/dist/core/codex-app-server-transport.js.map +1 -1
- package/dist/core/cursor-acp-runner.d.ts +1 -0
- package/dist/core/cursor-acp-runner.js +24 -9
- package/dist/core/cursor-acp-runner.js.map +1 -1
- package/dist/core/host-model-catalog.js +2 -0
- package/dist/core/host-model-catalog.js.map +1 -1
- package/dist/core/managed-session.js +2 -0
- package/dist/core/managed-session.js.map +1 -1
- package/dist/core/model-identity.js +2 -0
- package/dist/core/model-identity.js.map +1 -1
- package/dist/core/model-presets.js +3 -0
- package/dist/core/model-presets.js.map +1 -1
- package/dist/core/omp-model-catalog.d.ts +43 -0
- package/dist/core/omp-model-catalog.js +205 -0
- package/dist/core/omp-model-catalog.js.map +1 -0
- package/dist/core/omp-runner.d.ts +73 -0
- package/dist/core/omp-runner.js +990 -0
- package/dist/core/omp-runner.js.map +1 -0
- package/dist/core/omp-ui-mapper.d.ts +106 -0
- package/dist/core/omp-ui-mapper.js +882 -0
- package/dist/core/omp-ui-mapper.js.map +1 -0
- package/dist/core/opencode-server-runner.d.ts +18 -4
- package/dist/core/opencode-server-runner.js +337 -96
- package/dist/core/opencode-server-runner.js.map +1 -1
- package/dist/core/opencode-session-error.d.ts +11 -0
- package/dist/core/opencode-session-error.js +26 -0
- package/dist/core/opencode-session-error.js.map +1 -0
- package/dist/core/opencode-ui-mapper.js +4 -36
- package/dist/core/opencode-ui-mapper.js.map +1 -1
- package/dist/core/pi-runner.d.ts +1 -0
- package/dist/core/pi-runner.js +75 -12
- package/dist/core/pi-runner.js.map +1 -1
- package/dist/core/provider-auth.d.ts +9 -2
- package/dist/core/provider-auth.js +43 -6
- package/dist/core/provider-auth.js.map +1 -1
- package/dist/core/runner-factory.js +3 -0
- package/dist/core/runner-factory.js.map +1 -1
- package/dist/core/runner-model-catalog.js +1 -0
- package/dist/core/runner-model-catalog.js.map +1 -1
- package/dist/core/runner-stderr.d.ts +2 -0
- package/dist/core/runner-stderr.js +20 -0
- package/dist/core/runner-stderr.js.map +1 -0
- package/dist/core/session-process.d.ts +22 -0
- package/dist/core/session-process.js +202 -0
- package/dist/core/session-process.js.map +1 -0
- package/dist/core/ui-events.d.ts +1 -1
- package/dist/core/usage-limit.js +3 -60
- package/dist/core/usage-limit.js.map +1 -1
- package/dist/data-gitignore.d.ts +6 -0
- package/dist/data-gitignore.js +57 -0
- package/dist/data-gitignore.js.map +1 -0
- package/dist/delegation/capacity.d.ts +19 -0
- package/dist/delegation/capacity.js +34 -0
- package/dist/delegation/capacity.js.map +1 -0
- package/dist/delegation/cli.js +13 -4
- package/dist/delegation/cli.js.map +1 -1
- package/dist/delegation/conversations.d.ts +4 -2
- package/dist/delegation/conversations.js +8 -12
- package/dist/delegation/conversations.js.map +1 -1
- package/dist/delegation/credentials.d.ts +2 -0
- package/dist/delegation/credentials.js +10 -0
- package/dist/delegation/credentials.js.map +1 -1
- package/dist/delegation/destroy-observation.d.ts +42 -0
- package/dist/delegation/destroy-observation.js +115 -0
- package/dist/delegation/destroy-observation.js.map +1 -0
- package/dist/delegation/execution-identity.d.ts +6 -0
- package/dist/delegation/execution-identity.js +1 -1
- package/dist/delegation/execution-identity.js.map +1 -1
- package/dist/delegation/policy.d.ts +3 -2
- package/dist/delegation/policy.js +7 -5
- package/dist/delegation/policy.js.map +1 -1
- package/dist/delegation/process-liveness.d.ts +57 -20
- package/dist/delegation/process-liveness.js +114 -92
- package/dist/delegation/process-liveness.js.map +1 -1
- package/dist/delegation/provision.js +6 -6
- package/dist/delegation/provision.js.map +1 -1
- package/dist/delegation/results.js +4 -2
- package/dist/delegation/results.js.map +1 -1
- package/dist/delegation/retry-backoff.d.ts +23 -0
- package/dist/delegation/retry-backoff.js +18 -0
- package/dist/delegation/retry-backoff.js.map +1 -0
- package/dist/delegation/routes.d.ts +64 -58
- package/dist/delegation/scratch-cleanup.d.ts +29 -0
- package/dist/delegation/scratch-cleanup.js +143 -0
- package/dist/delegation/scratch-cleanup.js.map +1 -0
- package/dist/delegation/service.d.ts +20 -1
- package/dist/delegation/service.js +233 -70
- package/dist/delegation/service.js.map +1 -1
- package/dist/delegation/transport.d.ts +12 -6
- package/dist/delegation/workspace.d.ts +23 -4
- package/dist/delegation/workspace.js +147 -31
- package/dist/delegation/workspace.js.map +1 -1
- package/dist/git-worktree-lock-helper.js +8 -3
- package/dist/git-worktree-lock-helper.js.map +1 -1
- package/dist/git-worktree-lock.d.ts +6 -1
- package/dist/git-worktree-lock.js +10 -4
- package/dist/git-worktree-lock.js.map +1 -1
- package/dist/git-worktree-release.d.ts +20 -0
- package/dist/git-worktree-release.js +27 -0
- package/dist/git-worktree-release.js.map +1 -0
- package/dist/git-worktree.d.ts +8 -2
- package/dist/git-worktree.js +19 -2
- package/dist/git-worktree.js.map +1 -1
- package/dist/handoff.d.ts +2 -2
- package/dist/handoff.js +1 -1
- package/dist/index.js +109 -76
- package/dist/index.js.map +1 -1
- package/dist/paths.d.ts +1 -1
- package/dist/paths.js +5 -1
- package/dist/paths.js.map +1 -1
- package/dist/preview/cdp.d.ts +13 -0
- package/dist/preview/cdp.js +69 -0
- package/dist/preview/cdp.js.map +1 -0
- package/dist/preview/chromium.d.ts +75 -0
- package/dist/preview/chromium.js +394 -0
- package/dist/preview/chromium.js.map +1 -0
- package/dist/preview/dev-server.d.ts +68 -0
- package/dist/preview/dev-server.js +341 -0
- package/dist/preview/dev-server.js.map +1 -0
- package/dist/preview/host.d.ts +154 -0
- package/dist/preview/host.js +681 -0
- package/dist/preview/host.js.map +1 -0
- package/dist/preview/registration.d.ts +63 -0
- package/dist/preview/registration.js +109 -0
- package/dist/preview/registration.js.map +1 -0
- package/dist/preview/session.d.ts +73 -0
- package/dist/preview/session.js +347 -0
- package/dist/preview/session.js.map +1 -0
- package/dist/preview/url.d.ts +6 -0
- package/dist/preview/url.js +17 -0
- package/dist/preview/url.js.map +1 -0
- package/dist/runs/agent-tmpdir-removal.d.ts +15 -0
- package/dist/runs/agent-tmpdir-removal.js +60 -0
- package/dist/runs/agent-tmpdir-removal.js.map +1 -0
- package/dist/runs/agent-tmpdir.d.ts +12 -1
- package/dist/runs/agent-tmpdir.js +123 -34
- package/dist/runs/agent-tmpdir.js.map +1 -1
- package/dist/runs/delegation-state.d.ts +5 -1
- package/dist/runs/delegation-state.js +9 -0
- package/dist/runs/delegation-state.js.map +1 -1
- package/dist/runs/event-history.d.ts +4 -4
- package/dist/runs/event-history.js +97 -117
- package/dist/runs/event-history.js.map +1 -1
- package/dist/runs/history-compressor.d.ts +29 -0
- package/dist/runs/history-compressor.js +116 -0
- package/dist/runs/history-compressor.js.map +1 -0
- package/dist/runs/history-file.d.ts +36 -0
- package/dist/runs/history-file.js +463 -0
- package/dist/runs/history-file.js.map +1 -0
- package/dist/runs/human-ask-summary.js +18 -13
- package/dist/runs/human-ask-summary.js.map +1 -1
- package/dist/runs/legacy-index.d.ts +62 -0
- package/dist/runs/legacy-index.js +168 -0
- package/dist/runs/legacy-index.js.map +1 -0
- package/dist/runs/raw-record.d.ts +52 -0
- package/dist/runs/raw-record.js +145 -0
- package/dist/runs/raw-record.js.map +1 -0
- package/dist/runs/retention.d.ts +13 -6
- package/dist/runs/retention.js +42 -13
- package/dist/runs/retention.js.map +1 -1
- package/dist/runs/run-claims.d.ts +23 -0
- package/dist/runs/run-claims.js +24 -0
- package/dist/runs/run-claims.js.map +1 -0
- package/dist/runs/run-database.d.ts +385 -0
- package/dist/runs/run-database.js +930 -0
- package/dist/runs/run-database.js.map +1 -0
- package/dist/runs/run-index.d.ts +35 -9
- package/dist/runs/run-index.js +200 -25
- package/dist/runs/run-index.js.map +1 -1
- package/dist/runs/run-row.d.ts +44 -0
- package/dist/runs/run-row.js +84 -0
- package/dist/runs/run-row.js.map +1 -0
- package/dist/runs/run-search.d.ts +29 -0
- package/dist/runs/run-search.js +63 -0
- package/dist/runs/run-search.js.map +1 -0
- package/dist/runs/store-open-error.d.ts +24 -0
- package/dist/runs/store-open-error.js +61 -0
- package/dist/runs/store-open-error.js.map +1 -0
- package/dist/runs/store.d.ts +667 -32
- package/dist/runs/store.js +2153 -324
- package/dist/runs/store.js.map +1 -1
- package/dist/runs/transcript-facts.d.ts +77 -0
- package/dist/runs/transcript-facts.js +302 -0
- package/dist/runs/transcript-facts.js.map +1 -0
- package/dist/runs/webhook.js +5 -4
- package/dist/runs/webhook.js.map +1 -1
- package/dist/runs/worker-execution.d.ts +22 -0
- package/dist/runs/worker-execution.js +30 -0
- package/dist/runs/worker-execution.js.map +1 -0
- package/dist/server/capabilities.d.ts +1 -0
- package/dist/server/capabilities.js +2 -0
- package/dist/server/capabilities.js.map +1 -1
- package/dist/server/cockpit-ownership.d.ts +8 -0
- package/dist/server/cockpit-ownership.js +19 -0
- package/dist/server/cockpit-ownership.js.map +1 -1
- package/dist/server/forge/github.d.ts +1 -4
- package/dist/server/forge/github.js +47 -23
- package/dist/server/forge/github.js.map +1 -1
- package/dist/server/git.js +1 -1
- package/dist/server/git.js.map +1 -1
- package/dist/server/open-in-app.js +1 -0
- package/dist/server/open-in-app.js.map +1 -1
- package/dist/server/preview-socket.d.ts +36 -0
- package/dist/server/preview-socket.js +126 -0
- package/dist/server/preview-socket.js.map +1 -0
- package/dist/server/project-context.d.ts +31 -4
- package/dist/server/project-context.js +73 -9
- package/dist/server/project-context.js.map +1 -1
- package/dist/server/provider-action-gate.js +1 -0
- package/dist/server/provider-action-gate.js.map +1 -1
- package/dist/server/repo-branches.d.ts +4 -3
- package/dist/server/repo-branches.js +5 -4
- package/dist/server/repo-branches.js.map +1 -1
- package/dist/server/server.d.ts +4500 -2727
- package/dist/server/server.js +651 -202
- package/dist/server/server.js.map +1 -1
- package/dist/server/upgrade-router.d.ts +23 -0
- package/dist/server/upgrade-router.js +53 -0
- package/dist/server/upgrade-router.js.map +1 -0
- package/dist/server/validators.d.ts +6 -1
- package/dist/server/validators.js +4 -4
- package/dist/server/validators.js.map +1 -1
- package/dist/server/ws.d.ts +16 -16
- package/dist/server/ws.js +56 -54
- package/dist/server/ws.js.map +1 -1
- package/dist/skills-remote.d.ts +14 -0
- package/dist/skills-remote.js +91 -15
- package/dist/skills-remote.js.map +1 -1
- package/dist/task-cli/cli.js +40 -16
- package/dist/task-cli/cli.js.map +1 -1
- package/dist/task-cli/discovery.d.ts +4 -3
- package/dist/task-cli/discovery.js +4 -3
- package/dist/task-cli/discovery.js.map +1 -1
- package/dist/task-cli/http.d.ts +2 -0
- package/dist/task-cli/http.js +10 -4
- package/dist/task-cli/http.js.map +1 -1
- package/dist/task-cli/projections.d.ts +5 -2
- package/dist/task-cli/projections.js +13 -0
- package/dist/task-cli/projections.js.map +1 -1
- package/dist/task-cli/watch.d.ts +15 -2
- package/dist/task-cli/watch.js +31 -9
- package/dist/task-cli/watch.js.map +1 -1
- package/dist/ui-state.d.ts +4 -3
- package/dist/ui-state.js +8 -3
- package/dist/ui-state.js.map +1 -1
- package/dist/workflows/grouped-command.d.ts +31 -0
- package/dist/workflows/grouped-command.js +143 -0
- package/dist/workflows/grouped-command.js.map +1 -0
- package/dist/workflows/run.d.ts +223 -42
- package/dist/workflows/run.js +1316 -398
- package/dist/workflows/run.js.map +1 -1
- package/dist/workflows/types.d.ts +12 -0
- package/dist/workflows/types.js +20 -0
- package/dist/workflows/types.js.map +1 -1
- package/dist/workspace/agent-accounts.d.ts +5 -1
- package/dist/workspace/agent-accounts.js +1 -0
- package/dist/workspace/agent-accounts.js.map +1 -1
- package/dist/workspace/agent-profiles.js +1 -1
- package/dist/workspace/agent-profiles.js.map +1 -1
- package/dist/workspace/config.d.ts +3 -1
- package/dist/workspace/config.js +1 -0
- package/dist/workspace/config.js.map +1 -1
- package/dist/worktree-setup.d.ts +85 -0
- package/dist/worktree-setup.js +164 -0
- package/dist/worktree-setup.js.map +1 -0
- package/package.json +7 -5
- package/scripts/benchmark-run-store-fixture.ts +428 -0
- package/scripts/benchmark-run-store.ts +743 -0
- package/scripts/benchmark-stuck-destroys.ts +143 -0
- package/scripts/mock-autonomous.mjs +11 -0
- package/scripts/mock-ci-tool.mjs +20 -5
- package/scripts/mock-claude.mjs +110 -1
- package/scripts/mock-codex-app-server.mjs +117 -2
- package/scripts/mock-codex-coalesce-steer.mjs +59 -0
- package/scripts/mock-codex-reverse-steer.mjs +130 -0
- package/scripts/mock-cursor-acp.mjs +75 -1
- package/scripts/mock-omp-rpc.mjs +605 -0
- package/scripts/mock-opencode-serve.mjs +312 -22
- package/scripts/mock-pi-rpc.mjs +109 -3
- package/scripts/mock-runner-crash.mjs +42 -0
- package/scripts/mock-silent-tail.mjs +78 -0
- package/scripts/mock-turn-messages.mjs +5 -0
- package/scripts/omp-ci-wait.mjs +29 -0
- package/scripts/omp-restrict-delegation.yml +4 -0
- package/scripts/pi-ci-wait.mjs +12 -9
- package/scripts/seed-archive-profile.ts +3 -3
- package/web/dist/assets/agent-accounts-CFOBMVHc.js +1 -0
- package/web/dist/assets/alert-dialog-BrpNUwMt.js +1 -0
- package/web/dist/assets/app-window-CSHwDX9N.js +1 -0
- package/web/dist/assets/button-C_rd7CPp.js +43 -0
- package/web/dist/assets/compare-variants-D6WXhvXA.js +1 -0
- package/web/dist/assets/diff-DrEARXTV.js +3 -0
- package/web/dist/assets/diff-controls-B1-j7nFg.js +1 -0
- package/web/dist/assets/{diff-view-DPn6bQ5W.js → diff-view-DvGLjdTk.js} +3 -3
- package/web/dist/assets/dropdown-menu-BegOZFrj.js +1 -0
- package/web/dist/assets/format-DpNe3S0A.js +1 -0
- package/web/dist/assets/git-toolbar-Cw4ljmhB.js +1 -0
- package/web/dist/assets/github-CfV5sqL0.js +1 -0
- package/web/dist/assets/{image-preview-DJ8cgb1Z.js → image-preview-FOEfqpZP.js} +1 -1
- package/web/dist/assets/index-DgWWlOLy.js +13 -0
- package/web/dist/assets/index-fBFnuvxu.css +1 -0
- package/web/dist/assets/{nav-row-styles-3j64kH5s.js → nav-row-styles-vp2JbhuJ.js} +1 -1
- package/web/dist/assets/play-CmVAj_fu.js +1 -0
- package/web/dist/assets/popover-BohP6WTx.js +1 -0
- package/web/dist/assets/preview-pane-connected-qTA5KNoJ.js +3 -0
- package/web/dist/assets/preview-state-vuMviq6q.js +1 -0
- package/web/dist/assets/prompt-templates-Cubdut_C.js +15 -0
- package/web/dist/assets/repo-git-Cf4t5PD0.js +1 -0
- package/web/dist/assets/{run-diff-5-RRdcVz.js → run-diff-BpkUW37Z.js} +3 -3
- package/web/dist/assets/run-header-68WHf9wV.js +1 -0
- package/web/dist/assets/skills-DiVIgKeU.js +1 -0
- package/web/dist/assets/square-terminal-B6RYYieR.js +1 -0
- package/web/dist/assets/status-dot-CqTJHSP9.js +1 -0
- package/web/dist/assets/{tab-link-K0hjzEXR.js → tab-link-B07Hu4QX.js} +1 -1
- package/web/dist/assets/task-changes--lQF6EYf.js +1 -0
- package/web/dist/assets/task-commits-Cx0qy4DN.js +1 -0
- package/web/dist/assets/task-files-D50s-gKW.js +2 -0
- package/web/dist/assets/task-github-item-DaqmSwYT.js +1 -0
- package/web/dist/assets/task-thread-BPQ64lui.js +7 -0
- package/web/dist/assets/{task-thread-Bw_nQuTx.css → task-thread-zNE2Q9E-.css} +1 -1
- package/web/dist/assets/textarea-D20O4Pw0.js +1 -0
- package/web/dist/assets/thread-state-DWcFMrvb.js +4 -0
- package/web/dist/assets/use-hand-to-agent-state-BLIGmwbd.css +1 -0
- package/web/dist/assets/use-hand-to-agent-state-Cy9eyyfL.js +1 -0
- package/web/dist/assets/{utils-fzNfbPzA.js → utils-JsL3ZsJf.js} +1 -1
- package/web/dist/assets/workflows-jwG7uxqi.js +11 -0
- package/web/dist/assets/{zoomable-image-CDmZ_Rtl.js → zoomable-image-BJJ64KGc.js} +1 -1
- package/web/dist/cezarion-lockup-dark.svg +1 -13
- package/web/dist/cezarion-lockup-light.svg +1 -13
- package/web/dist/cezarion-mark-dark.svg +1 -12
- package/web/dist/cezarion-mark-light.svg +1 -12
- package/web/dist/index.html +16 -21
- package/web/dist/assets/agent-accounts-CU2ZDv24.js +0 -1
- package/web/dist/assets/alert-dialog-C-xYE_dg.js +0 -43
- package/web/dist/assets/chevron-right-BENZ-1ZN.js +0 -1
- package/web/dist/assets/collapsible-DD9J91Iv.js +0 -1
- package/web/dist/assets/compare-variants-5FmqpFSN.js +0 -1
- package/web/dist/assets/diff-controls-Bj0YrioE.js +0 -1
- package/web/dist/assets/diff-rE8rm75J2.js +0 -3
- package/web/dist/assets/diff-stat-BKDCSVfJ.js +0 -1
- package/web/dist/assets/dropdown-menu-Ch5BYQh_.js +0 -1
- package/web/dist/assets/format-dBiNDNVe.js +0 -1
- package/web/dist/assets/git-toolbar-ClVvf5iF.js +0 -1
- package/web/dist/assets/github-BkNOyour.css +0 -1
- package/web/dist/assets/github-DeZMeMwJ.js +0 -1
- package/web/dist/assets/github-item-detail-W5F6GVjF.js +0 -1
- package/web/dist/assets/github-sidebar-model-DcMu-IK2.js +0 -1
- package/web/dist/assets/highlighter-CdHqIwFr.js +0 -3
- package/web/dist/assets/index-CatemBEL.js +0 -8
- package/web/dist/assets/index-MvfD8bZJ.css +0 -1
- package/web/dist/assets/markdown-zdKeLIYA.js +0 -2
- package/web/dist/assets/mobile-run-bar-BCfmFi8V.js +0 -1
- package/web/dist/assets/prompt-templates-CBPp39IU.js +0 -15
- package/web/dist/assets/reference-chip-D6owncJT.js +0 -1
- package/web/dist/assets/repo-git-CtdkZFey.js +0 -1
- package/web/dist/assets/run-header-BcjoDi0W.js +0 -1
- package/web/dist/assets/sheet-ChnYVrp1.js +0 -1
- package/web/dist/assets/skills-C8YKp2vr.js +0 -1
- package/web/dist/assets/task-changes-BW8YdWG8.js +0 -1
- package/web/dist/assets/task-commits-q2kB2y6L.js +0 -1
- package/web/dist/assets/task-files-pPL6_3Cr.js +0 -2
- package/web/dist/assets/task-github-item-BXF5R52U.js +0 -1
- package/web/dist/assets/task-thread-DtkFL3EC.js +0 -10
- package/web/dist/assets/textarea-DxlkE-Yd.js +0 -1
- package/web/dist/assets/token-metrics-DIAjrPlM.js +0 -1
- package/web/dist/assets/use-desktop-CrDny2uG.js +0 -1
- package/web/dist/assets/use-submit-shortcut-KWJsC906.js +0 -1
- package/web/dist/assets/workflows-TmcPEj2K.js +0 -11
- /package/web/dist/assets/{lib-BQXq3kEf2.js → lib-BQXq3kEf.js} +0 -0
package/dist/runs/store.js
CHANGED
|
@@ -1,23 +1,34 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
-
import {
|
|
2
|
+
import { workerEvidenceRunIds, workerExecutionSchema } from './worker-execution.js';
|
|
3
|
+
import { agentTmpDirLocations, agentTmpDirMayExist, agentTmpDirOwnershipProven, removeAgentTmpDir } from './agent-tmpdir.js';
|
|
3
4
|
import { removeArtifacts } from '../artifacts/lifecycle.js';
|
|
4
5
|
import { randomUUID } from 'node:crypto';
|
|
5
6
|
import { isDeepStrictEqual } from 'node:util';
|
|
6
|
-
import { appendFileSync, closeSync, constants, fstatSync, fsyncSync, lstatSync, openSync, readSync, realpathSync, readdirSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync,
|
|
7
|
+
import { appendFileSync, closeSync, constants, fstatSync, fsyncSync, lstatSync, openSync, readSync, realpathSync, readdirSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
7
8
|
import { join, resolve } from 'node:path';
|
|
8
9
|
import { z } from 'zod';
|
|
9
|
-
import { ciWaitSchema, agentInputSchema, inboxClaimSchema, delegationStateSchema, workerCreationReceiptSchema, workerCollectedResultSchema, workerResultFileSchema, continuationMessageSchema, runRecordSchema as contractRunRecordSchema, } from '../contract/index.js';
|
|
10
|
+
import { ciWaitSchema, agentInputSchema, inboxClaimSchema, delegationStateSchema, workerCreationReceiptSchema, workerCollectedResultSchema, workerResultFileSchema, continuationMessageSchema, previewServerSchema, toRunSummary, workerDestroyRetrySchema, runRecordSchema as contractRunRecordSchema, } from '../contract/index.js';
|
|
11
|
+
import { matchesRunQuery, sqlPrefilterTokens } from './run-search.js';
|
|
10
12
|
import { storedDelegationStateSchema } from './delegation-state.js';
|
|
11
|
-
import {
|
|
13
|
+
import { HistoryCompressor } from './history-compressor.js';
|
|
14
|
+
import { emptyFacts, stampOf, TranscriptFactsIndex } from './transcript-facts.js';
|
|
15
|
+
import { hasPlainHistory, historyPaths, readHistoryText, readHistoryTextAsync, removeHistory, restoreHistory } from './history-file.js';
|
|
12
16
|
import { workerExecutionIdentitySchema } from '../delegation/execution-identity.js';
|
|
13
17
|
import { reconcileWorkerWait } from '../delegation/wait.js';
|
|
14
|
-
import {
|
|
18
|
+
import { capacityError, workerCapacity } from '../delegation/capacity.js';
|
|
19
|
+
import { inspectGeneration, isCurrentProcess, recordedProcessLive, processStartToken } from '../delegation/process-liveness.js';
|
|
15
20
|
import { collectSecretValues, redactDeep, redactSecrets } from '../core/secret-redaction.js';
|
|
16
21
|
// Pure, dependency-free reference helpers — the same sanity bound the marker parser applies.
|
|
17
22
|
import { MAX_REF } from './task-refs.js';
|
|
18
23
|
// Type-only module (zod + nothing else), so this cannot cycle back into the store.
|
|
19
24
|
import { workflowDefSchema } from '../workflows/types.js';
|
|
20
25
|
import { RUNNER_IDS } from '../core/agent-runner.js';
|
|
26
|
+
import { ARCHIVED_ROOT, RUNS_DB_FILE, RUNS_IMPORT_COMPLETE_KEY, RunConflictError, RunDatabase, RunDatabaseBusyError, } from './run-database.js';
|
|
27
|
+
import { encodeRunRow, isLiveRecord, isLiveStatus } from './run-row.js';
|
|
28
|
+
import { collectRawExtras, encodeRawRecord } from './raw-record.js';
|
|
29
|
+
import { assertNoLegacyCockpit, assertNoLegacyWriter, backUpLegacyIndex, LEGACY_INDEX_FILE, LegacyWriterError, readLegacyIndex } from './legacy-index.js';
|
|
30
|
+
import { toRunStoreOpenError } from './store-open-error.js';
|
|
31
|
+
import { claimOwnerLive, closeClaimSession, openClaimSession } from './run-claims.js';
|
|
21
32
|
const usageCounterSchema = z.number().finite().nonnegative();
|
|
22
33
|
/**
|
|
23
34
|
* A runner id as it may appear in a PERSISTED record, normalized to the three
|
|
@@ -88,8 +99,8 @@ const queuedMessageSchema = z.object({
|
|
|
88
99
|
images: z.array(z.string()).optional(),
|
|
89
100
|
createdAt: z.string(),
|
|
90
101
|
});
|
|
91
|
-
/** Exported for `./run-index.ts`, the read-only reader of the same
|
|
92
|
-
* parse
|
|
102
|
+
/** Exported for `./run-index.ts`, the read-only reader of the same records. Nothing else should
|
|
103
|
+
* parse a stored run — see `reconcileLoadedRun` for why a second parser is a correctness risk. */
|
|
93
104
|
export const runRecordSchema = z.object({
|
|
94
105
|
id: z.string(),
|
|
95
106
|
title: z.string(),
|
|
@@ -131,6 +142,8 @@ export const runRecordSchema = z.object({
|
|
|
131
142
|
lastCiWait: ciWaitSchema.optional(),
|
|
132
143
|
/** Retained recovery observation when previous CI metadata cannot be trusted. */
|
|
133
144
|
lastCiWaitError: z.string().max(256).optional(),
|
|
145
|
+
/** Dev servers the agent registered with `cezar_preview_serve` (#781). Optional so old files parse. */
|
|
146
|
+
previewServers: z.array(previewServerSchema).optional(),
|
|
134
147
|
/** URLs of images attached to the initial task prompt, for the thread's first bubble
|
|
135
148
|
* (#image-display) — persisted like agent screenshots, served from `/images/`. */
|
|
136
149
|
taskImages: z.array(z.string()).optional(),
|
|
@@ -185,6 +198,19 @@ export const runRecordSchema = z.object({
|
|
|
185
198
|
githubUrl: z.string().url(),
|
|
186
199
|
})
|
|
187
200
|
.optional(),
|
|
201
|
+
/** Provenance for a task a scheduled automation launched. Its own key so a pre-schedule cezar
|
|
202
|
+
* strips it instead of failing the whole index on a missing `githubUrl`. */
|
|
203
|
+
automationTrigger: z
|
|
204
|
+
.object({
|
|
205
|
+
automationId: z.string(),
|
|
206
|
+
automationRevision: z.number().int().positive(),
|
|
207
|
+
receiptId: z.string(),
|
|
208
|
+
trigger: z.enum(['schedule', 'catch-up', 'manual']),
|
|
209
|
+
occurrenceAt: z.string(),
|
|
210
|
+
})
|
|
211
|
+
.optional(),
|
|
212
|
+
/** A pending worker destroy's automatic retry state (hearsay-tools/cezarion#879); a malformed one is dropped, never the run. */
|
|
213
|
+
destroyRetry: contractRunRecordSchema.shape.destroyRetry.catch(undefined),
|
|
188
214
|
status: z.enum(['queued', 'running', 'waiting', 'review', 'done', 'failed', 'cancelled']),
|
|
189
215
|
stopping: contractRunRecordSchema.shape.stopping.catch(undefined),
|
|
190
216
|
/** Sub-state of `running` (spec 2026-07-18-subagent-monitoring-status, #490):
|
|
@@ -266,6 +292,8 @@ export const runRecordSchema = z.object({
|
|
|
266
292
|
* dir gone, recoverable via `git worktree add`"; it excludes the run from the
|
|
267
293
|
* retention budget until the dir is re-materialized (resume clears it). */
|
|
268
294
|
worktreeReclaimedAt: z.string().optional(),
|
|
295
|
+
/** Worktree setup (#917). An unreadable value drops the field, never the run. */
|
|
296
|
+
worktreeSetup: contractRunRecordSchema.shape.worktreeSetup.catch(undefined),
|
|
269
297
|
/** Parallel variants (spec 010): tasks sharing a groupId are one group. */
|
|
270
298
|
groupId: z.string().optional(),
|
|
271
299
|
/** Variant letter within the group — 'A' | 'B' | 'C' (kept as a string). */
|
|
@@ -337,11 +365,65 @@ export function parseRunRecords(raw) {
|
|
|
337
365
|
delete row.lastCiWait;
|
|
338
366
|
row.lastCiWaitError = 'CI wait unavailable — saved observation is unreadable; register a new wait.';
|
|
339
367
|
}
|
|
368
|
+
// One unreadable registration must not evict the run, nor its readable siblings (#781).
|
|
369
|
+
if (row.previewServers !== undefined) {
|
|
370
|
+
if (Array.isArray(row.previewServers))
|
|
371
|
+
row.previewServers = row.previewServers.filter((entry) => previewServerSchema.safeParse(entry).success);
|
|
372
|
+
else
|
|
373
|
+
delete row.previewServers;
|
|
374
|
+
}
|
|
340
375
|
}
|
|
341
376
|
return z.array(runRecordSchema).safeParse(raw);
|
|
342
377
|
}
|
|
378
|
+
/** Just the `StoredRow` part of a `coldBase` entry. */
|
|
379
|
+
function storedRow({ revision, seq, data, extras }) {
|
|
380
|
+
return { revision, seq, data, extras };
|
|
381
|
+
}
|
|
382
|
+
/** One stored record (a row's `data`) through `parseRunRecords`' salvage and schema, with what the
|
|
383
|
+
* schema dropped. Read after the salvage ran: what it removes (an unreadable CI wait, a preview
|
|
384
|
+
* server entry) the runtime has decided to drop, and keeps dropping on the next write, as it always
|
|
385
|
+
* has; only what the schema does not know is put back. */
|
|
386
|
+
function decodeStoredRecord(raw) {
|
|
387
|
+
const parsed = parseRunRecords([raw]);
|
|
388
|
+
if (!parsed.success)
|
|
389
|
+
return undefined;
|
|
390
|
+
const run = parsed.data[0];
|
|
391
|
+
return { run, extras: collectRawExtras(raw, run) };
|
|
392
|
+
}
|
|
393
|
+
/** One database row's `data` back into a record and its extras. Undefined when it does not parse:
|
|
394
|
+
* one unreadable row costs that row only. */
|
|
395
|
+
export function decodeRunRow(data) {
|
|
396
|
+
try {
|
|
397
|
+
return decodeStoredRecord(JSON.parse(data));
|
|
398
|
+
}
|
|
399
|
+
catch {
|
|
400
|
+
return undefined;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
/** One database row's `data` back into a record, for a reader that never writes it back. */
|
|
404
|
+
export function decodeRunRecord(data) {
|
|
405
|
+
try {
|
|
406
|
+
const parsed = parseRunRecords([JSON.parse(data)]);
|
|
407
|
+
return parsed.success ? parsed.data[0] : undefined;
|
|
408
|
+
}
|
|
409
|
+
catch {
|
|
410
|
+
return undefined;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
export { LEGACY_INDEX_BACKUP_FILE, LEGACY_INDEX_FILE } from './legacy-index.js';
|
|
414
|
+
/** The pauses between open attempts while the database is busy, when `open` may wait (a boot):
|
|
415
|
+
* about 3.5 s in all, with each attempt's own busy timeout. Long enough for another process to
|
|
416
|
+
* finish importing a large `runs.json`. */
|
|
417
|
+
const OPEN_RETRY_DELAYS_MS = [50, 100, 200, 400, 800, 1600];
|
|
418
|
+
/** Block this thread for `ms`. Only `open` does, at boot, when there is nothing else to run. */
|
|
419
|
+
function sleepSync(ms) {
|
|
420
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
421
|
+
}
|
|
343
422
|
const MAX_RUNS_KEPT = 300;
|
|
344
|
-
|
|
423
|
+
/** How many archived root runs the run lists carry per project (#864): `GET /run-summaries?archived=recent`
|
|
424
|
+
* and `GET /workspace/runs-index`. Unarchived runs are never cut; older archived runs page in
|
|
425
|
+
* through `GET /run-summaries/archived` and the workspace search. */
|
|
426
|
+
export const ARCHIVED_WINDOW = 200;
|
|
345
427
|
const PR_URL_RE = /https:\/\/github\.com\/[^/\s]+\/[^/\s]+\/pull\/\d+/;
|
|
346
428
|
const ISSUE_URL_RE = /https:\/\/github\.com\/[^/\s]+\/[^/\s]+\/issues\/\d+/;
|
|
347
429
|
// The transcript auto-link is convenience only (the cockpit's own `gh pr create` path sets the
|
|
@@ -677,6 +759,25 @@ export function reconcileLoadedRun(run, opts) {
|
|
|
677
759
|
// also scopes any referenced PR restored above without filtering candidate evidence first.
|
|
678
760
|
return run;
|
|
679
761
|
}
|
|
762
|
+
/**
|
|
763
|
+
* Settle a live run whose owning process died, when a control adopts it (#779, plan step 3), so
|
|
764
|
+
* that adopting never starts agent work: only Continue may, with the user's own input. A run that
|
|
765
|
+
* never started is cancelled before it began, which Continue restarts as it would after a Stop (the
|
|
766
|
+
* same "untouched" test as `isUntouchedCancelledRun`); any other live run is interrupted, exactly as
|
|
767
|
+
* an open that does not recover settles it, and Continue resumes its last session. With `stop`
|
|
768
|
+
* (the control is Stop, and this is its run) a live run is cancelled instead, as an accepted Stop
|
|
769
|
+
* always reads. Mutates `run`.
|
|
770
|
+
*/
|
|
771
|
+
export function settleOrphanedRun(run, opts = {}) {
|
|
772
|
+
if (opts.stop && (run.status === 'queued' || run.status === 'running' || run.status === 'waiting'))
|
|
773
|
+
run.stopping = true;
|
|
774
|
+
if (run.status === 'queued' && !run.startedAt && run.workflowDef !== undefined &&
|
|
775
|
+
run.steps.every((step) => step.status === 'pending' && !step.startedAt && !step.sessionId)) {
|
|
776
|
+
run.status = 'cancelled';
|
|
777
|
+
run.finishedAt ??= new Date().toISOString();
|
|
778
|
+
}
|
|
779
|
+
return reconcileLoadedRun(run, { keepLive: false });
|
|
780
|
+
}
|
|
680
781
|
/**
|
|
681
782
|
* Drop this run's referenced PR/issue if the project's handle proves it foreign and the prompt
|
|
682
783
|
* does not corroborate it (#945). Returns whether anything changed.
|
|
@@ -707,54 +808,877 @@ export function rescopeRun(run, handle) {
|
|
|
707
808
|
}
|
|
708
809
|
return changed;
|
|
709
810
|
}
|
|
811
|
+
/**
|
|
812
|
+
* Every field `RunStore.open` may rewrite while loading a record: what `reconcileLoadedRun` and
|
|
813
|
+
* `refreshHumanAskSummary` assign. Compared before and after, it tells open() which rows
|
|
814
|
+
* normalization changed, so it marks those dirty and no others. Edit it together with either
|
|
815
|
+
* function, or a normalized row stays unsaved (the store tests pin each field).
|
|
816
|
+
*/
|
|
817
|
+
function loadNormalizedFields(run) {
|
|
818
|
+
return JSON.stringify([
|
|
819
|
+
run.stopping, run.status, run.finishedAt, run.error, run.activity, run.monitoringWakeAt, run.autoResumeAt,
|
|
820
|
+
run.monitoringWakeCapReached, run.referencedPullRequestUrl, run.hasPendingHumanAsk, run.steps.map((step) => step.status),
|
|
821
|
+
]);
|
|
822
|
+
}
|
|
823
|
+
/** Test seam (#779): `beforeTransaction` runs once `runs.json` is read and parsed, before the
|
|
824
|
+
* import asks for the write lock; `beforeCommit` runs inside the import transaction, after every
|
|
825
|
+
* row is written and before the last check that no older cezar wrote `runs.json` meanwhile. */
|
|
826
|
+
let legacyImportHook;
|
|
827
|
+
export function __setLegacyImportHookForTests(hook) {
|
|
828
|
+
legacyImportHook = hook;
|
|
829
|
+
}
|
|
830
|
+
/** `runs.json`'s records as rows, and how many of its entries were left out. Each row's `data` is
|
|
831
|
+
* the record's own JSON as `runs.json` held it, so nothing this cezar's schema does not know is lost
|
|
832
|
+
* (raw-record.ts); its summary and columns come from the parsed record, which is what decoding that
|
|
833
|
+
* JSON gives back. An entry that does not parse is skipped and costs that entry only, as an
|
|
834
|
+
* unreadable row does once it is in the database. Undefined when the file is not a JSON array. */
|
|
835
|
+
function legacyIndexRows(bytes) {
|
|
836
|
+
let raw;
|
|
837
|
+
try {
|
|
838
|
+
raw = JSON.parse(bytes.toString('utf8'));
|
|
839
|
+
}
|
|
840
|
+
catch {
|
|
841
|
+
return undefined;
|
|
842
|
+
}
|
|
843
|
+
if (!Array.isArray(raw))
|
|
844
|
+
return undefined;
|
|
845
|
+
// A hand-edited index may repeat an id; the last one won when it loaded into a map, so it still does.
|
|
846
|
+
const rows = new Map();
|
|
847
|
+
let skipped = 0;
|
|
848
|
+
for (const entry of raw) {
|
|
849
|
+
// `parseRunRecords`' salvage rewrites top-level keys: the stored JSON is taken from a copy.
|
|
850
|
+
const stored = entry !== null && typeof entry === 'object' && !Array.isArray(entry) ? { ...entry } : entry;
|
|
851
|
+
const parsed = parseRunRecords([entry]);
|
|
852
|
+
if (!parsed.success) {
|
|
853
|
+
skipped += 1;
|
|
854
|
+
continue;
|
|
855
|
+
}
|
|
856
|
+
const run = parsed.data[0];
|
|
857
|
+
rows.set(run.id, encodeRunRow(run, JSON.stringify(stored)));
|
|
858
|
+
}
|
|
859
|
+
return { rows: [...rows.values()], skipped, total: raw.length };
|
|
860
|
+
}
|
|
861
|
+
/**
|
|
862
|
+
* Import `runs.json` into a database that has never completed an import (#779, plan step 4).
|
|
863
|
+
*
|
|
864
|
+
* Nothing is read while a live cockpit other than this process owns the project
|
|
865
|
+
* (`assertNoLegacyCockpit`): an older cezar still writes `runs.json` then, and the import could
|
|
866
|
+
* not commit. Otherwise the file is read once, through one descriptor, and parsed; it is checked
|
|
867
|
+
* again before anything is written, so a file that moved while it was parsed costs no write. Then
|
|
868
|
+
* ONE transaction writes every record and the completion marker, or nothing:
|
|
869
|
+
* - the marker is checked again once the transaction holds the write lock, so of two processes
|
|
870
|
+
* importing at once exactly one writes; the other finds the marker and writes nothing;
|
|
871
|
+
* - right before COMMIT it refuses (`LegacyWriterError`) when an older cezar may still be writing
|
|
872
|
+
* `runs.json` (`assertNoLegacyWriter`) — the check that decides;
|
|
873
|
+
* - only then are the exact bytes kept beside it (`backUpLegacyIndex`: synced, never half-written,
|
|
874
|
+
* an existing backup trusted only when it holds the same bytes), and the marker records what
|
|
875
|
+
* the import read: size, sha256 and the backup's name. A refused attempt leaves no backup;
|
|
876
|
+
* - a crash or refusal leaves no marker, so the next open imports again from the start.
|
|
877
|
+
* `runs.json` itself is left as it was, and nothing writes it again: an older cezar keeps reading
|
|
878
|
+
* the history as it stood at the upgrade, and what it writes afterwards is never imported.
|
|
879
|
+
*
|
|
880
|
+
* A record that does not parse is skipped and the rest are imported; an index that does not parse
|
|
881
|
+
* at all imports as an empty history. Either way the import that commits says so once, naming the
|
|
882
|
+
* backup: the bytes survive there and in `runs.json`, which used to be overwritten by the next save
|
|
883
|
+
* instead.
|
|
884
|
+
*/
|
|
885
|
+
function importLegacyIndex(db, dataDir) {
|
|
886
|
+
assertNoLegacyCockpit(dataDir);
|
|
887
|
+
const snapshot = readLegacyIndex(join(dataDir, LEGACY_INDEX_FILE));
|
|
888
|
+
const parsed = snapshot ? legacyIndexRows(snapshot.bytes) : undefined;
|
|
889
|
+
const rows = parsed?.rows ?? [];
|
|
890
|
+
const skipped = parsed?.skipped ?? 0;
|
|
891
|
+
const source = !snapshot ? 'none' : parsed ? LEGACY_INDEX_FILE : `${LEGACY_INDEX_FILE} (unparseable)`;
|
|
892
|
+
assertNoLegacyWriter(dataDir, snapshot);
|
|
893
|
+
legacyImportHook?.beforeTransaction?.();
|
|
894
|
+
let backup;
|
|
895
|
+
const commit = db.transaction({
|
|
896
|
+
upserts: rows,
|
|
897
|
+
deletes: [],
|
|
898
|
+
onlyIfMetaAbsent: RUNS_IMPORT_COMPLETE_KEY,
|
|
899
|
+
beforeCommit: () => {
|
|
900
|
+
legacyImportHook?.beforeCommit?.();
|
|
901
|
+
assertNoLegacyWriter(dataDir, snapshot);
|
|
902
|
+
backup = snapshot ? backUpLegacyIndex(dataDir, snapshot) : undefined;
|
|
903
|
+
const read = snapshot ? { bytes: snapshot.bytes.length, sha256: snapshot.sha256, backup } : {};
|
|
904
|
+
return { [RUNS_IMPORT_COMPLETE_KEY]: JSON.stringify({ at: new Date().toISOString(), source, records: rows.length, ...(skipped > 0 ? { skipped } : {}), ...read }) };
|
|
905
|
+
},
|
|
906
|
+
});
|
|
907
|
+
// Told once, by the import that committed: the marker means no later open reads runs.json again.
|
|
908
|
+
if (commit.skipped || backup === undefined)
|
|
909
|
+
return;
|
|
910
|
+
const kept = `Its exact bytes are kept in ${join(dataDir, backup)}; BACKWARD_COMPATIBILITY.md §3 says how to import it after a repair.`;
|
|
911
|
+
if (!parsed)
|
|
912
|
+
console.warn(`[cez] ${LEGACY_INDEX_FILE} could not be read; no run was imported, and this project starts with an empty history. ${kept}`);
|
|
913
|
+
else if (skipped > 0)
|
|
914
|
+
console.warn(`[cez] ${skipped} of ${parsed.total} runs in ${LEGACY_INDEX_FILE} could not be read and were not imported. ${kept}`);
|
|
915
|
+
}
|
|
916
|
+
/**
|
|
917
|
+
* Finished rows whose stored summary an open-time normalization would still change: a referenced
|
|
918
|
+
* PR an older cezar's created-PR declaration erased (see `reconcileLoadedRun`). Not live, so not
|
|
919
|
+
* in the `live` column; open decodes these few and keeps the ones it actually repaired.
|
|
920
|
+
*/
|
|
921
|
+
const LEGACY_REFERENCE_HEAL_SQL = "json_extract(summary, '$.markerRefs.pr') IS NOT NULL AND json_extract(summary, '$.referencedPullRequestUrl') IS NULL";
|
|
922
|
+
/** Mark-all-read's rule (see `RunStore.markAllRead`) over the columns and the stored summary. */
|
|
923
|
+
const MARK_ALL_READ_SQL = "archived = 0 AND status IN ('done', 'failed') AND finished_at IS NOT NULL" +
|
|
924
|
+
" AND NOT (status = 'failed' AND json_extract(summary, '$.autoResumeAt') IS NOT NULL)" +
|
|
925
|
+
" AND (json_extract(summary, '$.seenAt') IS NULL OR json_extract(summary, '$.seenAt') < finished_at)";
|
|
926
|
+
/** History retention's candidates among the given ids (a JSON array): finished, without
|
|
927
|
+
* delegation, and not pinned. */
|
|
928
|
+
const RETENTION_CANDIDATES_SQL = "id IN (SELECT value FROM json_each(?)) AND status NOT IN ('queued', 'running', 'waiting')" +
|
|
929
|
+
" AND parent_run_id IS NULL AND json_extract(summary, '$.delegation') IS NULL" +
|
|
930
|
+
" AND json_extract(summary, '$.pinned') IS NOT 1";
|
|
931
|
+
/** Rows whose stored summary names a referenced PR or issue: what a repository handle can veto. */
|
|
932
|
+
const REFERENCED_SQL = "json_extract(summary, '$.referencedPullRequestUrl') IS NOT NULL OR json_extract(summary, '$.referencedIssueUrl') IS NOT NULL";
|
|
933
|
+
/** A record as a branch owner, or undefined when it owns no branch and created no PR: the
|
|
934
|
+
* projection `listBranchOwners` reads off a row's summary and columns, computed from memory. */
|
|
935
|
+
export function branchOwnerOf(run) {
|
|
936
|
+
const branch = encodeRunRow(run).branch;
|
|
937
|
+
if (!branch && run.pullRequestUrl === undefined)
|
|
938
|
+
return undefined;
|
|
939
|
+
return {
|
|
940
|
+
id: run.id, title: run.title, status: run.status, archived: run.archived, createdAt: run.createdAt,
|
|
941
|
+
...(branch ? { branch } : {}),
|
|
942
|
+
...(run.baseBranch === undefined ? {} : { baseBranch: run.baseBranch }),
|
|
943
|
+
...(run.pullRequestUrl === undefined ? {} : { pullRequestUrl: run.pullRequestUrl }),
|
|
944
|
+
};
|
|
945
|
+
}
|
|
946
|
+
/** The one answer every control gives for a run another cezar process owns (`409 { error }`). */
|
|
947
|
+
export const RUN_IN_USE_ELSEWHERE = 'run is in use by another cezar process';
|
|
948
|
+
/** The answer for a run this process stopped writing after a conflicting write. */
|
|
949
|
+
export const RUN_QUARANTINED = 'run changed under this cezar process — restart cezar to reload it';
|
|
950
|
+
/** A write this store must not make: another process owns the run, its owner died with work
|
|
951
|
+
* still live (adopt it first), or a conflict quarantined it. Nothing was changed. */
|
|
952
|
+
export class RunWriteRefusedError extends Error {
|
|
953
|
+
runId;
|
|
954
|
+
ownership;
|
|
955
|
+
constructor(runId, ownership) {
|
|
956
|
+
super(ownership === 'quarantined' ? `${RUN_QUARANTINED}: ${runId}`
|
|
957
|
+
: ownership === 'orphaned' ? `run ${runId} belongs to a cezar process that exited; take it over before writing it`
|
|
958
|
+
: `${RUN_IN_USE_ELSEWHERE}: ${runId}`);
|
|
959
|
+
this.runId = runId;
|
|
960
|
+
this.ownership = ownership;
|
|
961
|
+
this.name = 'RunWriteRefusedError';
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
/** How many times open (or an adoption) tries to take its claims while another connection holds
|
|
965
|
+
* the write lock, each waiting out the 50 ms busy timeout: recovery cannot start without them. */
|
|
966
|
+
const ADOPT_CLAIM_ATTEMPTS = 10;
|
|
967
|
+
/** The `run_claims` family of a run: a worker's parent, else the run itself (`parent_run_id ?? id`). */
|
|
968
|
+
function familyKey(run) {
|
|
969
|
+
return run.delegation?.role === 'worker' ? run.delegation.parentRunId : run.id;
|
|
970
|
+
}
|
|
971
|
+
/** The same family, off a stored row's columns. */
|
|
972
|
+
function rowFamily(row) {
|
|
973
|
+
return row.parentRunId ?? row.id;
|
|
974
|
+
}
|
|
975
|
+
/** Make `target` hold exactly `source`'s fields, keeping the object every holder references. */
|
|
976
|
+
function replaceRecord(target, source) {
|
|
977
|
+
for (const key of Object.keys(target))
|
|
978
|
+
if (!(key in source))
|
|
979
|
+
delete target[key];
|
|
980
|
+
Object.assign(target, source);
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* A worker's destroy moving between two unfinished phases, and nothing else changing: nothing the
|
|
984
|
+
* family reconcile reads changed, so it may skip that pass (#880). The first request and the
|
|
985
|
+
* `complete` phase still reconcile: one changes delivery eligibility, the other settles requests.
|
|
986
|
+
*/
|
|
987
|
+
function isDestroyProgress(before, after) {
|
|
988
|
+
if (before?.role !== 'worker' || after.role !== 'worker' || !before.destroy || !after.destroy)
|
|
989
|
+
return false;
|
|
990
|
+
if (before.destroy.phase === 'complete' || after.destroy.phase === 'complete')
|
|
991
|
+
return false;
|
|
992
|
+
return isDeepStrictEqual({ ...before, destroy: undefined }, { ...after, destroy: undefined });
|
|
993
|
+
}
|
|
994
|
+
/**
|
|
995
|
+
* Two collected worker results carry the same evidence: equal but for when they were observed and
|
|
996
|
+
* which snapshot file holds them, and, with `ignoreCleanup`, the cleanup phase. A retried destroy
|
|
997
|
+
* that observed nothing new then rewrites and fsyncs nothing (hearsay-tools/cezarion#879).
|
|
998
|
+
*/
|
|
999
|
+
function sameWorkerResult(a, b, opts) {
|
|
1000
|
+
const comparable = (value) => ({ ...value, observedAt: undefined, ...(opts.ignoreCleanup ? { cleanup: undefined } : {}),
|
|
1001
|
+
diff: value.diff.state === 'available' ? { ...value.diff, snapshotId: undefined, path: undefined } : value.diff });
|
|
1002
|
+
return isDeepStrictEqual(comparable(a), comparable(b));
|
|
1003
|
+
}
|
|
1004
|
+
/** NDJSON transcript text to events; a damaged line is skipped, never fatal. */
|
|
1005
|
+
function parseEvents(raw) {
|
|
1006
|
+
return raw
|
|
1007
|
+
.split('\n')
|
|
1008
|
+
.filter(Boolean)
|
|
1009
|
+
.map((line) => {
|
|
1010
|
+
try {
|
|
1011
|
+
return JSON.parse(line);
|
|
1012
|
+
}
|
|
1013
|
+
catch {
|
|
1014
|
+
return null;
|
|
1015
|
+
}
|
|
1016
|
+
})
|
|
1017
|
+
.filter((e) => e !== null);
|
|
1018
|
+
}
|
|
1019
|
+
function newestFirst(a, b) {
|
|
1020
|
+
if (a.createdAt !== b.createdAt)
|
|
1021
|
+
return a.createdAt < b.createdAt ? 1 : -1;
|
|
1022
|
+
return a.seq < b.seq ? -1 : a.seq > b.seq ? 1 : 0;
|
|
1023
|
+
}
|
|
1024
|
+
/** An archived page's cursor: the last row's order, opaque to clients. A held run with no row yet
|
|
1025
|
+
* lists after every row (`seq` is infinite), which JSON cannot carry, so it travels as null. */
|
|
1026
|
+
function encodeArchivedCursor(key) {
|
|
1027
|
+
return Buffer.from(JSON.stringify({ c: key.createdAt, s: Number.isFinite(key.seq) ? key.seq : null })).toString('base64url');
|
|
1028
|
+
}
|
|
1029
|
+
function decodeArchivedCursor(cursor) {
|
|
1030
|
+
try {
|
|
1031
|
+
const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
|
|
1032
|
+
if (typeof parsed !== 'object' || parsed === null)
|
|
1033
|
+
return undefined;
|
|
1034
|
+
const { c, s } = parsed;
|
|
1035
|
+
if (typeof c !== 'string' || !(s === null || (typeof s === 'number' && Number.isInteger(s))))
|
|
1036
|
+
return undefined;
|
|
1037
|
+
return { createdAt: c, seq: s ?? Number.POSITIVE_INFINITY };
|
|
1038
|
+
}
|
|
1039
|
+
catch {
|
|
1040
|
+
return undefined;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
/** The items newest first, each by the order beside it. Stable: ties keep their input order. */
|
|
1044
|
+
function sortNewestFirst(entries) {
|
|
1045
|
+
return [...entries].sort(([, a], [, b]) => newestFirst(a, b)).map(([item]) => item);
|
|
1046
|
+
}
|
|
1047
|
+
/** The delegation family a run belongs to, by its root's id: a root is its own, a worker its
|
|
1048
|
+
* parent's. Workers cannot delegate, so a family is one root and its direct workers, never more. */
|
|
1049
|
+
function familyRootOf(run) {
|
|
1050
|
+
if (run.delegation?.role === 'root')
|
|
1051
|
+
return run.id;
|
|
1052
|
+
if (run.delegation?.role === 'worker')
|
|
1053
|
+
return run.delegation.parentRunId;
|
|
1054
|
+
return undefined;
|
|
1055
|
+
}
|
|
1056
|
+
/** Tests freeze the copies of cold runs, so an in-place write to one throws instead of vanishing. */
|
|
1057
|
+
function deepFreeze(value) {
|
|
1058
|
+
if (value && typeof value === 'object' && !Object.isFrozen(value)) {
|
|
1059
|
+
Object.freeze(value);
|
|
1060
|
+
for (const child of Object.values(value))
|
|
1061
|
+
deepFreeze(child);
|
|
1062
|
+
}
|
|
1063
|
+
return value;
|
|
1064
|
+
}
|
|
1065
|
+
/**
|
|
1066
|
+
* A stored summary, or undefined when the row holds none (a row seeded without one, or text that
|
|
1067
|
+
* is not a summary): the caller then projects the record instead. The column is written only by
|
|
1068
|
+
* `encodeRunRow`, from a record the schema accepted, so this checks the shape a list row cannot
|
|
1069
|
+
* do without rather than running the full contract schema — on a cockpit's thousand rows that
|
|
1070
|
+
* schema costs more than reading them. The cold reader (`run-index.ts`), which reads other
|
|
1071
|
+
* projects' databases, still validates in full.
|
|
1072
|
+
*/
|
|
1073
|
+
export function parseStoredSummary(text) {
|
|
1074
|
+
try {
|
|
1075
|
+
const raw = JSON.parse(text);
|
|
1076
|
+
if (!raw || typeof raw !== 'object')
|
|
1077
|
+
return undefined;
|
|
1078
|
+
const row = raw;
|
|
1079
|
+
return typeof row.id === 'string' && typeof row.title === 'string' && typeof row.status === 'string' &&
|
|
1080
|
+
typeof row.createdAt === 'string' && typeof row.archived === 'boolean' ? raw : undefined;
|
|
1081
|
+
}
|
|
1082
|
+
catch {
|
|
1083
|
+
return undefined;
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
710
1086
|
const WORKER_PROCESS_CAP = 32;
|
|
711
|
-
const recordedProcessSchema = z.object({ pid: z.number().int().positive(), startToken: z.string().min(1).max(128).optional()
|
|
1087
|
+
const recordedProcessSchema = z.object({ pid: z.number().int().positive(), startToken: z.string().min(1).max(128).optional(),
|
|
1088
|
+
pgid: z.number().int().positive().optional() }).strict();
|
|
712
1089
|
const workerProcessRecordSchema = z.object({ generation: z.string().uuid(), controller: recordedProcessSchema,
|
|
713
1090
|
processes: z.array(recordedProcessSchema).max(WORKER_PROCESS_CAP) }).strict();
|
|
714
1091
|
const startToken = (pid) => { const token = processStartToken(pid); return token === undefined ? {} : { startToken: token }; };
|
|
1092
|
+
/** The bulk-archive sweep's one predicate (#780): finished, not archived, not scheduled (a
|
|
1093
|
+
* `failed` run waiting out a usage limit), not an owned worker (those leave with their parent),
|
|
1094
|
+
* then filtered by pin state. Mirrored clause for clause by `isSweepable` in the cockpit's
|
|
1095
|
+
* `lib/tasks-table.ts`. */
|
|
1096
|
+
export function isSweepable(run, scope) {
|
|
1097
|
+
if (run.archived || !['done', 'failed', 'cancelled'].includes(run.status))
|
|
1098
|
+
return false;
|
|
1099
|
+
if (run.status === 'failed' && run.autoResumeAt !== undefined)
|
|
1100
|
+
return false;
|
|
1101
|
+
if (run.delegation?.role === 'worker')
|
|
1102
|
+
return false;
|
|
1103
|
+
if (scope === 'unpinned')
|
|
1104
|
+
return !run.pinned;
|
|
1105
|
+
if (scope === 'pinned')
|
|
1106
|
+
return run.pinned === true;
|
|
1107
|
+
return true;
|
|
1108
|
+
}
|
|
715
1109
|
/**
|
|
716
|
-
* File-backed run store: `runs.
|
|
717
|
-
*
|
|
718
|
-
*
|
|
719
|
-
*
|
|
1110
|
+
* File-backed run store: one row per run in `runs.db` (#779, `./run-database.ts`) plus one
|
|
1111
|
+
* append-only NDJSON event file per run. Also the in-process event bus the SSE endpoints
|
|
1112
|
+
* subscribe to: emits `('run', RunRecord)` and `('event', { runId, event: RunEvent })`.
|
|
1113
|
+
*
|
|
1114
|
+
* Only the held set lives in memory (#779, Amendment 2): live runs, the runs a RunManager pins,
|
|
1115
|
+
* their delegation families, and runs whose latest write has not settled yet (see `isHeld`).
|
|
1116
|
+
* `getRun` answers a held run with the held object, so `commitIndex` still installs into the
|
|
1117
|
+
* record a caller holds; any other run is read from `runs.db` as a fresh copy every time (frozen
|
|
1118
|
+
* under vitest, so an in-place write throws instead of being lost). Writes to a finished run go
|
|
1119
|
+
* through the methods here, which load it, change it and persist the row like any other.
|
|
1120
|
+
*
|
|
1121
|
+
* Two write paths, both through `writeIndex`:
|
|
1122
|
+
* - optimistic: a method changes the held record, marks it dirty and schedules the debounced
|
|
1123
|
+
* save, which writes the dirty rows and deletions only;
|
|
1124
|
+
* - durable (`commitIndex`): one transaction with the changed rows AND everything a debounced
|
|
1125
|
+
* save still owes. Only once it commits do the records change in memory, and only then do
|
|
1126
|
+
* subscribers hear about it.
|
|
1127
|
+
*
|
|
1128
|
+
* Several processes may open one project (`serve` and a headless `cez run`), so a store writes
|
|
1129
|
+
* only the delegation families it claims in `run_claims` (#779, plan step 3; `run-claims.ts`).
|
|
1130
|
+
* Everything it holds is claimed: open claims the live rows nobody alive owns and leaves the rest
|
|
1131
|
+
* alone, and a write to a run not in memory claims its family first, or refuses it while another
|
|
1132
|
+
* live process owns it. Claims are released once a family leaves memory, and all of them on
|
|
1133
|
+
* `close()`; a crashed owner's are taken over only once it is proven dead. Every write is fenced
|
|
1134
|
+
* in its transaction: if a row is not as this store last saw it, or its claim is gone, nothing is
|
|
1135
|
+
* written, the evidence goes to `run_conflicts` and this store stops writing that row
|
|
1136
|
+
* (`handleConflicts`).
|
|
720
1137
|
*/
|
|
721
1138
|
export class RunStore extends EventEmitter {
|
|
722
1139
|
dataDir;
|
|
723
|
-
|
|
1140
|
+
/** The held set: every record this store keeps in memory, by id. */
|
|
1141
|
+
held = new Map();
|
|
1142
|
+
/** Who holds a run beyond its record, by holder, so one holder's unpin never releases another. */
|
|
1143
|
+
pins = new Map();
|
|
1144
|
+
/** Staged ids while a durable commit installs and announces them: never evicted mid-commit. */
|
|
1145
|
+
committing = new Set();
|
|
724
1146
|
saveTimer = null;
|
|
1147
|
+
/** `null` once the store is closed, and in a `RunStore.unavailable` store (nothing is written,
|
|
1148
|
+
* nothing is evicted). */
|
|
1149
|
+
db = null;
|
|
1150
|
+
/** Why `open` failed, in a `RunStore.unavailable` store. */
|
|
1151
|
+
openFailure;
|
|
1152
|
+
/** Runs whose held record is ahead of its row; the next save or commit writes them. */
|
|
1153
|
+
dirty = new Set();
|
|
1154
|
+
/** Runs gone from memory whose rows the next save or commit deletes. */
|
|
1155
|
+
deleted = new Set();
|
|
1156
|
+
/** Deleted runs whose history files (events, handoff, images, artifacts) go once the delete of
|
|
1157
|
+
* their row commits: until then a conflict can undo the delete and bring the row back. */
|
|
1158
|
+
historyOwed = new Set();
|
|
725
1159
|
/** The repository this project IS (#945), armed after `open()` by `setRepoHandle`. Undefined
|
|
726
1160
|
* until it arrives and `null` when it cannot be known — both mean "unscoped", which is
|
|
727
1161
|
* exactly the pre-#945 behavior. */
|
|
728
1162
|
repoHandle;
|
|
1163
|
+
/** `open()`'s `keepLive`, which a run read later from `runs.db` is reconciled with too. */
|
|
1164
|
+
keepLive = false;
|
|
1165
|
+
/** This store's identity in `run_claims`; undefined without a database. */
|
|
1166
|
+
owner;
|
|
1167
|
+
/** The families this store claims, with the generation each acquisition got. */
|
|
1168
|
+
claimed = new Map();
|
|
1169
|
+
/** Families judged free whose claim a busy database kept this store from taking: the next write
|
|
1170
|
+
* of their rows takes it in its own transaction, if it is still as judged (null: absent). */
|
|
1171
|
+
pendingClaims = new Map();
|
|
1172
|
+
/** The revision and `data` of each held row (and pending deletion) as this store last read or
|
|
1173
|
+
* wrote it: what a write is fenced against, and a conflict's "original". No entry: this store
|
|
1174
|
+
* created the row and has not written it yet. `extras` is what that `data` holds beyond the
|
|
1175
|
+
* schema, which the next write of the row puts back (raw-record.ts). */
|
|
1176
|
+
base = new Map();
|
|
1177
|
+
/** Rows read cold in the current synchronous turn, so a commit staged from one is fenced against
|
|
1178
|
+
* the version it was computed from. `owned`: its family was already this store's when read.
|
|
1179
|
+
* Cleared at the next microtask: no commit spans a turn. */
|
|
1180
|
+
coldBase = new Map();
|
|
1181
|
+
/** The family of each pending deletion, whose claim the deleting write is fenced against. */
|
|
1182
|
+
deletedFamilies = new Map();
|
|
1183
|
+
/** Rows a conflicting write excluded: this store refuses to write them until it is reopened. */
|
|
1184
|
+
quarantined = new Set();
|
|
1185
|
+
/** Rows whose `data` this store tried to decode and could not: left in the database untouched,
|
|
1186
|
+
* and left out of the list rows from then on, as the cold reader leaves them out (run-index.ts). */
|
|
1187
|
+
unreadable = new Set();
|
|
1188
|
+
/** Background one-at-a-time compressor for archived transcripts (#818). */
|
|
1189
|
+
compressor;
|
|
1190
|
+
/** Transcript facts the delegation paths ask about, so they never re-read a transcript (#880). */
|
|
1191
|
+
facts;
|
|
729
1192
|
constructor(dataDir) {
|
|
730
1193
|
super();
|
|
731
1194
|
this.dataDir = dataDir;
|
|
732
1195
|
this.setMaxListeners(100);
|
|
1196
|
+
this.facts = new TranscriptFactsIndex(dataDir);
|
|
1197
|
+
this.compressor = new HistoryCompressor(dataDir, (id) => this.isHistoryCompressEligible(id), (id, plain, archive) => this.facts.adoptArchive(id, plain, stampOf(archive)));
|
|
733
1198
|
}
|
|
734
|
-
/**
|
|
1199
|
+
/**
|
|
1200
|
+
* Open the project's run store. See `reconcileLoadedRun` for what `keepLive` (#367) decides
|
|
1201
|
+
* about live-looking rows.
|
|
1202
|
+
*
|
|
1203
|
+
* Reads `runs.db` once its import is complete. Before that, `runs.json` is imported into it in
|
|
1204
|
+
* one transaction (`importLegacyIndex`) and left exactly as it was, for older cezars to read.
|
|
1205
|
+
* Either way only the held set is decoded: the live rows (by the indexed `live` column), the
|
|
1206
|
+
* few finished rows a normalization still repairs, and the delegation families of the live
|
|
1207
|
+
* ones.
|
|
1208
|
+
*
|
|
1209
|
+
* A store that cannot be opened is never an empty one (#779, plan step 4): `open` throws a
|
|
1210
|
+
* `RunStoreOpenError` naming the cause, and nothing is reset, restored or deleted. `retryBusy`
|
|
1211
|
+
* waits out a busy database (and a cockpit that may still be importing) with growing pauses,
|
|
1212
|
+
* about 3.5 s in all, before it gives up. Open is synchronous, so the wait blocks this thread:
|
|
1213
|
+
* only a boot passes it, while nothing else runs yet. Without it a busy database fails after
|
|
1214
|
+
* one busy timeout.
|
|
1215
|
+
*/
|
|
735
1216
|
static open(dataDir, opts) {
|
|
736
|
-
|
|
737
|
-
const
|
|
738
|
-
const
|
|
739
|
-
|
|
1217
|
+
const path = join(dataDir, RUNS_DB_FILE);
|
|
1218
|
+
const delays = opts?.retryBusy ? OPEN_RETRY_DELAYS_MS : [];
|
|
1219
|
+
const started = Date.now();
|
|
1220
|
+
for (let attempt = 0;; attempt++) {
|
|
740
1221
|
try {
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
}
|
|
748
|
-
store.runs.set(run.id, reconcileLoadedRun(run, opts));
|
|
749
|
-
}
|
|
1222
|
+
return RunStore.openOnce(dataDir, path, opts?.keepLive === true);
|
|
1223
|
+
}
|
|
1224
|
+
catch (error) {
|
|
1225
|
+
const transient = error instanceof RunDatabaseBusyError || (error instanceof LegacyWriterError && error.reason === 'cockpit');
|
|
1226
|
+
if (!transient || attempt >= delays.length) {
|
|
1227
|
+
throw toRunStoreOpenError(error, path, attempt > 0 ? { waitedMs: Date.now() - started } : {});
|
|
750
1228
|
}
|
|
1229
|
+
sleepSync(delays[attempt]);
|
|
751
1230
|
}
|
|
752
|
-
|
|
753
|
-
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
static openOnce(dataDir, path, keepLive) {
|
|
1234
|
+
mkdirSync(join(dataDir, 'runs'), { recursive: true });
|
|
1235
|
+
const store = new RunStore(dataDir);
|
|
1236
|
+
store.keepLive = keepLive;
|
|
1237
|
+
const db = RunDatabase.open(path);
|
|
1238
|
+
try {
|
|
1239
|
+
if (db.getMeta(RUNS_IMPORT_COMPLETE_KEY) === undefined)
|
|
1240
|
+
importLegacyIndex(db, dataDir);
|
|
1241
|
+
store.db = db;
|
|
1242
|
+
store.owner = openClaimSession();
|
|
1243
|
+
store.loadHeldRows();
|
|
1244
|
+
store.warmTranscriptFacts();
|
|
1245
|
+
}
|
|
1246
|
+
catch (error) {
|
|
1247
|
+
if (store.owner) {
|
|
1248
|
+
try {
|
|
1249
|
+
db.releaseClaims(store.owner.session);
|
|
1250
|
+
}
|
|
1251
|
+
catch { /* the session closes below: its claims are provably dead */ }
|
|
1252
|
+
closeClaimSession(store.owner.session);
|
|
754
1253
|
}
|
|
1254
|
+
store.owner = undefined;
|
|
1255
|
+
store.db = null;
|
|
1256
|
+
store.held.clear();
|
|
1257
|
+
store.dirty.clear();
|
|
1258
|
+
store.claimed.clear();
|
|
1259
|
+
store.base.clear();
|
|
1260
|
+
db.close();
|
|
1261
|
+
throw error;
|
|
755
1262
|
}
|
|
756
1263
|
return store;
|
|
757
1264
|
}
|
|
1265
|
+
/**
|
|
1266
|
+
* The store `serve` keeps for its boot project when `open` failed: no database, nothing held,
|
|
1267
|
+
* and every attempt to create or save a run refused with `failure`. The cockpit answers every
|
|
1268
|
+
* route of that project with `failure`'s message, and boot skips everything that would read
|
|
1269
|
+
* its empty run list as "no runs" (orphan pruning, retention, recovery, scratch sweeps). The
|
|
1270
|
+
* way out is a restart once the cause is gone.
|
|
1271
|
+
*/
|
|
1272
|
+
static unavailable(dataDir, failure) {
|
|
1273
|
+
const store = new RunStore(dataDir);
|
|
1274
|
+
store.openFailure = failure;
|
|
1275
|
+
return store;
|
|
1276
|
+
}
|
|
1277
|
+
/** Why this store has no database, when it is `RunStore.unavailable`'s: its runs are unknown,
|
|
1278
|
+
* not absent. */
|
|
1279
|
+
get unavailable() {
|
|
1280
|
+
return this.openFailure;
|
|
1281
|
+
}
|
|
1282
|
+
/**
|
|
1283
|
+
* Open-time load: the live rows, the finished rows a load normalization repairs, then the
|
|
1284
|
+
* delegation families of the live ones. A row that does not parse is skipped and left in the
|
|
1285
|
+
* database untouched; one warning says how many.
|
|
1286
|
+
*
|
|
1287
|
+
* Only rows this store can claim are loaded (#779, plan step 3). A live row whose family another
|
|
1288
|
+
* live process claims — `serve`'s parked run, seen by a headless `cez run` — is that process's:
|
|
1289
|
+
* it is not loaded, normalized or recovered here, and stays readable through `getRun` and the
|
|
1290
|
+
* list rows. A live row nobody alive claims is taken over, as before: this open recovers it
|
|
1291
|
+
* (`keepLive`) or settles it as interrupted.
|
|
1292
|
+
*/
|
|
1293
|
+
loadHeldRows() {
|
|
1294
|
+
let unreadable = 0;
|
|
1295
|
+
const decode = (row) => {
|
|
1296
|
+
const decoded = decodeRunRow(row.data);
|
|
1297
|
+
if (!decoded) {
|
|
1298
|
+
unreadable++;
|
|
1299
|
+
this.unreadable.add(row.id);
|
|
1300
|
+
}
|
|
1301
|
+
return decoded;
|
|
1302
|
+
};
|
|
1303
|
+
const live = this.db.listLive();
|
|
1304
|
+
const owned = this.claimFamilies(live.map(rowFamily), { allowLive: true, wait: true });
|
|
1305
|
+
for (const row of live) {
|
|
1306
|
+
if (!owned.has(rowFamily(row)))
|
|
1307
|
+
continue;
|
|
1308
|
+
const decoded = decode(row);
|
|
1309
|
+
if (decoded)
|
|
1310
|
+
this.adoptLoadedRun(decoded, { keepLive: this.keepLive }, row);
|
|
1311
|
+
}
|
|
1312
|
+
const heal = this.db.listWhere(LEGACY_REFERENCE_HEAL_SQL).filter((row) => !this.held.has(row.id));
|
|
1313
|
+
const healable = this.claimFamilies(heal.map(rowFamily));
|
|
1314
|
+
for (const row of heal) {
|
|
1315
|
+
if (!healable.has(rowFamily(row)))
|
|
1316
|
+
continue;
|
|
1317
|
+
const decoded = decode(row);
|
|
1318
|
+
// Held only when the repair changed it: the next save writes it, then it leaves.
|
|
1319
|
+
if (decoded)
|
|
1320
|
+
this.adoptLoadedRun(decoded, { keepLive: this.keepLive, onlyIfChanged: true }, row);
|
|
1321
|
+
}
|
|
1322
|
+
for (const rootId of this.anchoredFamilies())
|
|
1323
|
+
this.holdFamily(rootId, decode);
|
|
1324
|
+
this.releaseUnheldClaims();
|
|
1325
|
+
if (unreadable > 0)
|
|
1326
|
+
console.warn(`[cez] ${unreadable} run(s) in ${RUNS_DB_FILE} could not be read; they are left in the database untouched.`);
|
|
1327
|
+
}
|
|
1328
|
+
/** Normalize a record just read from `row` (see `reconcileLoadedRun`) and hold it. Marks the row
|
|
1329
|
+
* dirty only when normalization changed it, so the next save persists exactly the runs open()
|
|
1330
|
+
* rewrote. The caller has claimed the row's family. */
|
|
1331
|
+
adoptLoadedRun({ run, extras }, opts, row) {
|
|
1332
|
+
const before = loadNormalizedFields(run);
|
|
1333
|
+
if (run.delegation?.role === 'root' && (run.status === 'waiting' || run.delegation.wait !== undefined)) {
|
|
1334
|
+
this.syncHumanAskSummary(run);
|
|
1335
|
+
}
|
|
1336
|
+
if (opts.settle)
|
|
1337
|
+
settleOrphanedRun(run, { stop: opts.stop });
|
|
1338
|
+
else
|
|
1339
|
+
reconcileLoadedRun(run, opts);
|
|
1340
|
+
const changed = loadNormalizedFields(run) !== before;
|
|
1341
|
+
if (opts.onlyIfChanged && !changed)
|
|
1342
|
+
return;
|
|
1343
|
+
this.held.set(run.id, run);
|
|
1344
|
+
this.base.set(run.id, { revision: row.revision, seq: row.seq, data: row.data, extras });
|
|
1345
|
+
if (changed)
|
|
1346
|
+
this.dirty.add(run.id);
|
|
1347
|
+
}
|
|
1348
|
+
/**
|
|
1349
|
+
* Hold one delegation family: its root and the root's direct workers, read through the
|
|
1350
|
+
* `parent_run_id` index. No recursion: workers cannot delegate, so a worker's own id never
|
|
1351
|
+
* names a family. The caller has claimed it. `settle` settles its live rows instead of keeping
|
|
1352
|
+
* them for recovery (`settleOrphanedRun`); `stopId` is the run a Stop settles as cancelled.
|
|
1353
|
+
*/
|
|
1354
|
+
holdFamily(rootId, decode = (row) => decodeRunRow(row.data), settle = false, stopId) {
|
|
1355
|
+
const rows = [this.db.get(rootId), ...this.db.listByParent(rootId)];
|
|
1356
|
+
for (const row of rows) {
|
|
1357
|
+
if (!row || this.held.has(row.id) || this.deleted.has(row.id))
|
|
1358
|
+
continue;
|
|
1359
|
+
const decoded = decode(row);
|
|
1360
|
+
if (decoded)
|
|
1361
|
+
this.adoptLoadedRun(decoded, { keepLive: this.keepLive, settle, stop: row.id === stopId }, row);
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
/**
|
|
1365
|
+
* Make sure this store claims each of `families`, taking every one no live process holds, and
|
|
1366
|
+
* return the ones it now holds. Liveness is judged here (`claimOwnerLive`), outside the write
|
|
1367
|
+
* transaction; `takeClaims` then takes a claim only if it is still exactly what was judged.
|
|
1368
|
+
*
|
|
1369
|
+
* `allowLive` also takes a family that still has a live row although no live process owns it.
|
|
1370
|
+
* Open and `adoptFamily` pass it, because recovery follows both, and so does a write moving a
|
|
1371
|
+
* run this store holds into a new family (a worker quarantined to `invalid` becomes its own):
|
|
1372
|
+
* an ordinary write must not quietly inherit a dead process's live run and leave it unrecovered.
|
|
1373
|
+
* `wait` retries a busy database instead of leaving the claim to the next write: recovery
|
|
1374
|
+
* cannot start without it.
|
|
1375
|
+
*/
|
|
1376
|
+
claimFamilies(families, opts = {}) {
|
|
1377
|
+
const owned = new Set();
|
|
1378
|
+
const wanted = [];
|
|
1379
|
+
for (const family of new Set(families)) {
|
|
1380
|
+
if (this.claimed.has(family) || this.pendingClaims.has(family))
|
|
1381
|
+
owned.add(family);
|
|
1382
|
+
else
|
|
1383
|
+
wanted.push(family);
|
|
1384
|
+
}
|
|
1385
|
+
if (!this.db || !this.owner || wanted.length === 0)
|
|
1386
|
+
return owned;
|
|
1387
|
+
const claims = this.db.getClaims(wanted);
|
|
1388
|
+
const take = [];
|
|
1389
|
+
for (const family of wanted) {
|
|
1390
|
+
const claim = claims.get(family);
|
|
1391
|
+
if (claim && claimOwnerLive(claim))
|
|
1392
|
+
continue;
|
|
1393
|
+
if (!opts.allowLive && this.db.familyHasLive(family))
|
|
1394
|
+
continue;
|
|
1395
|
+
take.push({ family, expect: claim ? { session: claim.session, generation: claim.generation } : null });
|
|
1396
|
+
}
|
|
1397
|
+
if (take.length === 0)
|
|
1398
|
+
return owned;
|
|
1399
|
+
let taken;
|
|
1400
|
+
try {
|
|
1401
|
+
taken = this.takeClaims(take, opts.wait ? ADOPT_CLAIM_ATTEMPTS : 1);
|
|
1402
|
+
}
|
|
1403
|
+
catch (error) {
|
|
1404
|
+
if (!(error instanceof RunDatabaseBusyError))
|
|
1405
|
+
throw error;
|
|
1406
|
+
// Recovery follows an adoption, so it must hold the claim first: those families stay
|
|
1407
|
+
// unclaimed and orphaned (a control or the next restart adopts them). A write does not wait:
|
|
1408
|
+
// its next transaction takes the claim (`fence`), or meets whoever took it meanwhile.
|
|
1409
|
+
if (opts.wait)
|
|
1410
|
+
return owned;
|
|
1411
|
+
for (const { family, expect } of take) {
|
|
1412
|
+
this.pendingClaims.set(family, expect);
|
|
1413
|
+
owned.add(family);
|
|
1414
|
+
}
|
|
1415
|
+
this.scheduleSave();
|
|
1416
|
+
return owned;
|
|
1417
|
+
}
|
|
1418
|
+
for (const [family, generation] of taken) {
|
|
1419
|
+
this.claimed.set(family, generation);
|
|
1420
|
+
owned.add(family);
|
|
1421
|
+
}
|
|
1422
|
+
// A claim taken for a write that then changes nothing still leaves at the next sweep.
|
|
1423
|
+
if (taken.size > 0)
|
|
1424
|
+
this.scheduleSave();
|
|
1425
|
+
return owned;
|
|
1426
|
+
}
|
|
1427
|
+
/** `takeClaims`, retried while another connection holds the write lock (each try waits out the
|
|
1428
|
+
* busy timeout), then the busy error. */
|
|
1429
|
+
takeClaims(take, attempts) {
|
|
1430
|
+
for (let attempt = 1;; attempt++) {
|
|
1431
|
+
try {
|
|
1432
|
+
return this.db.takeClaims(this.owner, take);
|
|
1433
|
+
}
|
|
1434
|
+
catch (error) {
|
|
1435
|
+
if (!(error instanceof RunDatabaseBusyError) || attempt >= attempts)
|
|
1436
|
+
throw error;
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
/** Release the claim of every family with nothing left in memory, once nothing is pending: a
|
|
1441
|
+
* failed save keeps its families claimed until it succeeds. A failed release retries at the
|
|
1442
|
+
* next sweep; `close()` releases the rest. */
|
|
1443
|
+
releaseUnheldClaims() {
|
|
1444
|
+
if (!this.db || !this.owner || this.dirty.size > 0 || this.deleted.size > 0)
|
|
1445
|
+
return;
|
|
1446
|
+
const kept = new Set();
|
|
1447
|
+
for (const run of this.held.values())
|
|
1448
|
+
kept.add(familyKey(run));
|
|
1449
|
+
// A claim never taken has nothing to release in the database.
|
|
1450
|
+
for (const family of this.pendingClaims.keys())
|
|
1451
|
+
if (!kept.has(family))
|
|
1452
|
+
this.pendingClaims.delete(family);
|
|
1453
|
+
const released = [...this.claimed.keys()].filter((family) => !kept.has(family));
|
|
1454
|
+
if (released.length === 0)
|
|
1455
|
+
return;
|
|
1456
|
+
try {
|
|
1457
|
+
this.db.releaseClaims(this.owner.session, released);
|
|
1458
|
+
for (const family of released)
|
|
1459
|
+
this.claimed.delete(family);
|
|
1460
|
+
}
|
|
1461
|
+
catch {
|
|
1462
|
+
// Busy or failing: the next sweep tries again.
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
/**
|
|
1466
|
+
* Who may change this run right now (see `RunOwnership`), or undefined when there is no such
|
|
1467
|
+
* run. Read-only: nothing is claimed. Controls ask this first, so a run another process owns is
|
|
1468
|
+
* refused with `RUN_IN_USE_ELSEWHERE` and a dead process's run is adopted before it is changed.
|
|
1469
|
+
*/
|
|
1470
|
+
runOwnership(id) {
|
|
1471
|
+
if (this.quarantined.has(id))
|
|
1472
|
+
return 'quarantined';
|
|
1473
|
+
const held = this.held.get(id);
|
|
1474
|
+
if (!this.db)
|
|
1475
|
+
return held ? 'held' : undefined;
|
|
1476
|
+
const family = held ? familyKey(held) : this.deleted.has(id) ? undefined : this.db.familyOf(id);
|
|
1477
|
+
if (family === undefined)
|
|
1478
|
+
return undefined;
|
|
1479
|
+
if (this.claimed.has(family) || this.pendingClaims.has(family))
|
|
1480
|
+
return 'held';
|
|
1481
|
+
const claim = this.db.getClaim(family);
|
|
1482
|
+
if (claim && claimOwnerLive(claim))
|
|
1483
|
+
return 'foreign';
|
|
1484
|
+
// The same rule `claimFamilies` writes by: a dead owner left only settled runs → free.
|
|
1485
|
+
return this.db.familyHasLive(family) ? 'orphaned' : 'free';
|
|
1486
|
+
}
|
|
1487
|
+
/** Why a control on this run must be refused here, or undefined when this store may change it. */
|
|
1488
|
+
writeRefusal(id) {
|
|
1489
|
+
const ownership = this.runOwnership(id);
|
|
1490
|
+
if (ownership === 'quarantined')
|
|
1491
|
+
return RUN_QUARANTINED;
|
|
1492
|
+
return ownership === 'foreign' || ownership === 'orphaned' ? RUN_IN_USE_ELSEWHERE : undefined;
|
|
1493
|
+
}
|
|
1494
|
+
/**
|
|
1495
|
+
* Take over the delegation family of an `orphaned` run for a control: claim it from its dead
|
|
1496
|
+
* owner (or from nobody), then load it and settle its live rows (`settleOrphanedRun`) — never
|
|
1497
|
+
* keep them for recovery to resume, since only Continue may start agent work. With `stop` the
|
|
1498
|
+
* control is Stop: `id` itself is settled as cancelled rather than interrupted. Returns the
|
|
1499
|
+
* family's root id, or undefined when a live owner holds it after all or it is gone. The caller
|
|
1500
|
+
* runs recovery for that family next (`RunManager.adoptOrphanedRun`), which then repairs it.
|
|
1501
|
+
*/
|
|
1502
|
+
adoptFamily(id, opts = {}) {
|
|
1503
|
+
const family = this.db?.familyOf(id);
|
|
1504
|
+
if (family === undefined || !this.claimFamilies([family], { allowLive: true, wait: true }).has(family))
|
|
1505
|
+
return undefined;
|
|
1506
|
+
this.holdFamily(family, undefined, true, opts.stop ? id : undefined);
|
|
1507
|
+
return family;
|
|
1508
|
+
}
|
|
1509
|
+
/** Every run whose family another process owns — alive, or dead with a live run left to
|
|
1510
|
+
* recover (cleanup keeps their scratch: it is theirs, or recovery's once someone adopts them).
|
|
1511
|
+
* A dead owner's claim on a family with nothing live is no claim (see `runOwnership`). */
|
|
1512
|
+
listForeignClaimedRunIds() {
|
|
1513
|
+
const db = this.db, owner = this.owner;
|
|
1514
|
+
if (!db || !owner)
|
|
1515
|
+
return [];
|
|
1516
|
+
const kept = new Set(db.listClaims().filter((claim) => claim.session !== owner.session &&
|
|
1517
|
+
(claimOwnerLive(claim) || db.familyHasLive(claim.family))).map((claim) => claim.family));
|
|
1518
|
+
return kept.size === 0 ? [] : db.listForeignClaimedIds(owner.session).filter((row) => kept.has(row.family)).map((row) => row.id);
|
|
1519
|
+
}
|
|
1520
|
+
/**
|
|
1521
|
+
* Whether a held run anchors itself (and its family) in memory. Anything else held is a family
|
|
1522
|
+
* member or a settled write, and leaves at the next sweep (`evictSettled`).
|
|
1523
|
+
*
|
|
1524
|
+
* The held set is: every run that is live (`isLiveRecord`), pinned by a RunManager (`pin`),
|
|
1525
|
+
* dirty, or inside a durable commit; plus the whole delegation family (root and direct
|
|
1526
|
+
* workers) of any such run. Families are held because the RunManager reads a live run's family
|
|
1527
|
+
* on every run event — `GET /runs/:id/relationships`, the global `reconcileWorkerWaits` pass
|
|
1528
|
+
* (about twenty call sites), completion blockers and conversations — and decoding a cold
|
|
1529
|
+
* record costs about 200 µs (JSON.parse plus the schema). Measured on a copy of a real
|
|
1530
|
+
* cockpit's `runs.json` (958 runs, 957 of them delegated): a running root with 26 workers
|
|
1531
|
+
* paid about 5 ms per family read, and the live families together about 10 ms per global
|
|
1532
|
+
* pass, all on the event loop. Holding them costs memory in proportion to active work, not to
|
|
1533
|
+
* history.
|
|
1534
|
+
*/
|
|
1535
|
+
isAnchor(id, run) {
|
|
1536
|
+
return isLiveRecord(run) || this.pins.has(id) || this.dirty.has(id) || this.committing.has(id);
|
|
1537
|
+
}
|
|
1538
|
+
/** The family roots with an anchoring member in memory. */
|
|
1539
|
+
anchoredFamilies() {
|
|
1540
|
+
const roots = new Set();
|
|
1541
|
+
for (const [id, run] of this.held) {
|
|
1542
|
+
const root = familyRootOf(run);
|
|
1543
|
+
if (root !== undefined && this.isAnchor(id, run))
|
|
1544
|
+
roots.add(root);
|
|
1545
|
+
}
|
|
1546
|
+
return roots;
|
|
1547
|
+
}
|
|
1548
|
+
/** Drop every held run that is neither an anchor nor in an anchored family. A store without a
|
|
1549
|
+
* database keeps everything: memory is the only copy it has. */
|
|
1550
|
+
evictSettled() {
|
|
1551
|
+
if (!this.db)
|
|
1552
|
+
return;
|
|
1553
|
+
const families = this.anchoredFamilies();
|
|
1554
|
+
for (const [id, run] of this.held) {
|
|
1555
|
+
if (this.isAnchor(id, run))
|
|
1556
|
+
continue;
|
|
1557
|
+
const root = familyRootOf(run);
|
|
1558
|
+
if (root !== undefined && families.has(root))
|
|
1559
|
+
continue;
|
|
1560
|
+
this.held.delete(id);
|
|
1561
|
+
this.base.delete(id);
|
|
1562
|
+
}
|
|
1563
|
+
this.releaseUnheldClaims();
|
|
1564
|
+
}
|
|
1565
|
+
/** A run read from `runs.db`, normalized like a loaded one (`reconcileLoadedRun`, then the
|
|
1566
|
+
* repository scope) but held by nobody. Undefined when absent, deleted or unreadable. */
|
|
1567
|
+
loadCold(id) {
|
|
1568
|
+
if (!this.db || this.deleted.has(id))
|
|
1569
|
+
return undefined;
|
|
1570
|
+
const row = this.db.get(id);
|
|
1571
|
+
return row ? this.decodeCold(row) : undefined;
|
|
1572
|
+
}
|
|
1573
|
+
decodeCold(row) {
|
|
1574
|
+
const decoded = decodeRunRow(row.data);
|
|
1575
|
+
if (!decoded) {
|
|
1576
|
+
this.unreadable.add(row.id);
|
|
1577
|
+
return undefined;
|
|
1578
|
+
}
|
|
1579
|
+
this.unreadable.delete(row.id);
|
|
1580
|
+
const { run, extras } = decoded;
|
|
1581
|
+
if (this.coldBase.size === 0)
|
|
1582
|
+
queueMicrotask(() => this.coldBase.clear());
|
|
1583
|
+
const family = rowFamily(row);
|
|
1584
|
+
this.coldBase.set(row.id, { revision: row.revision, seq: row.seq, data: row.data, extras, owned: this.claimed.has(family) || this.pendingClaims.has(family) });
|
|
1585
|
+
reconcileLoadedRun(run, { keepLive: this.keepLive });
|
|
1586
|
+
rescopeRun(run, this.repoHandle);
|
|
1587
|
+
return run;
|
|
1588
|
+
}
|
|
1589
|
+
/** Hold a run just decoded from its row: the object every later read and write shares. */
|
|
1590
|
+
holdDecoded(run) {
|
|
1591
|
+
const read = this.coldBase.get(run.id);
|
|
1592
|
+
this.held.set(run.id, run);
|
|
1593
|
+
if (read)
|
|
1594
|
+
this.base.set(run.id, storedRow(read));
|
|
1595
|
+
return run;
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Bring a run into memory and return the held object: the record a write changes in place.
|
|
1599
|
+
* It leaves again at the first sweep after its write settles, unless something anchors it.
|
|
1600
|
+
* Its family is claimed first, then the row is read, so the version held is the claimed one;
|
|
1601
|
+
* undefined while another process owns it (see `claimFamilies`).
|
|
1602
|
+
*/
|
|
1603
|
+
hold(id) {
|
|
1604
|
+
const held = this.held.get(id);
|
|
1605
|
+
if (held)
|
|
1606
|
+
return held;
|
|
1607
|
+
const family = this.db && !this.deleted.has(id) ? this.db.familyOf(id) : undefined;
|
|
1608
|
+
if (family === undefined || !this.claimFamilies([family]).has(family))
|
|
1609
|
+
return undefined;
|
|
1610
|
+
const run = this.loadCold(id);
|
|
1611
|
+
if (!run)
|
|
1612
|
+
return undefined;
|
|
1613
|
+
this.holdDecoded(run);
|
|
1614
|
+
this.scheduleSave();
|
|
1615
|
+
return run;
|
|
1616
|
+
}
|
|
1617
|
+
/** The record a store write changes in place: held, or loaded to be. Undefined for a run this
|
|
1618
|
+
* store may not write (another process's, or quarantined), so the write changes nothing. */
|
|
1619
|
+
record(id) {
|
|
1620
|
+
if (this.quarantined.has(id))
|
|
1621
|
+
return undefined;
|
|
1622
|
+
return this.held.get(id) ?? this.hold(id);
|
|
1623
|
+
}
|
|
1624
|
+
/** The current record for reading: the held object, the held copy of a run whose family is in
|
|
1625
|
+
* memory, or else a fresh copy nobody holds. */
|
|
1626
|
+
peek(id) {
|
|
1627
|
+
const held = this.held.get(id);
|
|
1628
|
+
if (held)
|
|
1629
|
+
return held;
|
|
1630
|
+
const run = this.loadCold(id);
|
|
1631
|
+
return run && this.answerCold(run, false);
|
|
1632
|
+
}
|
|
1633
|
+
/**
|
|
1634
|
+
* A run just read off its row, as reads answer it: held when its family is in memory (a live
|
|
1635
|
+
* family's members are read on every run event), else a copy nobody holds — frozen under
|
|
1636
|
+
* vitest when it leaves the store (`freeze`).
|
|
1637
|
+
*/
|
|
1638
|
+
answerCold(run, freeze) {
|
|
1639
|
+
const root = familyRootOf(run);
|
|
1640
|
+
if (root !== undefined && this.anchoredFamilies().has(root))
|
|
1641
|
+
return this.holdDecoded(run);
|
|
1642
|
+
return freeze && process.env.VITEST ? deepFreeze(run) : run;
|
|
1643
|
+
}
|
|
1644
|
+
/** `getRun` for a row a query already read: the held object, else decoded from `row`. */
|
|
1645
|
+
fromRow(row) {
|
|
1646
|
+
const held = this.held.get(row.id);
|
|
1647
|
+
if (held)
|
|
1648
|
+
return held;
|
|
1649
|
+
if (this.deleted.has(row.id))
|
|
1650
|
+
return undefined;
|
|
1651
|
+
const run = this.decodeCold(row);
|
|
1652
|
+
return run && this.answerCold(run, true);
|
|
1653
|
+
}
|
|
1654
|
+
/** Whether the run exists, held or not, without decoding it. */
|
|
1655
|
+
hasRun(id) {
|
|
1656
|
+
return this.held.has(id) || (!!this.db && !this.deleted.has(id) && this.db.has(id));
|
|
1657
|
+
}
|
|
1658
|
+
/**
|
|
1659
|
+
* Keep a run in memory for `holder` until `unpin(id, holder)`: a RunManager pins the runs it is
|
|
1660
|
+
* executing (`active`) and the run a Continue is admitting (`continue`). A finished run is
|
|
1661
|
+
* loaded. Pins are per holder, so releasing one never releases another. Returns the held record.
|
|
1662
|
+
*/
|
|
1663
|
+
pin(id, holder) {
|
|
1664
|
+
const run = this.record(id);
|
|
1665
|
+
if (!run)
|
|
1666
|
+
return undefined;
|
|
1667
|
+
let holders = this.pins.get(id);
|
|
1668
|
+
if (!holders)
|
|
1669
|
+
this.pins.set(id, holders = new Set());
|
|
1670
|
+
holders.add(holder);
|
|
1671
|
+
return run;
|
|
1672
|
+
}
|
|
1673
|
+
/** Release `holder`'s pin. The run leaves memory at the next sweep once nothing else holds it. */
|
|
1674
|
+
unpin(id, holder) {
|
|
1675
|
+
const holders = this.pins.get(id);
|
|
1676
|
+
if (!holders?.delete(holder))
|
|
1677
|
+
return;
|
|
1678
|
+
if (holders.size === 0)
|
|
1679
|
+
this.pins.delete(id);
|
|
1680
|
+
this.sweepIfSettled([id]);
|
|
1681
|
+
}
|
|
758
1682
|
/**
|
|
759
1683
|
* Tell the store which repository this project IS (#945), so the referenced tier stops adopting
|
|
760
1684
|
* another repo's PR/issue as the task's subject. See `isRepoScopedRef` for the rule.
|
|
@@ -767,7 +1691,9 @@ export class RunStore extends EventEmitter {
|
|
|
767
1691
|
* Arming also HEALS records already poisoned by the un-scoped rule, on the `reconcileLoadedRun`
|
|
768
1692
|
* precedent: the evidence is all still on the record (`referenced*Candidates`), only the
|
|
769
1693
|
* conclusion drawn from it was wrong, so re-deciding beats asking for a migration. It rewrites
|
|
770
|
-
* values, never the format, and is one-directional by construction — see `rescopeRun`.
|
|
1694
|
+
* values, never the format, and is one-directional by construction — see `rescopeRun`. Held
|
|
1695
|
+
* runs are re-checked in memory; of the rest, only rows whose stored summary references another
|
|
1696
|
+
* repository are decoded, and the healed ones are written, so their summaries heal too.
|
|
771
1697
|
*/
|
|
772
1698
|
setRepoHandle(handle) {
|
|
773
1699
|
this.repoHandle = handle;
|
|
@@ -776,55 +1702,419 @@ export class RunStore extends EventEmitter {
|
|
|
776
1702
|
// `touch` per healed run: the cockpit is already live when the handle lands, so a corrected
|
|
777
1703
|
// chip has to reach the open page over SSE, not just the next `runs.json` write.
|
|
778
1704
|
let healed = false;
|
|
779
|
-
|
|
1705
|
+
const own = `${handle.owner}/${handle.name}`.toLowerCase();
|
|
1706
|
+
const foreign = (url) => url !== undefined && refUrlRepo(url) !== own;
|
|
1707
|
+
const stored = this.db?.listWhere(REFERENCED_SQL).filter((row) => {
|
|
1708
|
+
if (this.held.has(row.id) || this.deleted.has(row.id))
|
|
1709
|
+
return false;
|
|
1710
|
+
const summary = parseStoredSummary(row.summary);
|
|
1711
|
+
return !summary || foreign(summary.referencedPullRequestUrl) || foreign(summary.referencedIssueUrl);
|
|
1712
|
+
}) ?? [];
|
|
1713
|
+
for (const run of [...this.held.values()]) {
|
|
780
1714
|
if (rescopeRun(run, this.repoHandle)) {
|
|
781
1715
|
this.touch(run);
|
|
782
1716
|
healed = true;
|
|
783
1717
|
}
|
|
784
1718
|
}
|
|
1719
|
+
// Only rows this store may write: another process's runs heal when their owner arms its handle.
|
|
1720
|
+
const writable = this.claimFamilies(stored.map(rowFamily));
|
|
1721
|
+
for (const row of stored) {
|
|
1722
|
+
if (!writable.has(rowFamily(row)))
|
|
1723
|
+
continue;
|
|
1724
|
+
const run = this.decodeCold(row);
|
|
1725
|
+
// `decodeCold` already scoped it with the handle just armed; write it when that changed it.
|
|
1726
|
+
if (!run || !this.storedScopeDiffers(row, run))
|
|
1727
|
+
continue;
|
|
1728
|
+
this.holdDecoded(run);
|
|
1729
|
+
this.touch(run);
|
|
1730
|
+
healed = true;
|
|
1731
|
+
}
|
|
785
1732
|
// Discovery may finish after headless shutdown's final flush. Persist repairs now: the
|
|
786
1733
|
// debounced save is unref'd, so it cannot keep the CLI alive once the lookup completes.
|
|
787
1734
|
if (healed)
|
|
788
1735
|
this.flush();
|
|
789
1736
|
}
|
|
1737
|
+
/** Whether the stored summary still names a reference the scoped record no longer has. */
|
|
1738
|
+
storedScopeDiffers(row, run) {
|
|
1739
|
+
const summary = parseStoredSummary(row.summary);
|
|
1740
|
+
return summary?.referencedPullRequestUrl !== run.referencedPullRequestUrl ||
|
|
1741
|
+
summary?.referencedIssueUrl !== run.referencedIssueUrl || summary?.issueNumber !== run.issueNumber;
|
|
1742
|
+
}
|
|
1743
|
+
/**
|
|
1744
|
+
* The live set, newest first: runs whose record is live (`isLiveRecord`) or that a RunManager
|
|
1745
|
+
* pins. Finished runs are not here — read one by id (`getRun`), by an indexed query
|
|
1746
|
+
* (`listWorkersOf`, `findRunByClientRequestId`, `listGroupRuns`, `listRunsWithWorktree`,
|
|
1747
|
+
* `listBranchOwners`), or as list rows (`listRunSummaries`).
|
|
1748
|
+
*/
|
|
790
1749
|
listRuns() {
|
|
791
|
-
return [...this.
|
|
1750
|
+
return sortNewestFirst([...this.held].filter(([id, run]) => isLiveRecord(run) || this.pins.has(id)).map(([, run]) => [run, this.listOrder(run)]));
|
|
1751
|
+
}
|
|
1752
|
+
/** Where a held run lists: its row's insertion order, or after every row while this store has
|
|
1753
|
+
* not written it yet. The held set keeps such runs in creation order, and so does every list
|
|
1754
|
+
* that sorts them in held-set order (`sortNewestFirst` is stable). */
|
|
1755
|
+
listOrder(run) {
|
|
1756
|
+
return { createdAt: run.createdAt, seq: this.base.get(run.id)?.seq ?? Number.POSITIVE_INFINITY };
|
|
1757
|
+
}
|
|
1758
|
+
/**
|
|
1759
|
+
* EVERY run as a full record, newest first — decoded row by row. Only the legacy `GET /runs`
|
|
1760
|
+
* (older clients; the cockpit reads `GET /run-summaries`) may call this: it is exactly the
|
|
1761
|
+
* whole-history parse #779 exists to remove from everything else.
|
|
1762
|
+
*/
|
|
1763
|
+
listAllRunsForLegacyRoute() {
|
|
1764
|
+
const runs = new Map();
|
|
1765
|
+
for (const row of this.db?.listAll() ?? []) {
|
|
1766
|
+
if (this.deleted.has(row.id) || this.held.has(row.id))
|
|
1767
|
+
continue;
|
|
1768
|
+
const run = this.decodeCold(row);
|
|
1769
|
+
if (run)
|
|
1770
|
+
runs.set(row.id, [run, row]);
|
|
1771
|
+
}
|
|
1772
|
+
for (const [id, run] of this.held)
|
|
1773
|
+
runs.set(id, [run, this.listOrder(run)]);
|
|
1774
|
+
return sortNewestFirst(runs.values());
|
|
792
1775
|
}
|
|
1776
|
+
/**
|
|
1777
|
+
* Every run as its list row, newest first: the stored `summary` column, with each held run's
|
|
1778
|
+
* own projection laid over it — a debounced save keeps memory up to 300 ms ahead of the row,
|
|
1779
|
+
* and a run created since the last save has no row yet. `usage` is the caller's to attach.
|
|
1780
|
+
*
|
|
1781
|
+
* With `archivedWindow` (#864), the window the cockpit's lists read: every unarchived run, plus
|
|
1782
|
+
* the newest `archivedWindow` archived ROOT runs, and whether older archived roots were left
|
|
1783
|
+
* out. Archived workers are never in it (see `RunDatabase.listWindowSummaries`); with `roots`,
|
|
1784
|
+
* no worker is — the runs index, whose readers list no workers. A held run
|
|
1785
|
+
* lists by its record, so one archived or unarchived since the last save is ranked as it is now.
|
|
1786
|
+
*
|
|
1787
|
+
* A row this store found unreadable is left out, as the cold reader leaves out a row it had to
|
|
1788
|
+
* decode and could not. A finished row nobody has decoded is served from its summary by both,
|
|
1789
|
+
* which decodes nothing; reading it is what finds out (`isUnreadable`).
|
|
1790
|
+
*/
|
|
1791
|
+
listRunSummaries(options = {}) {
|
|
1792
|
+
const { archivedWindow, roots = false } = options;
|
|
1793
|
+
// Every skipped row may be an archived root, so ask for that many more: one past the window
|
|
1794
|
+
// after skipping is how "older ones were left out" is known without counting them.
|
|
1795
|
+
const rows = (archivedWindow === undefined
|
|
1796
|
+
? this.db?.listSummaries()
|
|
1797
|
+
: this.db?.listWindowSummaries(archivedWindow + 1 + this.deleted.size + this.unreadable.size + this.held.size, { roots })) ?? [];
|
|
1798
|
+
const summaries = new Map();
|
|
1799
|
+
for (const row of rows) {
|
|
1800
|
+
if (this.deleted.has(row.id) || this.held.has(row.id) || this.unreadable.has(row.id))
|
|
1801
|
+
continue;
|
|
1802
|
+
const summary = parseStoredSummary(row.summary) ?? this.coldSummary(row.id);
|
|
1803
|
+
if (summary)
|
|
1804
|
+
summaries.set(row.id, [summary, row]);
|
|
1805
|
+
}
|
|
1806
|
+
for (const [id, run] of this.held) {
|
|
1807
|
+
if (archivedWindow !== undefined && (run.archived || roots) && run.delegation?.role === 'worker')
|
|
1808
|
+
continue;
|
|
1809
|
+
summaries.set(id, [toRunSummary(run), this.listOrder(run)]);
|
|
1810
|
+
}
|
|
1811
|
+
const runs = sortNewestFirst(summaries.values());
|
|
1812
|
+
if (archivedWindow === undefined)
|
|
1813
|
+
return { runs, truncated: false };
|
|
1814
|
+
let archived = 0;
|
|
1815
|
+
const kept = runs.filter((run) => !run.archived || ++archived <= archivedWindow);
|
|
1816
|
+
return { runs: kept, truncated: archived > archivedWindow };
|
|
1817
|
+
}
|
|
1818
|
+
/**
|
|
1819
|
+
* One page of archived ROOT runs, newest first (#864): `GET /run-summaries/archived`, the
|
|
1820
|
+
* archived runs past the window. Held runs list by their record, as in `listRunSummaries`.
|
|
1821
|
+
* `before` is the previous page's `nextCursor`; with `q`, only runs `matchesRunQuery` keeps, and
|
|
1822
|
+
* `total` counts those. A cursor this store did not write answers `{ error }`.
|
|
1823
|
+
*
|
|
1824
|
+
* Ranking reads every archived root's key (id, created_at, seq: no record, no summary), so the
|
|
1825
|
+
* page and `total` agree with the held overlay; only the page's own rows are read in full.
|
|
1826
|
+
*/
|
|
1827
|
+
listArchivedRuns(options) {
|
|
1828
|
+
let after;
|
|
1829
|
+
if (options.before !== undefined) {
|
|
1830
|
+
after = decodeArchivedCursor(options.before);
|
|
1831
|
+
if (!after)
|
|
1832
|
+
return { error: 'before is not a cursor this server wrote' };
|
|
1833
|
+
}
|
|
1834
|
+
const q = options.q?.trim() ?? '';
|
|
1835
|
+
const skipped = (id) => this.deleted.has(id) || this.held.has(id) || this.unreadable.has(id);
|
|
1836
|
+
const ranked = new Map();
|
|
1837
|
+
if (q === '') {
|
|
1838
|
+
for (const key of this.db?.listKeysWhere(ARCHIVED_ROOT) ?? [])
|
|
1839
|
+
if (!skipped(key.id))
|
|
1840
|
+
ranked.set(key.id, key);
|
|
1841
|
+
}
|
|
1842
|
+
else {
|
|
1843
|
+
for (const row of this.db?.searchRootSummaries(sqlPrefilterTokens(q)) ?? []) {
|
|
1844
|
+
if (skipped(row.id))
|
|
1845
|
+
continue;
|
|
1846
|
+
const summary = parseStoredSummary(row.summary) ?? this.coldSummary(row.id);
|
|
1847
|
+
if (summary?.archived && matchesRunQuery(summary, q))
|
|
1848
|
+
ranked.set(row.id, { ...row, summary });
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
for (const [id, run] of this.held) {
|
|
1852
|
+
if (!run.archived || run.delegation?.role === 'worker')
|
|
1853
|
+
continue;
|
|
1854
|
+
const summary = toRunSummary(run);
|
|
1855
|
+
if (q === '' || matchesRunQuery(summary, q))
|
|
1856
|
+
ranked.set(id, { ...this.listOrder(run), summary });
|
|
1857
|
+
}
|
|
1858
|
+
const ordered = [...ranked.entries()].sort(([, a], [, b]) => newestFirst(a, b));
|
|
1859
|
+
const rest = after ? ordered.filter(([, key]) => newestFirst(key, after) > 0) : ordered;
|
|
1860
|
+
const page = rest.slice(0, options.limit);
|
|
1861
|
+
const cold = new Map((this.db?.getSummaries(page.filter(([, key]) => !key.summary).map(([id]) => id)) ?? [])
|
|
1862
|
+
.map((row) => [row.id, parseStoredSummary(row.summary) ?? this.coldSummary(row.id)]));
|
|
1863
|
+
const runs = page.flatMap(([id, key]) => {
|
|
1864
|
+
const summary = key.summary ?? cold.get(id);
|
|
1865
|
+
return summary ? [summary] : [];
|
|
1866
|
+
});
|
|
1867
|
+
const last = page.at(-1)?.[1];
|
|
1868
|
+
return {
|
|
1869
|
+
runs,
|
|
1870
|
+
nextCursor: last && rest.length > page.length ? encodeArchivedCursor(last) : null,
|
|
1871
|
+
total: ordered.length,
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
/**
|
|
1875
|
+
* The ROOT runs, archived or not, that match `query` (`matchesRunQuery`), newest first: at most
|
|
1876
|
+
* `limit`, and whether more matched. The owned half of the workspace search (#864); held runs
|
|
1877
|
+
* match by their record.
|
|
1878
|
+
*/
|
|
1879
|
+
searchRunSummaries(query, limit) {
|
|
1880
|
+
const matched = new Map();
|
|
1881
|
+
// Rows come newest first, so the newest `limit + 1` row matches are all a page can use; held
|
|
1882
|
+
// runs, which may be newer than any row, are ranked in below.
|
|
1883
|
+
this.db?.visitRootSummaries(sqlPrefilterTokens(query), (row) => {
|
|
1884
|
+
if (this.deleted.has(row.id) || this.held.has(row.id) || this.unreadable.has(row.id))
|
|
1885
|
+
return true;
|
|
1886
|
+
const summary = parseStoredSummary(row.summary) ?? this.coldSummary(row.id);
|
|
1887
|
+
if (summary && matchesRunQuery(summary, query))
|
|
1888
|
+
matched.set(row.id, [summary, row]);
|
|
1889
|
+
return matched.size <= limit;
|
|
1890
|
+
});
|
|
1891
|
+
for (const [id, run] of this.held) {
|
|
1892
|
+
if (run.delegation?.role === 'worker')
|
|
1893
|
+
continue;
|
|
1894
|
+
const summary = toRunSummary(run);
|
|
1895
|
+
if (matchesRunQuery(summary, query))
|
|
1896
|
+
matched.set(id, [summary, this.listOrder(run)]);
|
|
1897
|
+
}
|
|
1898
|
+
const runs = sortNewestFirst(matched.values());
|
|
1899
|
+
return { runs: runs.slice(0, limit), truncated: runs.length > limit };
|
|
1900
|
+
}
|
|
1901
|
+
/** A row whose stored summary does not fit the contract any more, projected from its record. */
|
|
1902
|
+
coldSummary(id) {
|
|
1903
|
+
const run = this.loadCold(id);
|
|
1904
|
+
return run ? toRunSummary(run) : undefined;
|
|
1905
|
+
}
|
|
1906
|
+
/**
|
|
1907
|
+
* The run, or undefined. A held run is the held object (writes install into it). Any other run
|
|
1908
|
+
* is a fresh copy of its row each call, normalized like a loaded record — frozen under vitest —
|
|
1909
|
+
* so changing it means a store method, never an assignment.
|
|
1910
|
+
*/
|
|
793
1911
|
getRun(id) {
|
|
794
|
-
|
|
1912
|
+
const held = this.held.get(id);
|
|
1913
|
+
if (held)
|
|
1914
|
+
return held;
|
|
1915
|
+
const run = this.loadCold(id);
|
|
1916
|
+
return run && this.answerCold(run, true);
|
|
1917
|
+
}
|
|
1918
|
+
/** Whether the run's row is in `runs.db` but its record could not be read here: what tells a
|
|
1919
|
+
* run that cannot open from one that does not exist. Known once something tried to read it. */
|
|
1920
|
+
isUnreadable(id) {
|
|
1921
|
+
return this.unreadable.has(id) && !this.held.has(id) && !this.deleted.has(id);
|
|
1922
|
+
}
|
|
1923
|
+
/** The ids in memory right now: the held set, for diagnostics and the benchmark's heap pass. */
|
|
1924
|
+
heldIds() {
|
|
1925
|
+
return [...this.held.keys()];
|
|
795
1926
|
}
|
|
796
1927
|
/**
|
|
797
1928
|
* The workers a root run owns, newest first, read off its own `receipts` (#659). Every
|
|
798
1929
|
* `createOwnedRun` writes the receipt and the worker record in one transaction, and the
|
|
799
|
-
* receipt list is capped at
|
|
1930
|
+
* receipt list is capped at 1,024 creations (#816), so this is a bounded lookup rather than a walk of the whole
|
|
800
1931
|
* project index. An ordinary or worker run owns nothing and costs one map read; a receipt
|
|
801
1932
|
* whose record has been deleted is skipped, never invented. A parent quarantined to
|
|
802
1933
|
* `invalid` has no readable receipts and reports none, which is also what the cockpit
|
|
803
1934
|
* assumes for that role.
|
|
804
1935
|
*/
|
|
805
1936
|
listOwnedWorkers(parentId) {
|
|
806
|
-
const parent = this.
|
|
1937
|
+
const parent = this.getRun(parentId);
|
|
807
1938
|
if (parent?.delegation?.role !== 'root')
|
|
808
1939
|
return [];
|
|
809
1940
|
const workers = [];
|
|
810
1941
|
for (const receipt of parent.delegation.receipts) {
|
|
811
|
-
const worker = this.
|
|
1942
|
+
const worker = this.getRun(receipt.workerId);
|
|
812
1943
|
if (worker?.delegation?.role === 'worker' && worker.delegation.parentRunId === parentId)
|
|
813
1944
|
workers.push(worker);
|
|
814
1945
|
}
|
|
815
1946
|
return workers.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
|
816
1947
|
}
|
|
1948
|
+
/** Every worker filed under `parentId` (the `parent_run_id` index plus memory), newest first:
|
|
1949
|
+
* receipts or not, which is what the reconcile and deletion checks ask. */
|
|
1950
|
+
listWorkersOf(parentId) {
|
|
1951
|
+
const workers = new Map();
|
|
1952
|
+
for (const run of this.held.values()) {
|
|
1953
|
+
if (run.delegation?.role === 'worker' && run.delegation.parentRunId === parentId)
|
|
1954
|
+
workers.set(run.id, [run, this.listOrder(run)]);
|
|
1955
|
+
}
|
|
1956
|
+
// Ids first: a live family's workers are usually all held, and then no record is read at all.
|
|
1957
|
+
const cold = (this.db?.listIdsByParent(parentId) ?? []).filter((id) => !this.held.has(id) && !this.deleted.has(id));
|
|
1958
|
+
for (const row of cold.length > 0 ? this.db.getMany(cold) : []) {
|
|
1959
|
+
const run = this.fromRow(row);
|
|
1960
|
+
if (run?.delegation?.role === 'worker' && run.delegation.parentRunId === parentId)
|
|
1961
|
+
workers.set(row.id, [run, row]);
|
|
1962
|
+
}
|
|
1963
|
+
return sortNewestFirst(workers.values());
|
|
1964
|
+
}
|
|
1965
|
+
/** The id of every worker, held or stored, without decoding any (recovery's worker passes). */
|
|
1966
|
+
listWorkerIds() {
|
|
1967
|
+
const ids = new Set(this.db?.listWorkerIds() ?? []);
|
|
1968
|
+
for (const id of this.deleted)
|
|
1969
|
+
ids.delete(id);
|
|
1970
|
+
for (const run of this.held.values())
|
|
1971
|
+
if (run.delegation?.role === 'worker')
|
|
1972
|
+
ids.add(run.id);
|
|
1973
|
+
return [...ids];
|
|
1974
|
+
}
|
|
1975
|
+
/** The id of every run whose delegation is quarantined (`invalid`), held or stored: its stored
|
|
1976
|
+
* summary says so, so nothing is decoded (scratch cleanup). */
|
|
1977
|
+
listQuarantinedRunIds() {
|
|
1978
|
+
const ids = new Set((this.db?.listKeysWhere("json_extract(summary, '$.delegation.role') = 'invalid'") ?? []).map((key) => key.id));
|
|
1979
|
+
for (const id of this.deleted)
|
|
1980
|
+
ids.delete(id);
|
|
1981
|
+
for (const [id, run] of this.held) {
|
|
1982
|
+
if (run.delegation?.role === 'invalid')
|
|
1983
|
+
ids.add(id);
|
|
1984
|
+
else
|
|
1985
|
+
ids.delete(id);
|
|
1986
|
+
}
|
|
1987
|
+
return [...ids];
|
|
1988
|
+
}
|
|
1989
|
+
/**
|
|
1990
|
+
* The runs automation receipts launched, by receipt id (`automation.receiptId` or
|
|
1991
|
+
* `automationTrigger.receiptId`). Memory answers first; a receipt it does not know costs one
|
|
1992
|
+
* pass over `runs.db` that reads the records' JSON in SQLite — only a crash leftover (a
|
|
1993
|
+
* receipt still `reserved` at startup) ever asks.
|
|
1994
|
+
*/
|
|
1995
|
+
findRunIdsByAutomationReceipt(receiptIds) {
|
|
1996
|
+
const wanted = new Set(receiptIds);
|
|
1997
|
+
const found = new Map();
|
|
1998
|
+
const receiptOf = (run) => run.automation?.receiptId ?? run.automationTrigger?.receiptId;
|
|
1999
|
+
for (const run of this.held.values()) {
|
|
2000
|
+
const receipt = receiptOf(run);
|
|
2001
|
+
if (receipt !== undefined && wanted.has(receipt))
|
|
2002
|
+
found.set(receipt, run.id);
|
|
2003
|
+
}
|
|
2004
|
+
const missing = [...wanted].filter((receipt) => !found.has(receipt));
|
|
2005
|
+
if (missing.length === 0 || !this.db)
|
|
2006
|
+
return found;
|
|
2007
|
+
const where = "json_extract(data, '$.automation.receiptId') IN (SELECT value FROM json_each(?))" +
|
|
2008
|
+
" OR json_extract(data, '$.automationTrigger.receiptId') IN (SELECT value FROM json_each(?))";
|
|
2009
|
+
for (const key of this.db.listKeysWhere(where, [JSON.stringify(missing), JSON.stringify(missing)])) {
|
|
2010
|
+
if (this.held.has(key.id) || this.deleted.has(key.id))
|
|
2011
|
+
continue;
|
|
2012
|
+
const run = this.getRun(key.id);
|
|
2013
|
+
const receipt = run && receiptOf(run);
|
|
2014
|
+
if (receipt !== undefined && wanted.has(receipt) && !found.has(receipt))
|
|
2015
|
+
found.set(receipt, key.id);
|
|
2016
|
+
}
|
|
2017
|
+
return found;
|
|
2018
|
+
}
|
|
2019
|
+
/** The id of every root whose delegation carries a conversation, held or stored: recovery's
|
|
2020
|
+
* full reconcile pass (#661). The stored summary names the role, so only roots' records are
|
|
2021
|
+
* read, in SQLite. */
|
|
2022
|
+
listConversationRootIds() {
|
|
2023
|
+
const where = "json_extract(summary, '$.delegation.role') = 'root' AND json_extract(data, '$.delegation.conversation') IS NOT NULL";
|
|
2024
|
+
const ids = new Set((this.db?.listKeysWhere(where) ?? []).map((key) => key.id));
|
|
2025
|
+
for (const id of this.deleted)
|
|
2026
|
+
ids.delete(id);
|
|
2027
|
+
for (const [id, run] of this.held) {
|
|
2028
|
+
if (run.delegation?.role === 'root' && run.delegation.conversation)
|
|
2029
|
+
ids.add(id);
|
|
2030
|
+
else
|
|
2031
|
+
ids.delete(id);
|
|
2032
|
+
}
|
|
2033
|
+
return [...ids];
|
|
2034
|
+
}
|
|
2035
|
+
/** The id of every run, held or stored, without decoding any (orphan-worktree sweeps). */
|
|
2036
|
+
listRunIds() {
|
|
2037
|
+
const ids = new Set(this.db?.listIds() ?? []);
|
|
2038
|
+
for (const id of this.deleted)
|
|
2039
|
+
ids.delete(id);
|
|
2040
|
+
for (const id of this.held.keys())
|
|
2041
|
+
ids.add(id);
|
|
2042
|
+
return [...ids];
|
|
2043
|
+
}
|
|
2044
|
+
/** The members of one parallel-variant group (spec 010), in no particular order. */
|
|
2045
|
+
listGroupRuns(groupId) {
|
|
2046
|
+
return this.queryRuns(this.db?.listByGroup(groupId) ?? [], (run) => run.groupId === groupId).map(([run]) => run);
|
|
2047
|
+
}
|
|
2048
|
+
/** Runs with a materialized worktree directory (`worktreePath`, not reclaimed), most recently
|
|
2049
|
+
* finished first, then newest first: worktree retention and the worktrees panel. */
|
|
2050
|
+
listRunsWithWorktree() {
|
|
2051
|
+
const runs = this.queryRuns(this.db?.listWithWorktree() ?? [], (run) => run.worktreePath !== undefined && run.worktreeReclaimedAt === undefined);
|
|
2052
|
+
const recency = ([run]) => run.finishedAt ?? run.createdAt;
|
|
2053
|
+
return runs.sort((a, b) => (recency(a) < recency(b) ? 1 : recency(a) > recency(b) ? -1 : newestFirst(a[1], b[1]))).map(([run]) => run);
|
|
2054
|
+
}
|
|
2055
|
+
/**
|
|
2056
|
+
* Every run that owns a branch or names the PR it created, as the few fields branch cleanup and
|
|
2057
|
+
* git-log attribution read (issue 08 §B3, §B5; a squash commit is attributed by its PR number):
|
|
2058
|
+
* the stored summary plus the `branch` and `base_branch` columns, so a Git-tab request decodes
|
|
2059
|
+
* no record. Held runs answer from memory.
|
|
2060
|
+
*/
|
|
2061
|
+
listBranchOwners() {
|
|
2062
|
+
const owners = new Map();
|
|
2063
|
+
for (const row of this.db?.listBranchOwners("branch IS NOT NULL OR json_extract(summary, '$.pullRequestUrl') IS NOT NULL") ?? []) {
|
|
2064
|
+
if (this.deleted.has(row.id) || this.held.has(row.id))
|
|
2065
|
+
continue;
|
|
2066
|
+
const summary = parseStoredSummary(row.summary);
|
|
2067
|
+
owners.set(row.id, {
|
|
2068
|
+
id: row.id, createdAt: row.createdAt, status: row.status, archived: row.archived,
|
|
2069
|
+
...(row.branch === null ? {} : { branch: row.branch }),
|
|
2070
|
+
...(row.baseBranch === null ? {} : { baseBranch: row.baseBranch }),
|
|
2071
|
+
title: summary?.title ?? row.id,
|
|
2072
|
+
...(summary?.pullRequestUrl === undefined ? {} : { pullRequestUrl: summary.pullRequestUrl }),
|
|
2073
|
+
});
|
|
2074
|
+
}
|
|
2075
|
+
for (const run of this.held.values()) {
|
|
2076
|
+
const owner = branchOwnerOf(run);
|
|
2077
|
+
if (owner)
|
|
2078
|
+
owners.set(run.id, owner);
|
|
2079
|
+
}
|
|
2080
|
+
return [...owners.values()];
|
|
2081
|
+
}
|
|
2082
|
+
/** The rows a column query returned, as records, with memory laid over them: a held run
|
|
2083
|
+
* answers from memory (matching `matches` there, not on its maybe-older row). */
|
|
2084
|
+
queryRuns(rows, matches) {
|
|
2085
|
+
const runs = new Map();
|
|
2086
|
+
for (const run of this.held.values())
|
|
2087
|
+
if (matches(run))
|
|
2088
|
+
runs.set(run.id, [run, this.listOrder(run)]);
|
|
2089
|
+
for (const row of rows) {
|
|
2090
|
+
if (runs.has(row.id) || this.held.has(row.id) || this.deleted.has(row.id))
|
|
2091
|
+
continue;
|
|
2092
|
+
const run = this.fromRow(row);
|
|
2093
|
+
if (run && matches(run))
|
|
2094
|
+
runs.set(row.id, [run, row]);
|
|
2095
|
+
}
|
|
2096
|
+
return [...runs.values()];
|
|
2097
|
+
}
|
|
817
2098
|
/** The run an idempotent start already created (#504), archived or not. A deleted run is gone
|
|
818
2099
|
* with its record, so a retry after deletion honestly creates a fresh one. */
|
|
819
2100
|
findRunByClientRequestId(clientRequestId) {
|
|
820
|
-
for (const run of this.
|
|
2101
|
+
for (const run of this.held.values())
|
|
821
2102
|
if (run.clientRequestId === clientRequestId)
|
|
822
2103
|
return run;
|
|
823
|
-
|
|
2104
|
+
const row = this.db?.findByClientRequestId(clientRequestId);
|
|
2105
|
+
if (!row || this.held.has(row.id))
|
|
2106
|
+
return undefined;
|
|
2107
|
+
const run = this.getRun(row.id);
|
|
2108
|
+
return run?.clientRequestId === clientRequestId ? run : undefined;
|
|
824
2109
|
}
|
|
825
2110
|
createRun(input) {
|
|
2111
|
+
if (this.openFailure)
|
|
2112
|
+
throw this.openFailure;
|
|
826
2113
|
const run = this.buildRun(input, randomUUID());
|
|
827
|
-
|
|
2114
|
+
// A new family, so nobody else can hold it; claimed before anything (an NDJSON event) is written.
|
|
2115
|
+
if (this.db && !this.claimFamilies([run.id]).has(run.id))
|
|
2116
|
+
throw new RunWriteRefusedError(run.id, 'foreign');
|
|
2117
|
+
this.held.set(run.id, run);
|
|
828
2118
|
this.pruneOldRuns();
|
|
829
2119
|
this.touch(run);
|
|
830
2120
|
return run;
|
|
@@ -875,29 +2165,27 @@ export class RunStore extends EventEmitter {
|
|
|
875
2165
|
}
|
|
876
2166
|
/** Accept the family ledger and its recipient queue in a single durable index replacement. */
|
|
877
2167
|
commitConversation(rootId, conversation, delivery) {
|
|
878
|
-
const root = this.
|
|
2168
|
+
const root = this.peek(rootId);
|
|
879
2169
|
if (root?.delegation?.role !== 'root')
|
|
880
2170
|
throw new Error('missing conversation root');
|
|
881
|
-
const
|
|
882
|
-
|
|
2171
|
+
const staged = new Map();
|
|
2172
|
+
staged.set(rootId, { ...root, delegation: delegationStateSchema.parse({ ...root.delegation, conversation: { ...conversation,
|
|
883
2173
|
messages: conversation.messages.map(message => ({ ...message, text: this.redactText(message.text) })),
|
|
884
2174
|
} }) });
|
|
885
|
-
const changed = new Set([rootId]);
|
|
886
2175
|
if (delivery) {
|
|
887
|
-
const recipient =
|
|
2176
|
+
const recipient = staged.get(delivery.recipientRunId) ?? this.peek(delivery.recipientRunId);
|
|
888
2177
|
if (!recipient)
|
|
889
2178
|
throw new Error('missing conversation recipient');
|
|
890
2179
|
const input = agentInputSchema.parse({ ...delivery.input, text: this.redactText(delivery.input.text) });
|
|
891
|
-
|
|
892
|
-
changed.add(recipient.id);
|
|
2180
|
+
staged.set(recipient.id, { ...recipient, agentInputs: [...(recipient.agentInputs ?? []), input] });
|
|
893
2181
|
}
|
|
894
|
-
this.commitIndex(
|
|
2182
|
+
this.commitIndex(staged);
|
|
895
2183
|
}
|
|
896
2184
|
/** Observable atomic input checkpoint: a failed write publishes nothing. */
|
|
897
2185
|
/** Accepted input the harness will never read returns to the queue (#505): only its
|
|
898
2186
|
* delivery and consumption receipts are cleared; order and identity are kept. */
|
|
899
2187
|
requeueUnconsumedAgentInputs(id, ids) {
|
|
900
|
-
const run = this.
|
|
2188
|
+
const run = this.peek(id);
|
|
901
2189
|
if (!run?.agentInputs || !ids.length)
|
|
902
2190
|
return;
|
|
903
2191
|
this.commitAgentInputs(id, run.agentInputs.map(input => {
|
|
@@ -909,7 +2197,7 @@ export class RunStore extends EventEmitter {
|
|
|
909
2197
|
}
|
|
910
2198
|
/** Model consumption observed by the current session (#505). Never un-sets deliveredAt. */
|
|
911
2199
|
commitAgentInputsConsumed(id, ids, at) {
|
|
912
|
-
const run = this.
|
|
2200
|
+
const run = this.peek(id);
|
|
913
2201
|
if (!run?.agentInputs || !ids.length)
|
|
914
2202
|
return;
|
|
915
2203
|
this.commitAgentInputs(id, run.agentInputs.map(input => {
|
|
@@ -921,7 +2209,7 @@ export class RunStore extends EventEmitter {
|
|
|
921
2209
|
}
|
|
922
2210
|
/** Delivered, but the harness never confirmed reading it (#505): stop awaiting a read. */
|
|
923
2211
|
commitAgentInputsUnconfirmed(id, ids) {
|
|
924
|
-
const run = this.
|
|
2212
|
+
const run = this.peek(id);
|
|
925
2213
|
if (!run?.agentInputs || !ids.length)
|
|
926
2214
|
return;
|
|
927
2215
|
this.commitAgentInputs(id, run.agentInputs.map(input => {
|
|
@@ -934,12 +2222,12 @@ export class RunStore extends EventEmitter {
|
|
|
934
2222
|
/** Crash recovery (#505): input a harness accepted but was never seen reading goes
|
|
935
2223
|
* back to the queue. Returns the requeued IDs. */
|
|
936
2224
|
requeueAwaitingReadInputs(id) {
|
|
937
|
-
const ids = (this.
|
|
2225
|
+
const ids = (this.peek(id)?.agentInputs ?? []).filter(input => input.awaitingRead && input.deliveredAt && !input.consumedAt).map(input => input.id);
|
|
938
2226
|
this.requeueUnconsumedAgentInputs(id, ids);
|
|
939
2227
|
return ids;
|
|
940
2228
|
}
|
|
941
2229
|
commitAgentInputs(id, inputs, openingContinuationInputId) {
|
|
942
|
-
const run = this.
|
|
2230
|
+
const run = this.peek(id);
|
|
943
2231
|
if (!run)
|
|
944
2232
|
throw new Error('missing agent input target');
|
|
945
2233
|
const agentInputs = inputs.map(input => agentInputSchema.parse(input));
|
|
@@ -956,13 +2244,11 @@ export class RunStore extends EventEmitter {
|
|
|
956
2244
|
if (openingContinuationInputId && (run.continuationMessage?.agentInputId !== openingContinuationInputId ||
|
|
957
2245
|
!agentInputs.some(input => input.id === openingContinuationInputId && input.deliveredAt)))
|
|
958
2246
|
throw new Error('opening agent input checkpoint changed');
|
|
959
|
-
|
|
960
|
-
proposed.set(id, { ...run, agentInputs, ...(openingContinuationInputId ? { continuationMessage: undefined } : {}) });
|
|
961
|
-
this.commitIndex(proposed, new Set([id]));
|
|
2247
|
+
this.commitIndex(new Map([[id, { ...run, agentInputs, ...(openingContinuationInputId ? { continuationMessage: undefined } : {}) }]]));
|
|
962
2248
|
}
|
|
963
2249
|
/** Reserve only unread conversation inputs in one durable index replacement. */
|
|
964
2250
|
claimInboxInputs(runId, ids, claim) {
|
|
965
|
-
const run = this.
|
|
2251
|
+
const run = this.peek(runId);
|
|
966
2252
|
if (!run)
|
|
967
2253
|
throw new Error('missing inbox recipient');
|
|
968
2254
|
const selected = z.array(z.uuid()).min(1).max(32).refine(values => new Set(values).size === values.length).parse(ids);
|
|
@@ -980,14 +2266,12 @@ export class RunStore extends EventEmitter {
|
|
|
980
2266
|
}
|
|
981
2267
|
}
|
|
982
2268
|
const selectedIds = new Set(selected);
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
? agentInputSchema.parse({ ...input, inboxClaim: receipt }) : input) });
|
|
986
|
-
this.commitIndex(proposed, new Set([runId]));
|
|
2269
|
+
this.commitIndex(new Map([[runId, { ...run, agentInputs: inputs.map(input => selectedIds.has(input.id)
|
|
2270
|
+
? agentInputSchema.parse({ ...input, inboxClaim: receipt }) : input) }]]));
|
|
987
2271
|
}
|
|
988
2272
|
/** A receipt owns every matching input or none; exact ACK retries preserve the first timestamp. */
|
|
989
2273
|
ackInboxInputs(runId, receiptId, generation, at) {
|
|
990
|
-
const run = this.
|
|
2274
|
+
const run = this.peek(runId);
|
|
991
2275
|
if (!run)
|
|
992
2276
|
throw new Error('missing inbox recipient');
|
|
993
2277
|
inboxClaimSchema.shape.receiptId.parse(receiptId);
|
|
@@ -1008,14 +2292,12 @@ export class RunStore extends EventEmitter {
|
|
|
1008
2292
|
if (matching.some(input => input.deliveredAt || input.inboxClaim?.acknowledgedAt || input.inboxClaim.expiresAt <= at)) {
|
|
1009
2293
|
throw new Error('inbox receipt expired or displaced');
|
|
1010
2294
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
? agentInputSchema.parse({ ...input, deliveredAt: at, inboxClaim: { ...input.inboxClaim, acknowledgedAt: at } }) : input) });
|
|
1014
|
-
this.commitIndex(proposed, new Set([runId]));
|
|
2295
|
+
this.commitIndex(new Map([[runId, { ...run, agentInputs: run.agentInputs.map(input => input.inboxClaim?.receiptId === receiptId
|
|
2296
|
+
? agentInputSchema.parse({ ...input, deliveredAt: at, inboxClaim: { ...input.inboxClaim, acknowledgedAt: at } }) : input) }]]));
|
|
1015
2297
|
return 'acknowledged';
|
|
1016
2298
|
}
|
|
1017
2299
|
releaseInboxInputs(runId, receiptId, generation) {
|
|
1018
|
-
const run = this.
|
|
2300
|
+
const run = this.peek(runId);
|
|
1019
2301
|
if (!run)
|
|
1020
2302
|
throw new Error('missing inbox recipient');
|
|
1021
2303
|
inboxClaimSchema.shape.receiptId.parse(receiptId);
|
|
@@ -1024,108 +2306,93 @@ export class RunStore extends EventEmitter {
|
|
|
1024
2306
|
if (!matching.length || matching.some(input => input.inboxClaim?.generation !== generation ||
|
|
1025
2307
|
input.inboxClaim?.acknowledgedAt || input.deliveredAt))
|
|
1026
2308
|
throw new Error('inbox receipt changed');
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
}) });
|
|
1034
|
-
this.commitIndex(proposed, new Set([runId]));
|
|
2309
|
+
this.commitIndex(new Map([[runId, { ...run, agentInputs: run.agentInputs.map(input => {
|
|
2310
|
+
if (input.inboxClaim?.receiptId !== receiptId)
|
|
2311
|
+
return input;
|
|
2312
|
+
const { inboxClaim: _claim, ...unclaimed } = input;
|
|
2313
|
+
return agentInputSchema.parse(unclaimed);
|
|
2314
|
+
}) }]]));
|
|
1035
2315
|
return 'released';
|
|
1036
2316
|
}
|
|
1037
2317
|
clearExpiredInboxClaims(runId, now) {
|
|
1038
|
-
const run = this.
|
|
2318
|
+
const run = this.peek(runId);
|
|
1039
2319
|
if (!run)
|
|
1040
2320
|
throw new Error('missing inbox recipient');
|
|
1041
2321
|
inboxClaimSchema.shape.expiresAt.parse(now);
|
|
1042
2322
|
if (!run.agentInputs?.some(input => input.inboxClaim && !input.inboxClaim.acknowledgedAt && input.inboxClaim.expiresAt <= now))
|
|
1043
2323
|
return;
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
}) });
|
|
1051
|
-
this.commitIndex(proposed, new Set([runId]));
|
|
2324
|
+
this.commitIndex(new Map([[runId, { ...run, agentInputs: run.agentInputs.map(input => {
|
|
2325
|
+
if (!input.inboxClaim || input.inboxClaim.acknowledgedAt || input.inboxClaim.expiresAt > now)
|
|
2326
|
+
return input;
|
|
2327
|
+
const { inboxClaim: _claim, ...unclaimed } = input;
|
|
2328
|
+
return agentInputSchema.parse(unclaimed);
|
|
2329
|
+
}) }]]));
|
|
1052
2330
|
}
|
|
1053
2331
|
/** CI intent and its wake entry share one atomic index replacement. */
|
|
1054
2332
|
commitCiWait(id, wait, input) {
|
|
1055
|
-
const run = this.
|
|
2333
|
+
const run = this.peek(id);
|
|
1056
2334
|
if (!run)
|
|
1057
2335
|
throw new Error('missing CI wait target');
|
|
1058
2336
|
const ciWait = ciWaitSchema.parse(wait);
|
|
1059
2337
|
const entry = input ? agentInputSchema.parse(input) : undefined;
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
? run.agentInputs : [...(run.agentInputs ?? []), entry] } : {}) });
|
|
1063
|
-
this.commitIndex(proposed, new Set([id]));
|
|
2338
|
+
this.commitIndex(new Map([[id, { ...run, ciWait, lastCiWaitError: undefined, ...(entry ? { agentInputs: run.agentInputs?.some(row => row.id === entry.id)
|
|
2339
|
+
? run.agentInputs : [...(run.agentInputs ?? []), entry] } : {}) }]]));
|
|
1064
2340
|
}
|
|
1065
2341
|
/** Retire the wait and precisely its pending wake, optionally accepting human input. */
|
|
1066
2342
|
commitCiWaitWithdrawal(id, acceptedHumanMessage) {
|
|
1067
|
-
const run = this.
|
|
2343
|
+
const run = this.peek(id);
|
|
1068
2344
|
if (!run?.ciWait)
|
|
1069
2345
|
return;
|
|
1070
2346
|
const { ciWait, ...rest } = run;
|
|
1071
2347
|
const lastCiWait = { ...ciWait, phase: 'withdrawn' };
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
});
|
|
1077
|
-
this.commitIndex(proposed, new Set([id]));
|
|
2348
|
+
this.commitIndex(new Map([[id, { ...rest, ciWait: undefined, lastCiWait,
|
|
2349
|
+
...(run.agentInputs ? { agentInputs: run.agentInputs.filter(input => input.id !== ciWait.wakeId || input.deliveredAt) } : {}),
|
|
2350
|
+
...(acceptedHumanMessage ? { queuedMessages: [...(run.queuedMessages ?? []), queuedMessageSchema.parse(acceptedHumanMessage)] } : {}),
|
|
2351
|
+
}]]));
|
|
1078
2352
|
}
|
|
1079
2353
|
/** Provider acceptance and receipt retirement have one delivery checkpoint. */
|
|
1080
2354
|
commitCiWaitDelivery(id, inputId) {
|
|
1081
|
-
const run = this.
|
|
2355
|
+
const run = this.peek(id);
|
|
1082
2356
|
if (!run?.ciWait || run.ciWait.wakeId !== inputId)
|
|
1083
2357
|
return;
|
|
1084
2358
|
const { ciWait, ...rest } = run;
|
|
1085
2359
|
const deliveredAt = new Date().toISOString();
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
agentInputs: (run.agentInputs ?? []).map(input => input.id === inputId ? { ...input, deliveredAt } : input),
|
|
1089
|
-
});
|
|
1090
|
-
this.commitIndex(proposed, new Set([id]));
|
|
2360
|
+
this.commitIndex(new Map([[id, { ...rest, ciWait: undefined, lastCiWait: { ...ciWait, phase: 'delivered', deliveredAt },
|
|
2361
|
+
agentInputs: (run.agentInputs ?? []).map(input => input.id === inputId ? { ...input, deliveredAt } : input), }]]));
|
|
1091
2362
|
}
|
|
1092
2363
|
/** Retain the settled receipt and retire exactly one wait together with any human message that superseded it. */
|
|
1093
2364
|
commitWorkerWaitWithdrawal(id, waitId, acceptedHumanMessage) {
|
|
1094
|
-
const run = this.
|
|
2365
|
+
const run = this.peek(id);
|
|
1095
2366
|
if (!run?.delegation || run.delegation.role === 'invalid' || run.delegation.wait?.id !== waitId) {
|
|
1096
2367
|
throw new Error('worker wait changed before withdrawal');
|
|
1097
2368
|
}
|
|
1098
2369
|
const { wait, ...delegation } = run.delegation;
|
|
1099
2370
|
const message = acceptedHumanMessage ? queuedMessageSchema.parse(acceptedHumanMessage) : undefined;
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
});
|
|
1114
|
-
this.commitIndex(proposed, new Set([id]));
|
|
2371
|
+
this.commitIndex(new Map([[id, { ...run, delegation: { ...delegation, lastWait: wait.phase === 'wake-pending'
|
|
2372
|
+
? reconcileWorkerWait(wait, [], new Date().toISOString())
|
|
2373
|
+
: { ...wait, phase: 'wake-pending', reason: wait.reason ?? 'cancelled', wakeId: wait.wakeId ?? wait.id } },
|
|
2374
|
+
// An adopted agent input belongs to its sender, even when it carries the wait receipt.
|
|
2375
|
+
...(run.agentInputs ? { agentInputs: run.agentInputs.filter(input => input.id !== wait.wakeId || input.deliveredAt || input.source === 'agent') } : {}),
|
|
2376
|
+
...(message ? {
|
|
2377
|
+
queuedMessages: [...(run.queuedMessages ?? []), message],
|
|
2378
|
+
...(run.continuationMessage ? { continuationMessage: {
|
|
2379
|
+
...run.continuationMessage, origin: 'human',
|
|
2380
|
+
text: run.continuationMessage.origin === 'lifecycle' ? '' : run.continuationMessage.text,
|
|
2381
|
+
} } : {}),
|
|
2382
|
+
} : {}),
|
|
2383
|
+
}]]));
|
|
1115
2384
|
}
|
|
1116
2385
|
/** Successful deferred human delivery consumes only its persisted queue ID.
|
|
1117
2386
|
* A crash before this checkpoint may replay that same message on recovery. */
|
|
1118
2387
|
commitQueuedMessageDelivery(id, messageId) {
|
|
1119
|
-
const run = this.
|
|
2388
|
+
const run = this.peek(id);
|
|
1120
2389
|
if (!run)
|
|
1121
2390
|
throw new Error('missing queued message target');
|
|
1122
|
-
|
|
1123
|
-
proposed.set(id, { ...run, queuedMessages: (run.queuedMessages ?? []).filter(message => message.id !== messageId) });
|
|
1124
|
-
this.commitIndex(proposed, new Set([id]));
|
|
2391
|
+
this.commitIndex(new Map([[id, { ...run, queuedMessages: (run.queuedMessages ?? []).filter(message => message.id !== messageId) }]]));
|
|
1125
2392
|
}
|
|
1126
2393
|
/** An acknowledged inactive-root Finish must survive the async diff and restart. */
|
|
1127
2394
|
commitRootFinishIntent(id) {
|
|
1128
|
-
const run = this.
|
|
2395
|
+
const run = this.peek(id);
|
|
1129
2396
|
if (run?.status !== 'waiting' || run.delegation?.role !== 'root')
|
|
1130
2397
|
throw new Error('root is not waiting');
|
|
1131
2398
|
if (run.delegation.finishRequestedAt)
|
|
@@ -1135,77 +2402,87 @@ export class RunStore extends EventEmitter {
|
|
|
1135
2402
|
/** Cancellation supersedes pending Finish in one durable checkpoint. Also
|
|
1136
2403
|
* repairs cancelled-plus-intent snapshots left by older controllers. */
|
|
1137
2404
|
commitRootFinishCancellation(id) {
|
|
1138
|
-
const run = this.
|
|
2405
|
+
const run = this.peek(id);
|
|
1139
2406
|
if (run?.delegation?.role !== 'root' || !run.delegation.finishRequestedAt ||
|
|
1140
2407
|
!['waiting', 'cancelled'].includes(run.status))
|
|
1141
2408
|
return false;
|
|
1142
2409
|
const { finishRequestedAt: _intent, ...delegation } = run.delegation;
|
|
1143
|
-
|
|
1144
|
-
proposed.set(id, { ...run, delegation, status: 'cancelled', finishedAt: run.finishedAt ?? new Date().toISOString() });
|
|
1145
|
-
this.commitIndex(proposed, new Set([id]));
|
|
2410
|
+
this.commitIndex(new Map([[id, { ...run, delegation, status: 'cancelled', finishedAt: run.finishedAt ?? new Date().toISOString() }]]));
|
|
1146
2411
|
return true;
|
|
1147
2412
|
}
|
|
1148
2413
|
/** Publish terminal success and completed steps only after their atomic checkpoint.
|
|
1149
2414
|
* A concurrent explicit cancellation is never overwritten by the async diff. */
|
|
1150
2415
|
commitRootFinishSuccess(id, status) {
|
|
1151
|
-
const run = this.
|
|
2416
|
+
const run = this.peek(id);
|
|
1152
2417
|
if (run?.status !== 'waiting' || run.delegation?.role !== 'root' || !run.delegation.finishRequestedAt)
|
|
1153
2418
|
return false;
|
|
1154
2419
|
const { finishRequestedAt: _intent, ...delegation } = run.delegation;
|
|
1155
2420
|
const finishedAt = new Date().toISOString();
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
});
|
|
1162
|
-
this.commitIndex(proposed, new Set([id]));
|
|
2421
|
+
this.commitIndex(new Map([[id, { ...run, delegation, status, finishedAt, currentStepId: undefined, autoResumeAttempts: undefined,
|
|
2422
|
+
activity: undefined, monitoringWakeAt: undefined, monitoringWakeCapReached: undefined,
|
|
2423
|
+
steps: run.steps.map(step => step.status === 'waiting' || step.status === 'running'
|
|
2424
|
+
? { ...step, status: 'done', finishedAt: step.finishedAt ?? finishedAt } : step),
|
|
2425
|
+
}]]));
|
|
1163
2426
|
return true;
|
|
1164
2427
|
}
|
|
1165
2428
|
/** Persist all authority patches before exposing any of them to the engine or subscribers. */
|
|
1166
2429
|
commitDelegation(patches) {
|
|
1167
2430
|
if (patches.length === 0)
|
|
1168
2431
|
return;
|
|
1169
|
-
const
|
|
1170
|
-
|
|
2432
|
+
const staged = new Map();
|
|
2433
|
+
let destroyProgress = true;
|
|
1171
2434
|
for (const patch of patches) {
|
|
1172
|
-
const run =
|
|
1173
|
-
if (!run ||
|
|
2435
|
+
const run = this.peek(patch.id);
|
|
2436
|
+
if (!run || staged.has(patch.id))
|
|
1174
2437
|
throw new Error('missing or duplicate delegation patch target');
|
|
1175
2438
|
const delegation = delegationStateSchema.parse(patch.delegation);
|
|
1176
|
-
|
|
1177
|
-
|
|
2439
|
+
if (!isDestroyProgress(run.delegation, delegation))
|
|
2440
|
+
destroyProgress = false;
|
|
2441
|
+
staged.set(patch.id, { ...run, delegation });
|
|
1178
2442
|
}
|
|
1179
2443
|
// In-process cause only: consumers can skip metadata replay when delegation
|
|
1180
2444
|
// is disabled without dropping real status or termination-proof notifications.
|
|
1181
|
-
this.commitIndex(
|
|
2445
|
+
this.commitIndex(staged, destroyProgress ? 'delegation-destroy-progress' : 'delegation-checkpoint');
|
|
2446
|
+
}
|
|
2447
|
+
/** A pending destroy's automatic retry state, or its removal (hearsay-tools/cezarion#879). Kept on
|
|
2448
|
+
* the record beside the strict delegation, so an older cezar strips it rather than quarantining the
|
|
2449
|
+
* worker. Destroy progress: nothing the family reconcile reads changes. */
|
|
2450
|
+
commitDestroyRetry(id, retry) {
|
|
2451
|
+
const run = this.peek(id);
|
|
2452
|
+
if (run?.delegation?.role !== 'worker' || !run.delegation.destroy)
|
|
2453
|
+
throw new Error('missing pending destroy');
|
|
2454
|
+
if (!retry && !run.destroyRetry)
|
|
2455
|
+
return;
|
|
2456
|
+
// An explicit `undefined`, not an omitted key: commitIndex assigns onto the live record.
|
|
2457
|
+
this.commitIndex(new Map([[id, { ...run, destroyRetry: retry ? workerDestroyRetrySchema.parse(retry) : undefined }]]), 'delegation-destroy-progress');
|
|
1182
2458
|
}
|
|
1183
2459
|
/** Accepted execution revision is public lifecycle identity, separate from process generations. */
|
|
1184
2460
|
commitWorkerContinuation(id, patch, step, conversation) {
|
|
1185
|
-
const run = this.
|
|
2461
|
+
const run = this.peek(id);
|
|
1186
2462
|
if (run?.delegation?.role !== 'worker')
|
|
1187
2463
|
throw new Error('missing worker continuation target');
|
|
1188
2464
|
const delegation = delegationStateSchema.parse({ ...run.delegation,
|
|
1189
2465
|
executionRevision: (run.delegation.executionRevision ?? 0) + 1,
|
|
1190
|
-
executionStartSeq: this.
|
|
2466
|
+
executionStartSeq: this.transcriptFacts(id).lastSeq,
|
|
1191
2467
|
});
|
|
1192
|
-
const
|
|
1193
|
-
|
|
2468
|
+
const staged = new Map();
|
|
2469
|
+
const next = { ...run, ...this.redactPatch(patch), delegation,
|
|
1194
2470
|
...(step ? { steps: [...run.steps, { ...step, status: 'pending', iterations: 0, tokensUsed: 0 }] } : {}),
|
|
1195
|
-
}
|
|
1196
|
-
|
|
2471
|
+
};
|
|
2472
|
+
staged.set(id, next);
|
|
2473
|
+
if (Object.prototype.hasOwnProperty.call(patch, 'archived'))
|
|
2474
|
+
this.syncTranscriptForm(id, next.archived);
|
|
1197
2475
|
if (conversation) {
|
|
1198
|
-
const root = this.
|
|
2476
|
+
const root = this.peek(conversation.rootId);
|
|
1199
2477
|
if (root?.delegation?.role !== 'root' || run.delegation.parentRunId !== root.id)
|
|
1200
2478
|
throw new Error('missing conversation ownership');
|
|
1201
|
-
|
|
2479
|
+
staged.set(root.id, { ...root, delegation: delegationStateSchema.parse({ ...root.delegation, conversation: { ...conversation.state,
|
|
1202
2480
|
messages: conversation.state.messages.map(message => ({ ...message, text: this.redactText(message.text) })),
|
|
1203
2481
|
} }) });
|
|
1204
2482
|
const input = agentInputSchema.parse({ ...conversation.input, text: this.redactText(conversation.input.text) });
|
|
1205
|
-
|
|
1206
|
-
changed.add(root.id);
|
|
2483
|
+
staged.set(id, { ...staged.get(id), agentInputs: [...(run.agentInputs ?? []), input] });
|
|
1207
2484
|
}
|
|
1208
|
-
this.commitIndex(
|
|
2485
|
+
this.commitIndex(staged);
|
|
1209
2486
|
}
|
|
1210
2487
|
workerResultsDir(parentId) {
|
|
1211
2488
|
z.uuid().parse(parentId);
|
|
@@ -1219,7 +2496,7 @@ export class RunStore extends EventEmitter {
|
|
|
1219
2496
|
return join(this.dataDir, 'runs', `${z.uuid().parse(parentId)}-worker-results`, `${z.uuid().parse(workerId)}.${z.uuid().parse(snapshotId)}.json`);
|
|
1220
2497
|
}
|
|
1221
2498
|
readWorkerResultFile(parentId, workerId) {
|
|
1222
|
-
const parent = this.
|
|
2499
|
+
const parent = this.peek(parentId);
|
|
1223
2500
|
if (parent?.delegation?.role !== 'root' || !parent.delegation.receipts.some(receipt => receipt.workerId === workerId))
|
|
1224
2501
|
return undefined;
|
|
1225
2502
|
const reference = parent.delegation.results?.find(result => result.workerId === workerId);
|
|
@@ -1256,11 +2533,13 @@ export class RunStore extends EventEmitter {
|
|
|
1256
2533
|
readWorkerResultDiff(parentId, workerId) {
|
|
1257
2534
|
return this.readWorkerResultFile(parentId, workerId)?.diffSnapshot;
|
|
1258
2535
|
}
|
|
1259
|
-
/** Publish a pointer only after its redacted immutable snapshot reaches disk. Failed index writes retain the old evidence.
|
|
1260
|
-
|
|
2536
|
+
/** Publish a pointer only after its redacted immutable snapshot reaches disk. Failed index writes retain the old evidence.
|
|
2537
|
+
* A result whose evidence is already stored returns the stored one and writes nothing; `ignoreCleanup` is for a
|
|
2538
|
+
* caller that only needs the payload durable, whatever cleanup phase the stored copy names. */
|
|
2539
|
+
commitWorkerResult(parentId, value, diffSnapshot, opts = {}) {
|
|
1261
2540
|
const result = workerCollectedResultSchema.parse(this.redact({ type: 'worker-result', seq: 0, ts: value.observedAt, result: value }).result);
|
|
1262
|
-
const parent = this.
|
|
1263
|
-
const worker = this.
|
|
2541
|
+
const parent = this.peek(parentId);
|
|
2542
|
+
const worker = this.peek(result.workerId);
|
|
1264
2543
|
if (parent?.delegation?.role !== 'root' || result.parentRunId !== parentId ||
|
|
1265
2544
|
!parent.delegation.receipts.some(receipt => receipt.workerId === result.workerId))
|
|
1266
2545
|
throw new Error('missing result ownership');
|
|
@@ -1268,6 +2547,10 @@ export class RunStore extends EventEmitter {
|
|
|
1268
2547
|
worker.delegation.workspace.ownerRunId !== worker.id || (worker.delegation.executionRevision ?? 0) !== result.revision || worker.status !== result.status))
|
|
1269
2548
|
throw new Error('worker result revision changed');
|
|
1270
2549
|
const old = parent.delegation.results?.find(entry => entry.workerId === result.workerId);
|
|
2550
|
+
const stored = old && this.readWorkerResultFile(parentId, result.workerId);
|
|
2551
|
+
if (stored && sameWorkerResult(stored.result, result, opts) &&
|
|
2552
|
+
stored.diffSnapshot === (diffSnapshot === undefined ? undefined : this.redactText(diffSnapshot)))
|
|
2553
|
+
return stored.result;
|
|
1271
2554
|
if ((!worker && !this.readWorkerResult(parentId, result.workerId)) || (old && (old.revision > result.revision || (old.revision === result.revision && old.observedAt > result.observedAt))))
|
|
1272
2555
|
throw new Error('worker result is obsolete');
|
|
1273
2556
|
const snapshotId = result.diff.state === 'available' ? result.diff.snapshotId : randomUUID();
|
|
@@ -1321,7 +2604,7 @@ export class RunStore extends EventEmitter {
|
|
|
1321
2604
|
* workspace.ownerRunId is the trusted service's preallocated worker UUID.
|
|
1322
2605
|
*/
|
|
1323
2606
|
createOwnedRun(input, parentId, requestId, worker, requestHash, executionIdentity = { kind: 'internal' }) {
|
|
1324
|
-
const parent = this.
|
|
2607
|
+
const parent = this.peek(parentId);
|
|
1325
2608
|
const authority = delegationStateSchema.safeParse(parent?.delegation);
|
|
1326
2609
|
if (!parent || !authority.success || authority.data.role !== 'root' || authority.data.historyDeletion) {
|
|
1327
2610
|
throw new Error('invalid delegation parent');
|
|
@@ -1333,7 +2616,7 @@ export class RunStore extends EventEmitter {
|
|
|
1333
2616
|
if (receipt) {
|
|
1334
2617
|
if (receipt.requestHash !== identity.requestHash)
|
|
1335
2618
|
throw new Error('request ID payload conflict');
|
|
1336
|
-
const existing = this.
|
|
2619
|
+
const existing = this.peek(receipt.workerId);
|
|
1337
2620
|
const metadata = delegationStateSchema.safeParse(existing?.delegation);
|
|
1338
2621
|
if (!existing || !metadata.success || metadata.data.role !== 'worker' ||
|
|
1339
2622
|
metadata.data.parentRunId !== parentId || metadata.data.workspace.ownerRunId !== existing.id ||
|
|
@@ -1341,67 +2624,130 @@ export class RunStore extends EventEmitter {
|
|
|
1341
2624
|
throw new Error('invalid request receipt ownership');
|
|
1342
2625
|
return existing;
|
|
1343
2626
|
}
|
|
2627
|
+
// #816: the same check the spawn policy makes, inside the receipt transaction, so no path
|
|
2628
|
+
// writes a receipt past either limit.
|
|
2629
|
+
const refusal = capacityError(workerCapacity(parent, id => this.peek(id)));
|
|
2630
|
+
if (refusal)
|
|
2631
|
+
throw new Error(refusal);
|
|
1344
2632
|
const metadata = delegationStateSchema.parse(worker);
|
|
1345
2633
|
if (metadata.role !== 'worker' || metadata.parentRunId !== parentId ||
|
|
1346
2634
|
metadata.workspace.ownerRunId === parentId || input.worktree === false) {
|
|
1347
2635
|
throw new Error('invalid worker ownership');
|
|
1348
2636
|
}
|
|
1349
2637
|
const workspace = metadata.workspace;
|
|
1350
|
-
if (this.
|
|
2638
|
+
if (this.hasRun(workspace.ownerRunId))
|
|
1351
2639
|
throw new Error('worker run ID collision');
|
|
1352
|
-
for (const run of this.
|
|
2640
|
+
for (const run of this.held.values()) {
|
|
1353
2641
|
const owned = run.delegation?.role === 'worker' ? run.delegation.workspace : undefined;
|
|
1354
2642
|
if ((owned && (owned.resourceId === workspace.resourceId || owned.path === workspace.path ||
|
|
1355
2643
|
owned.branch === workspace.branch)) || run.worktreePath === workspace.path || run.branch === workspace.branch) {
|
|
1356
2644
|
throw new Error('worker resource ownership collision');
|
|
1357
2645
|
}
|
|
1358
2646
|
}
|
|
2647
|
+
// Runs not in memory answer through the branch and worktree columns. A worker's planned
|
|
2648
|
+
// branch is its `branch` column; its path and resource id derive from its own run id, which
|
|
2649
|
+
// the id check above already proved unused.
|
|
2650
|
+
if ((this.db?.findResourceHolders({ branch: workspace.branch, worktreePath: workspace.path }) ?? [])
|
|
2651
|
+
.some((id) => !this.held.has(id) && !this.deleted.has(id)))
|
|
2652
|
+
throw new Error('worker resource ownership collision');
|
|
1359
2653
|
const delegation = delegationStateSchema.parse({
|
|
1360
2654
|
...authority.data,
|
|
1361
2655
|
receipts: [...authority.data.receipts, { ...identity, workerId: workspace.ownerRunId }],
|
|
1362
2656
|
});
|
|
1363
2657
|
const run = { ...this.buildRun(input, workspace.ownerRunId), delegation: metadata };
|
|
1364
|
-
const proposed = new Map(this.runs);
|
|
1365
|
-
proposed.set(parentId, { ...parent, delegation });
|
|
1366
|
-
proposed.set(run.id, run);
|
|
1367
2658
|
// This transaction precedes materialization. Every launch must rotate this
|
|
1368
2659
|
// generation to starting before touching the workspace or a session.
|
|
1369
2660
|
this.writeWorkerIdentity(run.id, workerExecutionIdentitySchema.parse(executionIdentity));
|
|
1370
2661
|
this.writeWorkerExecution(run.id, { generation: randomUUID(), phase: 'queued' }, true);
|
|
1371
2662
|
// No pruning here: deleting run history cannot be part of a proposed index transaction.
|
|
1372
|
-
this.commitIndex(
|
|
2663
|
+
this.commitIndex(new Map([[parentId, { ...parent, delegation }], [run.id, run]]));
|
|
1373
2664
|
return run;
|
|
1374
2665
|
}
|
|
1375
|
-
/**
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
2666
|
+
/**
|
|
2667
|
+
* The durable write: `staged` holds the records this operation replaces (`null` deletes one),
|
|
2668
|
+
* in the order subscribers hear about them. The transaction is the durability boundary;
|
|
2669
|
+
* flush() is deliberately best-effort.
|
|
2670
|
+
*
|
|
2671
|
+
* One transaction carries the staged rows and every row a debounced save still owes, so the
|
|
2672
|
+
* database never holds a commit without the optimistic writes made before it. If it fails,
|
|
2673
|
+
* nothing changed anywhere: the held records, the database, the pending dirty rows and the
|
|
2674
|
+
* subscribers all see the store as it was.
|
|
2675
|
+
*/
|
|
2676
|
+
commitIndex(staged, source) {
|
|
2677
|
+
if (!this.db)
|
|
2678
|
+
throw this.openFailure ?? new Error('runs database unavailable: nothing can be saved');
|
|
2679
|
+
this.assertWritable(staged);
|
|
2680
|
+
this.persist(staged);
|
|
1382
2681
|
// Preserve existing record references, but expose the entire transaction before its first event.
|
|
1383
|
-
for
|
|
1384
|
-
|
|
2682
|
+
// A finished run joins the held set for the commit and leaves at the next sweep.
|
|
2683
|
+
for (const [id, next] of staged) {
|
|
1385
2684
|
if (!next) {
|
|
1386
|
-
this.
|
|
2685
|
+
this.held.delete(id);
|
|
1387
2686
|
continue;
|
|
1388
2687
|
}
|
|
1389
|
-
|
|
2688
|
+
this.committing.add(id);
|
|
2689
|
+
const current = this.held.get(id);
|
|
1390
2690
|
if (current)
|
|
1391
2691
|
Object.assign(current, next);
|
|
1392
2692
|
else
|
|
1393
|
-
this.
|
|
2693
|
+
this.held.set(id, next);
|
|
1394
2694
|
}
|
|
1395
|
-
|
|
1396
|
-
const
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
2695
|
+
try {
|
|
2696
|
+
for (const id of staged.keys()) {
|
|
2697
|
+
const run = this.held.get(id);
|
|
2698
|
+
if (run)
|
|
2699
|
+
this.emit('run', run, source);
|
|
2700
|
+
else
|
|
2701
|
+
this.emit('deleted', id);
|
|
2702
|
+
}
|
|
2703
|
+
}
|
|
2704
|
+
finally {
|
|
2705
|
+
for (const id of staged.keys())
|
|
2706
|
+
this.committing.delete(id);
|
|
2707
|
+
this.sweepIfSettled(staged.keys());
|
|
2708
|
+
}
|
|
2709
|
+
}
|
|
2710
|
+
/**
|
|
2711
|
+
* Refuse (throwing, with nothing changed) a commit that stages a quarantined run, or a run whose
|
|
2712
|
+
* family this store does not claim and cannot take: another process owns it, or a dead process
|
|
2713
|
+
* left it live and it has not been adopted.
|
|
2714
|
+
*/
|
|
2715
|
+
assertWritable(staged) {
|
|
2716
|
+
const families = new Map();
|
|
2717
|
+
const held = new Set();
|
|
2718
|
+
for (const [id, next] of staged) {
|
|
2719
|
+
if (this.quarantined.has(id))
|
|
2720
|
+
throw new RunWriteRefusedError(id, 'quarantined');
|
|
2721
|
+
const family = next ? familyKey(next) : this.familyOf(id);
|
|
2722
|
+
families.set(id, family);
|
|
2723
|
+
if (this.held.has(id))
|
|
2724
|
+
held.add(family);
|
|
2725
|
+
}
|
|
2726
|
+
// A run this store holds may move to a new family; any other row is claimed as a plain write.
|
|
2727
|
+
const owned = new Set([...this.claimFamilies(held, { allowLive: true }),
|
|
2728
|
+
...this.claimFamilies([...families.values()].filter((family) => !held.has(family)))]);
|
|
2729
|
+
for (const [id, family] of families) {
|
|
2730
|
+
if (!owned.has(family))
|
|
2731
|
+
throw new RunWriteRefusedError(id, this.runOwnership(id) === 'orphaned' ? 'orphaned' : 'foreign');
|
|
2732
|
+
}
|
|
2733
|
+
}
|
|
2734
|
+
/** The family of a run this store knows: held, else its row; a run nowhere is its own. */
|
|
2735
|
+
familyOf(id) {
|
|
2736
|
+
const held = this.held.get(id);
|
|
2737
|
+
return held ? familyKey(held) : this.deletedFamilies.get(id) ?? this.db?.familyOf(id) ?? id;
|
|
2738
|
+
}
|
|
2739
|
+
/** Schedule the sweep that lets any of `ids` leave memory, when one of them now can. */
|
|
2740
|
+
sweepIfSettled(ids) {
|
|
2741
|
+
for (const id of ids) {
|
|
2742
|
+
const run = this.held.get(id);
|
|
2743
|
+
if (run && !this.isAnchor(id, run)) {
|
|
2744
|
+
this.scheduleSave();
|
|
2745
|
+
return;
|
|
2746
|
+
}
|
|
1401
2747
|
}
|
|
1402
2748
|
}
|
|
1403
2749
|
updateRun(id, patch) {
|
|
1404
|
-
const run = this.
|
|
2750
|
+
const run = this.record(id);
|
|
1405
2751
|
if (!run)
|
|
1406
2752
|
return undefined;
|
|
1407
2753
|
if (Object.prototype.hasOwnProperty.call(patch, 'issueNumber')) {
|
|
@@ -1420,10 +2766,23 @@ export class RunStore extends EventEmitter {
|
|
|
1420
2766
|
if (normalized.status && ['running', 'waiting', 'queued'].includes(normalized.status)) {
|
|
1421
2767
|
normalized.autoResumeAt = undefined;
|
|
1422
2768
|
}
|
|
2769
|
+
const archived = Object.prototype.hasOwnProperty.call(normalized, 'archived')
|
|
2770
|
+
? Boolean(normalized.archived)
|
|
2771
|
+
: undefined;
|
|
2772
|
+
if (archived !== undefined) {
|
|
2773
|
+
delete normalized.archived;
|
|
2774
|
+
delete normalized.archivedAt;
|
|
2775
|
+
}
|
|
2776
|
+
// Restore before mutating so a throw leaves status and archived unchanged (#818 S3).
|
|
2777
|
+
if (archived === false)
|
|
2778
|
+
restoreHistory(this.dataDir, id);
|
|
1423
2779
|
Object.assign(run, this.redactPatch(normalized));
|
|
1424
2780
|
if (normalized.task !== undefined)
|
|
1425
2781
|
this.resolveEditedTaskRefs(run);
|
|
1426
|
-
|
|
2782
|
+
if (archived !== undefined)
|
|
2783
|
+
this.applyArchived(run, archived);
|
|
2784
|
+
else
|
|
2785
|
+
this.touch(run);
|
|
1427
2786
|
return run;
|
|
1428
2787
|
}
|
|
1429
2788
|
/** Prompt edits can both revoke and restore a reference. Resolve the retained working sets
|
|
@@ -1499,14 +2858,14 @@ export class RunStore extends EventEmitter {
|
|
|
1499
2858
|
}
|
|
1500
2859
|
/** Append a step to an existing run (used by "Continue" — spec 003). */
|
|
1501
2860
|
addStep(runId, step) {
|
|
1502
|
-
const run = this.
|
|
2861
|
+
const run = this.record(runId);
|
|
1503
2862
|
if (!run || run.steps.some((s) => s.id === step.id))
|
|
1504
2863
|
return;
|
|
1505
2864
|
run.steps.push({ ...step, status: 'pending', iterations: 0, tokensUsed: 0 });
|
|
1506
2865
|
this.touch(run);
|
|
1507
2866
|
}
|
|
1508
2867
|
updateStep(runId, stepId, patch) {
|
|
1509
|
-
const run = this.
|
|
2868
|
+
const run = this.record(runId);
|
|
1510
2869
|
const step = run?.steps.find((s) => s.id === stepId);
|
|
1511
2870
|
if (!run || !step)
|
|
1512
2871
|
return;
|
|
@@ -1535,7 +2894,7 @@ export class RunStore extends EventEmitter {
|
|
|
1535
2894
|
this.touch(run);
|
|
1536
2895
|
}
|
|
1537
2896
|
setArchived(id, archived) {
|
|
1538
|
-
const run = this.
|
|
2897
|
+
const run = this.record(id);
|
|
1539
2898
|
if (!run)
|
|
1540
2899
|
return undefined;
|
|
1541
2900
|
this.applyArchivedCascade(run, archived);
|
|
@@ -1549,17 +2908,18 @@ export class RunStore extends EventEmitter {
|
|
|
1549
2908
|
// Owned workers nest under their parent only while both share Active/Archived (#250).
|
|
1550
2909
|
// Cascade from the parent only — archiving one worker stays independent.
|
|
1551
2910
|
if (run.delegation?.role !== 'worker') {
|
|
1552
|
-
for (const
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
}
|
|
2911
|
+
for (const { id } of this.listWorkersOf(run.id)) {
|
|
2912
|
+
const child = this.record(id);
|
|
2913
|
+
if (child && this.applyArchived(child, archived))
|
|
2914
|
+
changed++;
|
|
1557
2915
|
}
|
|
1558
2916
|
}
|
|
1559
2917
|
return changed;
|
|
1560
2918
|
}
|
|
1561
2919
|
applyArchived(run, archived) {
|
|
1562
2920
|
const changed = run.archived !== archived;
|
|
2921
|
+
// Transcript first: a restore throw aborts with the record unchanged (#818 S3).
|
|
2922
|
+
this.syncTranscriptForm(run.id, archived);
|
|
1563
2923
|
run.archived = archived;
|
|
1564
2924
|
run.archivedAt = archived ? new Date().toISOString() : undefined;
|
|
1565
2925
|
if (archived) {
|
|
@@ -1569,6 +2929,74 @@ export class RunStore extends EventEmitter {
|
|
|
1569
2929
|
this.touch(run);
|
|
1570
2930
|
return changed;
|
|
1571
2931
|
}
|
|
2932
|
+
/** Transcript form follows `archived`: enqueue compression, or restore the plain file now. */
|
|
2933
|
+
syncTranscriptForm(id, archived) {
|
|
2934
|
+
if (archived)
|
|
2935
|
+
this.compressor.enqueue(id);
|
|
2936
|
+
else {
|
|
2937
|
+
this.compressor.cancel(id);
|
|
2938
|
+
restoreHistory(this.dataDir, id);
|
|
2939
|
+
}
|
|
2940
|
+
}
|
|
2941
|
+
/**
|
|
2942
|
+
* Whether this store may compress `id`'s transcript: the run exists, is archived, is not in a
|
|
2943
|
+
* live status (queued/running/waiting), and this process holds the family's claim in `claimed`
|
|
2944
|
+
* (not merely `pendingClaims`). Two processes never both write a run (#779, #818 B1).
|
|
2945
|
+
*/
|
|
2946
|
+
isHistoryCompressEligible(id) {
|
|
2947
|
+
if (this.deleted.has(id))
|
|
2948
|
+
return false;
|
|
2949
|
+
try {
|
|
2950
|
+
const run = this.held.get(id) ?? this.peek(id);
|
|
2951
|
+
if (!run || run.archived !== true || isLiveStatus(run.status))
|
|
2952
|
+
return false;
|
|
2953
|
+
return this.holdsCompressClaim(id);
|
|
2954
|
+
}
|
|
2955
|
+
catch {
|
|
2956
|
+
return false;
|
|
2957
|
+
}
|
|
2958
|
+
}
|
|
2959
|
+
/** Take the family claim if free and require it to be in `this.claimed`, not only pending. */
|
|
2960
|
+
holdsCompressClaim(id) {
|
|
2961
|
+
const family = this.familyOf(id);
|
|
2962
|
+
return this.claimFamilies([family]).has(family) && this.claimed.has(family);
|
|
2963
|
+
}
|
|
2964
|
+
/** Startup sweep: enqueue every archived non-live run that still has a plain transcript. */
|
|
2965
|
+
compressArchivedHistory() {
|
|
2966
|
+
try {
|
|
2967
|
+
const ids = new Set();
|
|
2968
|
+
for (const key of this.db?.listKeysWhere('archived = 1 AND live = 0') ?? []) {
|
|
2969
|
+
if (!this.deleted.has(key.id))
|
|
2970
|
+
ids.add(key.id);
|
|
2971
|
+
}
|
|
2972
|
+
for (const run of this.held.values()) {
|
|
2973
|
+
if (run.archived && !isLiveStatus(run.status) && !this.deleted.has(run.id))
|
|
2974
|
+
ids.add(run.id);
|
|
2975
|
+
}
|
|
2976
|
+
for (const id of ids) {
|
|
2977
|
+
try {
|
|
2978
|
+
const { plain } = historyPaths(this.dataDir, id);
|
|
2979
|
+
// Claim only ids with work: a plain file to compress, including both-present leftovers.
|
|
2980
|
+
if (!existsSync(plain))
|
|
2981
|
+
continue;
|
|
2982
|
+
if (!this.holdsCompressClaim(id))
|
|
2983
|
+
continue;
|
|
2984
|
+
if (hasPlainHistory(this.dataDir, id))
|
|
2985
|
+
this.compressor.enqueue(id);
|
|
2986
|
+
}
|
|
2987
|
+
catch {
|
|
2988
|
+
// Best effort: one id must not stop the sweep or boot (#818 S2).
|
|
2989
|
+
}
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2992
|
+
catch {
|
|
2993
|
+
// listKeysWhere / open-time fs: never throw out of boot.
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
/** Resolves when the transcript compressor is idle (tests, shutdown). */
|
|
2997
|
+
historyIdle() {
|
|
2998
|
+
return this.compressor.idle();
|
|
2999
|
+
}
|
|
1572
3000
|
/** Pin one run to the top of this project's task list, or unpin it (#935). Mirrors
|
|
1573
3001
|
* `setArchived`: sets the fields, then persists + broadcasts via `touch`, so the updated
|
|
1574
3002
|
* record rides the existing `run` SSE with no new event. Idempotent — re-pinning a pinned
|
|
@@ -1577,7 +3005,7 @@ export class RunStore extends EventEmitter {
|
|
|
1577
3005
|
* Every active status is pinnable. Archived requests clear stale metadata instead, so a
|
|
1578
3006
|
* late click racing archive cannot leave a hidden pin that reappears on unarchive. */
|
|
1579
3007
|
setPinned(id, pinned) {
|
|
1580
|
-
const run = this.
|
|
3008
|
+
const run = this.record(id);
|
|
1581
3009
|
if (!run)
|
|
1582
3010
|
return undefined;
|
|
1583
3011
|
if (pinned && !run.archived) {
|
|
@@ -1590,28 +3018,43 @@ export class RunStore extends EventEmitter {
|
|
|
1590
3018
|
this.touch(run);
|
|
1591
3019
|
return run;
|
|
1592
3020
|
}
|
|
1593
|
-
/** Bulk-archive
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
3021
|
+
/** Bulk-archive finished runs (#780). `isSweepable` decides which; `archived` counts every
|
|
3022
|
+
* record that flipped (cascaded workers included), `ids` lists the top-level runs picked and
|
|
3023
|
+
* `pinnedIds` the subset that carried a pin before `clearPin` dropped it. */
|
|
3024
|
+
archiveFinished(scope) {
|
|
3025
|
+
let archived = 0;
|
|
3026
|
+
// Snapshot first: cascade may archive still-live workers, mutating the held set, and the pin
|
|
3027
|
+
// has to be read before `applyArchived` clears it. Runs not in memory are picked off their
|
|
3028
|
+
// stored summary, which carries every field `isSweepable` reads.
|
|
3029
|
+
const picked = [...this.held.values()].filter((run) => isSweepable(run, scope)).map((run) => [run, this.listOrder(run)]);
|
|
3030
|
+
for (const row of this.db?.listWhere("archived = 0 AND status IN ('done', 'failed', 'cancelled') AND parent_run_id IS NULL") ?? []) {
|
|
3031
|
+
if (this.held.has(row.id) || this.deleted.has(row.id))
|
|
3032
|
+
continue;
|
|
3033
|
+
const summary = parseStoredSummary(row.summary);
|
|
3034
|
+
if (summary && isSweepable(summary, scope))
|
|
3035
|
+
picked.push([summary, row]);
|
|
3036
|
+
}
|
|
3037
|
+
// Another process's runs are not this store's to archive (#779, plan step 3): they stay out.
|
|
3038
|
+
// A sweepable run is never a worker (`isSweepable`), so each is its own family.
|
|
3039
|
+
const writable = this.claimFamilies(picked.map(([run]) => run.id));
|
|
3040
|
+
const mine = sortNewestFirst(picked.filter(([run]) => writable.has(run.id)));
|
|
3041
|
+
const ids = mine.map((run) => run.id);
|
|
3042
|
+
const pinnedIds = mine.filter((run) => run.pinned).map((run) => run.id);
|
|
3043
|
+
for (const id of ids) {
|
|
3044
|
+
// Re-check: a prior cascade may already have archived this id.
|
|
3045
|
+
const run = this.record(id);
|
|
1603
3046
|
if (!run || run.archived)
|
|
1604
3047
|
continue;
|
|
1605
|
-
|
|
3048
|
+
archived += this.applyArchivedCascade(run, true);
|
|
1606
3049
|
}
|
|
1607
|
-
return
|
|
3050
|
+
return { archived, ids, pinnedIds };
|
|
1608
3051
|
}
|
|
1609
3052
|
/** Mark one run as read (#unread-done-items): stamp the read receipt now. Mirrors
|
|
1610
3053
|
* `setArchived` — sets the field then persists + broadcasts via `touch`, so the
|
|
1611
3054
|
* updated record rides the existing `run` SSE with no new event. Idempotent by
|
|
1612
3055
|
* design: opening an already-read thread just re-stamps a later `seenAt`. */
|
|
1613
3056
|
setRead(id) {
|
|
1614
|
-
const run = this.
|
|
3057
|
+
const run = this.record(id);
|
|
1615
3058
|
if (!run)
|
|
1616
3059
|
return undefined;
|
|
1617
3060
|
run.seenAt = new Date().toISOString();
|
|
@@ -1632,7 +3075,7 @@ export class RunStore extends EventEmitter {
|
|
|
1632
3075
|
* the marker. WHETHER the action means anything for a given run is UI policy, and lives
|
|
1633
3076
|
* in the cockpit's `runActionFlags` — the same split the rest of the store keeps. */
|
|
1634
3077
|
setUnread(id) {
|
|
1635
|
-
const run = this.
|
|
3078
|
+
const run = this.record(id);
|
|
1636
3079
|
if (!run)
|
|
1637
3080
|
return undefined;
|
|
1638
3081
|
delete run.seenAt;
|
|
@@ -1666,7 +3109,18 @@ export class RunStore extends EventEmitter {
|
|
|
1666
3109
|
markAllRead() {
|
|
1667
3110
|
const now = new Date().toISOString();
|
|
1668
3111
|
let count = 0;
|
|
1669
|
-
|
|
3112
|
+
// Runs not in memory are picked by the same rule over their stored summary, then re-checked
|
|
3113
|
+
// on the record before they are stamped.
|
|
3114
|
+
const candidates = [...this.held.values()];
|
|
3115
|
+
const stored = (this.db?.listWhere(MARK_ALL_READ_SQL) ?? []).filter((row) => !this.held.has(row.id) && !this.deleted.has(row.id));
|
|
3116
|
+
// One claim transaction for them all; another process's runs are skipped (`record` refuses).
|
|
3117
|
+
this.claimFamilies(stored.map(rowFamily));
|
|
3118
|
+
for (const row of stored) {
|
|
3119
|
+
const run = this.record(row.id);
|
|
3120
|
+
if (run)
|
|
3121
|
+
candidates.push(run);
|
|
3122
|
+
}
|
|
3123
|
+
for (const run of candidates) {
|
|
1670
3124
|
const unread = !run.archived &&
|
|
1671
3125
|
(run.status === 'done' || run.status === 'failed') &&
|
|
1672
3126
|
!(run.status === 'failed' && run.autoResumeAt !== undefined) &&
|
|
@@ -1681,9 +3135,13 @@ export class RunStore extends EventEmitter {
|
|
|
1681
3135
|
return count;
|
|
1682
3136
|
}
|
|
1683
3137
|
appendEvent(runId, event) {
|
|
1684
|
-
const run = this.
|
|
1685
|
-
if (!run)
|
|
3138
|
+
const run = this.record(runId);
|
|
3139
|
+
if (!run) {
|
|
3140
|
+
const ownership = this.runOwnership(runId);
|
|
3141
|
+
if (ownership === 'foreign' || ownership === 'orphaned' || ownership === 'quarantined')
|
|
3142
|
+
throw new RunWriteRefusedError(runId, ownership);
|
|
1686
3143
|
throw new Error(`unknown run: ${runId}`);
|
|
3144
|
+
}
|
|
1687
3145
|
const seq = this.nextSeq(runId);
|
|
1688
3146
|
// Scrub credentials before the event touches disk or the live wire (#427):
|
|
1689
3147
|
// tool-result output is persisted verbatim and served back over the API, so
|
|
@@ -1691,9 +3149,14 @@ export class RunStore extends EventEmitter {
|
|
|
1691
3149
|
const full = this.redact({ ...event, seq, ts: new Date().toISOString() });
|
|
1692
3150
|
// Sync append keeps event order without a write queue; local NDJSON
|
|
1693
3151
|
// appends at agent-event rates are effectively free.
|
|
1694
|
-
|
|
3152
|
+
if (!hasPlainHistory(this.dataDir, runId))
|
|
3153
|
+
restoreHistory(this.dataDir, runId);
|
|
3154
|
+
const line = `${JSON.stringify(full)}\n`;
|
|
3155
|
+
appendFileSync(this.eventsPath(runId), line, 'utf8');
|
|
3156
|
+
this.facts.append(runId, full, Buffer.byteLength(line));
|
|
3157
|
+
this.maybeEnqueueHistoryCompress(run);
|
|
1695
3158
|
if ((full.type === 'ask.requested' || full.type === 'human-input-delivered') &&
|
|
1696
|
-
|
|
3159
|
+
this.syncHumanAskSummary(run))
|
|
1697
3160
|
this.touch(run);
|
|
1698
3161
|
this.emit('event', { runId, event: full });
|
|
1699
3162
|
// The janitor trick: agents print the PR URL after `gh pr create` — the
|
|
@@ -1813,7 +3276,7 @@ export class RunStore extends EventEmitter {
|
|
|
1813
3276
|
* re-declare after it opens a PR, and taking that literally cost the task the PR it was about.
|
|
1814
3277
|
*/
|
|
1815
3278
|
applyMarkerRefs(runId, refs) {
|
|
1816
|
-
const run = this.
|
|
3279
|
+
const run = this.record(runId);
|
|
1817
3280
|
if (!run || (refs.pr === undefined && refs.issue === undefined))
|
|
1818
3281
|
return run;
|
|
1819
3282
|
run.markerRefs = {
|
|
@@ -1882,21 +3345,58 @@ export class RunStore extends EventEmitter {
|
|
|
1882
3345
|
this.secretValues = collectSecretValues();
|
|
1883
3346
|
return this.secretValues;
|
|
1884
3347
|
}
|
|
3348
|
+
/** The run's indexed transcript facts (#880): a read-only view, never a transcript read once
|
|
3349
|
+
* the run's index is loaded. An unreadable transcript answers as empty, as readEvents does. */
|
|
3350
|
+
transcriptFacts(runId) {
|
|
3351
|
+
return this.facts.get(runId) ?? emptyFacts();
|
|
3352
|
+
}
|
|
3353
|
+
hasProjection(runId, projectionId) {
|
|
3354
|
+
return this.transcriptFacts(runId).projectionIds.includes(projectionId);
|
|
3355
|
+
}
|
|
3356
|
+
factsWarming = Promise.resolve();
|
|
3357
|
+
/** After open, build the indexes the delegation reconcile will ask for first, off the request
|
|
3358
|
+
* path: every member of every family with a conversation (#880). */
|
|
3359
|
+
warmTranscriptFacts() {
|
|
3360
|
+
// A microtask, not a timer: open returns first, and no timer outlives a store that never warms.
|
|
3361
|
+
this.factsWarming = Promise.resolve().then(() => {
|
|
3362
|
+
const ids = [];
|
|
3363
|
+
for (const rootId of this.listConversationRootIds())
|
|
3364
|
+
ids.push(rootId, ...(this.db?.listIdsByParent(rootId) ?? []));
|
|
3365
|
+
return this.facts.warm(ids);
|
|
3366
|
+
}).catch(() => undefined);
|
|
3367
|
+
}
|
|
3368
|
+
/** Resolves when the open-time warm-up is done (tests). */
|
|
3369
|
+
factsWarmIdle() {
|
|
3370
|
+
return this.factsWarming;
|
|
3371
|
+
}
|
|
3372
|
+
/** `readEvents` off the event loop (async read and async brotli), for collection paths. */
|
|
3373
|
+
async readEventsAsync(runId) {
|
|
3374
|
+
try {
|
|
3375
|
+
const raw = await readHistoryTextAsync(this.dataDir, runId);
|
|
3376
|
+
return raw === undefined ? [] : parseEvents(raw);
|
|
3377
|
+
}
|
|
3378
|
+
catch {
|
|
3379
|
+
return [];
|
|
3380
|
+
}
|
|
3381
|
+
}
|
|
3382
|
+
/** The owned store's `refreshHumanAskSummary`, from the index, on the same rules: no history
|
|
3383
|
+
* (legacy runs) keeps the summary, an unreadable one requests attention. True when it changed. */
|
|
3384
|
+
syncHumanAskSummary(run) {
|
|
3385
|
+
const facts = this.facts.get(run.id);
|
|
3386
|
+
const next = !facts ? true
|
|
3387
|
+
: facts.bytes > 0 || facts.archive !== undefined ? facts.pendingAsk !== undefined
|
|
3388
|
+
: run.hasPendingHumanAsk === true;
|
|
3389
|
+
if (next === run.hasPendingHumanAsk)
|
|
3390
|
+
return false;
|
|
3391
|
+
run.hasPendingHumanAsk = next;
|
|
3392
|
+
return true;
|
|
3393
|
+
}
|
|
1885
3394
|
readEvents(runId) {
|
|
1886
3395
|
try {
|
|
1887
|
-
const raw =
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
.map((line) => {
|
|
1892
|
-
try {
|
|
1893
|
-
return JSON.parse(line);
|
|
1894
|
-
}
|
|
1895
|
-
catch {
|
|
1896
|
-
return null;
|
|
1897
|
-
}
|
|
1898
|
-
})
|
|
1899
|
-
.filter((e) => e !== null);
|
|
3396
|
+
const raw = readHistoryText(this.dataDir, runId);
|
|
3397
|
+
if (raw === undefined)
|
|
3398
|
+
return [];
|
|
3399
|
+
return parseEvents(raw);
|
|
1900
3400
|
}
|
|
1901
3401
|
catch {
|
|
1902
3402
|
return [];
|
|
@@ -1950,13 +3450,11 @@ export class RunStore extends EventEmitter {
|
|
|
1950
3450
|
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
1951
3451
|
try {
|
|
1952
3452
|
const info = fstatSync(fd);
|
|
1953
|
-
if (!info.isFile() || info.size >
|
|
3453
|
+
if (!info.isFile() || info.size > 16384 || (info.mode & 0o077))
|
|
1954
3454
|
return undefined;
|
|
1955
|
-
const buffer = Buffer.alloc(
|
|
3455
|
+
const buffer = Buffer.alloc(16385);
|
|
1956
3456
|
const count = readSync(fd, buffer, 0, buffer.length, 0);
|
|
1957
|
-
return
|
|
1958
|
-
.refine(proof => !proof.neverMaterialized || proof.phase === 'complete')
|
|
1959
|
-
.parse(JSON.parse(buffer.subarray(0, count).toString('utf8')));
|
|
3457
|
+
return workerExecutionSchema.parse(JSON.parse(buffer.subarray(0, count).toString('utf8')));
|
|
1960
3458
|
}
|
|
1961
3459
|
finally {
|
|
1962
3460
|
closeSync(fd);
|
|
@@ -1993,14 +3491,85 @@ export class RunStore extends EventEmitter {
|
|
|
1993
3491
|
rmSync(temporary, { force: true });
|
|
1994
3492
|
}
|
|
1995
3493
|
}
|
|
3494
|
+
/** Complete execution is not permission to reuse/delete persistent resources (#738).
|
|
3495
|
+
* Synchronous, fresh and generation/resource fenced; callers hold admission off across awaits. */
|
|
3496
|
+
workerResourcesSafe(id, generation, resourceId, admittingQueued = false) {
|
|
3497
|
+
return this.workerResourceHolders(id, generation, resourceId, { admittingQueued }) === 'safe';
|
|
3498
|
+
}
|
|
3499
|
+
/** `workerResourcesSafe` with the refusal's live PIDs (empty when no PID explains it). */
|
|
3500
|
+
workerResourceHolders(id, generation, resourceId, opts = {}) {
|
|
3501
|
+
const run = this.peek(id);
|
|
3502
|
+
const proof = this.readWorkerExecution(id);
|
|
3503
|
+
if (run?.delegation?.role !== 'worker' || run.delegation.workspace.ownerRunId !== id ||
|
|
3504
|
+
run.delegation.workspace.resourceId !== resourceId || proof?.generation !== generation ||
|
|
3505
|
+
(proof.phase !== 'complete' && !(opts.admittingQueued && proof.phase === 'queued')))
|
|
3506
|
+
return [];
|
|
3507
|
+
const record = this.readWorkerProcesses(id, generation);
|
|
3508
|
+
if (record === 'unknown' || !agentTmpDirOwnershipProven(this.dataDir, id))
|
|
3509
|
+
return [];
|
|
3510
|
+
const paths = [run.delegation.workspace.path, ...agentTmpDirLocations(this.dataDir, id)];
|
|
3511
|
+
// Nothing can be deleted/reused at an absent path. This lets explicit history deletion
|
|
3512
|
+
// retire completed cleanup despite ambient denial, while recorded survivors/unknown evidence
|
|
3513
|
+
// still block. Only ENOENT proves absence; existsSync would also hide access errors.
|
|
3514
|
+
const absent = paths.every(path => {
|
|
3515
|
+
try {
|
|
3516
|
+
lstatSync(path);
|
|
3517
|
+
return false;
|
|
3518
|
+
}
|
|
3519
|
+
catch (error) {
|
|
3520
|
+
return error.code === 'ENOENT';
|
|
3521
|
+
}
|
|
3522
|
+
});
|
|
3523
|
+
if (absent)
|
|
3524
|
+
return record === 'absent' ||
|
|
3525
|
+
((!recordedProcessLive(record.controller) || isCurrentProcess(record.controller)) && !record.processes.some(recordedProcessLive)) ? 'safe' : [];
|
|
3526
|
+
const probe = inspectGeneration({ ...(record === 'absent' ? {} : { record }), paths, cwds: opts.cwds });
|
|
3527
|
+
return probe.liveness === 'gone' ? 'safe' : probe.controller !== undefined ? [probe.controller] : probe.pids;
|
|
3528
|
+
}
|
|
3529
|
+
/** Upgrade a legacy completed checkpoint while terminal task ownership is still known.
|
|
3530
|
+
* No holder scan here: settlement and scheduling must never wait on cleanup proof. */
|
|
3531
|
+
retainWorkerScratchCleanup(id) {
|
|
3532
|
+
const run = this.peek(id), proof = this.readWorkerExecution(id);
|
|
3533
|
+
if (run?.delegation?.role !== 'worker' || ['queued', 'running', 'waiting'].includes(run.status) || proof?.phase !== 'complete')
|
|
3534
|
+
return;
|
|
3535
|
+
const { resourceId, path } = run.delegation.workspace;
|
|
3536
|
+
if (proof.scratchCleanup?.resourceId === resourceId && proof.scratchCleanup.path === path)
|
|
3537
|
+
return;
|
|
3538
|
+
this.writeWorkerExecution(id, { ...proof, scratchCleanup: { resourceId, path } });
|
|
3539
|
+
}
|
|
3540
|
+
/** Cleanup can outlive its index row, but never its generation or terminal task intent.
|
|
3541
|
+
* `cwds` is the scratch reprobe tick's shared `/proc` snapshot (hearsay-tools/cezarion#879). */
|
|
3542
|
+
workerScratchResourcesSafe(id, generation, resourceId, cwds) {
|
|
3543
|
+
const run = this.peek(id), proof = this.readWorkerExecution(id);
|
|
3544
|
+
if (proof?.phase !== 'complete' || proof.generation !== generation || proof.scratchCleanup?.resourceId !== resourceId ||
|
|
3545
|
+
(run && ['queued', 'running', 'waiting'].includes(run.status)))
|
|
3546
|
+
return false;
|
|
3547
|
+
if (run?.delegation?.role === 'worker')
|
|
3548
|
+
return run.delegation.workspace.path === proof.scratchCleanup.path &&
|
|
3549
|
+
this.workerResourceHolders(id, generation, resourceId, { cwds }) === 'safe';
|
|
3550
|
+
// A valid different role contradicts the retained intent. Quarantined/missing metadata
|
|
3551
|
+
// supplies no new authority; only the private terminal checkpoint authorizes scratch.
|
|
3552
|
+
if (run && run.delegation?.role !== 'invalid')
|
|
3553
|
+
return false;
|
|
3554
|
+
const record = this.readWorkerProcesses(id, generation);
|
|
3555
|
+
if (record === 'unknown' || !agentTmpDirOwnershipProven(this.dataDir, id))
|
|
3556
|
+
return false;
|
|
3557
|
+
return inspectGeneration({ ...(record === 'absent' ? {} : { record }),
|
|
3558
|
+
paths: [proof.scratchCleanup.path, ...agentTmpDirLocations(this.dataDir, id)], cwds }).liveness === 'gone';
|
|
3559
|
+
}
|
|
1996
3560
|
commitWorkerExecutionStart(id) {
|
|
1997
|
-
const run = this.
|
|
3561
|
+
const run = this.peek(id);
|
|
1998
3562
|
if (run?.delegation?.role !== 'worker' || run.delegation.destroy)
|
|
1999
3563
|
throw new Error('Worker cannot start');
|
|
2000
3564
|
const prior = this.readWorkerExecution(id);
|
|
3565
|
+
if (prior?.abandoned)
|
|
3566
|
+
throw new Error('Worker execution was abandoned; spawn a new worker instead of resuming it');
|
|
2001
3567
|
if (!prior || (prior.phase !== 'queued' && prior.phase !== 'complete')) {
|
|
2002
3568
|
throw new Error('Worker execution checkpoint does not prove safe admission');
|
|
2003
3569
|
}
|
|
3570
|
+
if (!this.workerResourcesSafe(id, prior.generation, run.delegation.workspace.resourceId, true)) {
|
|
3571
|
+
throw new Error('Worker resources may still be held by a process; reuse is not proven safe');
|
|
3572
|
+
}
|
|
2004
3573
|
const generation = randomUUID();
|
|
2005
3574
|
// #469: the controller is durable before `starting`, so a crash leaves a provable generation.
|
|
2006
3575
|
this.writeWorkerProcesses(id, { generation, controller: { pid: process.pid, ...startToken(process.pid) }, processes: [] });
|
|
@@ -2062,11 +3631,11 @@ export class RunStore extends EventEmitter {
|
|
|
2062
3631
|
}
|
|
2063
3632
|
/** Bound to the current generation. Past the cap the record never drops an entry: the
|
|
2064
3633
|
* working-directory scan is the remaining evidence. */
|
|
2065
|
-
appendWorkerProcess(id, generation, pid) {
|
|
3634
|
+
appendWorkerProcess(id, generation, pid, pgid) {
|
|
2066
3635
|
const record = this.readWorkerProcesses(id, generation);
|
|
2067
3636
|
if (typeof record === 'string' || this.readWorkerExecution(id)?.generation !== generation)
|
|
2068
3637
|
return false;
|
|
2069
|
-
const entry = { pid, ...startToken(pid) };
|
|
3638
|
+
const entry = { pid, ...startToken(pid), ...(pgid === undefined ? {} : { pgid }) };
|
|
2070
3639
|
if (record.processes.some(known => known.pid === entry.pid && known.startToken === entry.startToken))
|
|
2071
3640
|
return true;
|
|
2072
3641
|
if (record.processes.length >= WORKER_PROCESS_CAP)
|
|
@@ -2078,20 +3647,25 @@ export class RunStore extends EventEmitter {
|
|
|
2078
3647
|
const proof = this.readWorkerExecution(id);
|
|
2079
3648
|
if (!proof || proof.generation !== generation)
|
|
2080
3649
|
return false;
|
|
2081
|
-
if (proof.phase === 'queued' && this.
|
|
3650
|
+
if (proof.phase === 'queued' && this.peek(id)?.status !== 'cancelled')
|
|
2082
3651
|
return false;
|
|
2083
3652
|
try {
|
|
2084
|
-
const run = this.
|
|
3653
|
+
const run = this.peek(id);
|
|
2085
3654
|
if (run?.delegation?.role !== 'worker')
|
|
2086
3655
|
return false;
|
|
2087
|
-
this.commitIndex(new Map(
|
|
3656
|
+
this.commitIndex(new Map([[id, run]]));
|
|
2088
3657
|
if (this.readWorkerExecution(id)?.generation !== generation)
|
|
2089
3658
|
return false;
|
|
2090
3659
|
this.writeWorkerExecution(id, { generation, phase: 'complete',
|
|
2091
|
-
|
|
3660
|
+
// Nothing abandons an execution any more (hearsay-tools/cezarion#889); keep a checkpoint that already was.
|
|
3661
|
+
...(proof.abandoned ? { abandoned: true } : {}),
|
|
3662
|
+
...(proof.phase === 'queued' || proof.neverMaterialized ? { neverMaterialized: true } : {}),
|
|
3663
|
+
...(!['queued', 'running', 'waiting'].includes(run.status) ? { scratchCleanup: {
|
|
3664
|
+
resourceId: run.delegation.workspace.resourceId, path: run.delegation.workspace.path
|
|
3665
|
+
} } : {}) });
|
|
2092
3666
|
// The earlier index event cannot attest exit: subscribers must observe the
|
|
2093
3667
|
// durable private proof before a terminal worker can satisfy a lifecycle wait.
|
|
2094
|
-
this.emit('run', this.
|
|
3668
|
+
this.emit('run', this.peek(id));
|
|
2095
3669
|
return true;
|
|
2096
3670
|
}
|
|
2097
3671
|
catch {
|
|
@@ -2100,18 +3674,16 @@ export class RunStore extends EventEmitter {
|
|
|
2100
3674
|
}
|
|
2101
3675
|
/** Cancellation must hit disk before queued no-start evidence authorizes cleanup. */
|
|
2102
3676
|
commitWorkerCancellation(id) {
|
|
2103
|
-
const run = this.
|
|
3677
|
+
const run = this.peek(id);
|
|
2104
3678
|
if (run?.delegation?.role !== 'worker')
|
|
2105
3679
|
throw new Error('Worker not found');
|
|
2106
|
-
|
|
2107
|
-
proposed.set(id, { ...run, status: 'cancelled', finishedAt: new Date().toISOString() });
|
|
2108
|
-
this.commitIndex(proposed, new Set([id]));
|
|
3680
|
+
this.commitIndex(new Map([[id, { ...run, status: 'cancelled', finishedAt: new Date().toISOString() }]]));
|
|
2109
3681
|
}
|
|
2110
3682
|
/** Complete replacement evidence can stand in for absent child history, never for a present execution. */
|
|
2111
3683
|
readDeletedWorkerResult(parentId, workerId) {
|
|
2112
|
-
if (this.
|
|
3684
|
+
if (this.hasRun(workerId))
|
|
2113
3685
|
return undefined;
|
|
2114
|
-
const parent = this.
|
|
3686
|
+
const parent = this.peek(parentId);
|
|
2115
3687
|
if (parent?.delegation?.role !== 'root' || parent.delegation.historyDeletion)
|
|
2116
3688
|
return undefined;
|
|
2117
3689
|
const deletion = parent.delegation.receipts.find(receipt => receipt.workerId === workerId)?.deletion;
|
|
@@ -2124,12 +3696,12 @@ export class RunStore extends EventEmitter {
|
|
|
2124
3696
|
}
|
|
2125
3697
|
/** A deleted child's receipt replaces the private proof only after the deletion checkpoint. */
|
|
2126
3698
|
workerDeletionEvidence(id) {
|
|
2127
|
-
const child = this.
|
|
3699
|
+
const child = this.peek(id);
|
|
2128
3700
|
if (child?.delegation?.role !== 'worker' || child.delegation.workspace.ownerRunId !== id ||
|
|
2129
3701
|
child.delegation.destroy?.phase !== 'complete' || child.delegation.destroy.remaining.length ||
|
|
2130
3702
|
!['review', 'done', 'failed', 'cancelled'].includes(child.status))
|
|
2131
3703
|
return false;
|
|
2132
|
-
const parent = this.
|
|
3704
|
+
const parent = this.peek(child.delegation.parentRunId);
|
|
2133
3705
|
if (parent?.delegation?.role !== 'root' || parent.delegation.historyDeletion)
|
|
2134
3706
|
return false;
|
|
2135
3707
|
const receipt = parent.delegation.receipts.find(entry => entry.workerId === id);
|
|
@@ -2141,6 +3713,12 @@ export class RunStore extends EventEmitter {
|
|
|
2141
3713
|
if (JSON.stringify(workspace) !== JSON.stringify(child.delegation.workspace))
|
|
2142
3714
|
return false;
|
|
2143
3715
|
const proof = this.readWorkerExecution(id);
|
|
3716
|
+
// Retain process evidence until scratch is safely removed. A prior pending deletion may
|
|
3717
|
+
// already have removed those files; it never authorizes deleting newly appeared scratch.
|
|
3718
|
+
if (proof && !this.workerResourcesSafe(id, proof.generation, workspace.resourceId))
|
|
3719
|
+
return false;
|
|
3720
|
+
if (!proof && agentTmpDirMayExist(this.dataDir, id))
|
|
3721
|
+
return false;
|
|
2144
3722
|
if (receipt.deletion)
|
|
2145
3723
|
return receipt.deletion.revision === result.revision &&
|
|
2146
3724
|
receipt.deletion.resourceId === workspace.resourceId && receipt.deletion.phase === 'pending' &&
|
|
@@ -2148,19 +3726,22 @@ export class RunStore extends EventEmitter {
|
|
|
2148
3726
|
return proof?.phase === 'complete';
|
|
2149
3727
|
}
|
|
2150
3728
|
canDeleteRun(id) {
|
|
2151
|
-
const run = this.
|
|
3729
|
+
const run = this.peek(id);
|
|
2152
3730
|
if (run?.delegation?.role === 'invalid')
|
|
2153
3731
|
return false;
|
|
2154
3732
|
if (run?.delegation?.role === 'worker')
|
|
2155
3733
|
return this.workerDeletionEvidence(id);
|
|
3734
|
+
const privateWorkers = workerEvidenceRunIds(this.dataDir);
|
|
3735
|
+
if (!privateWorkers || privateWorkers.includes(id))
|
|
3736
|
+
return false;
|
|
2156
3737
|
if (run?.delegation?.role === 'root') {
|
|
2157
3738
|
if (run.delegation.finishRequestedAt && ['queued', 'running', 'waiting'].includes(run.status))
|
|
2158
3739
|
return false;
|
|
2159
3740
|
// Keep the parent's receipt and result ownership until each child history is explicitly removed.
|
|
2160
|
-
if (
|
|
3741
|
+
if (this.listWorkersOf(id).length > 0)
|
|
2161
3742
|
return false;
|
|
2162
3743
|
return run.delegation.receipts.every(receipt => {
|
|
2163
|
-
if (this.
|
|
3744
|
+
if (this.hasRun(receipt.workerId) || receipt.deletion?.phase !== 'complete')
|
|
2164
3745
|
return false;
|
|
2165
3746
|
// A pending parent deletion already validated the results before removing their bytes.
|
|
2166
3747
|
if (run.delegation?.role === 'root' && run.delegation.historyDeletion === 'pending')
|
|
@@ -2168,14 +3749,14 @@ export class RunStore extends EventEmitter {
|
|
|
2168
3749
|
return this.readDeletedWorkerResult(id, receipt.workerId) !== undefined;
|
|
2169
3750
|
});
|
|
2170
3751
|
}
|
|
2171
|
-
return
|
|
3752
|
+
return this.listWorkersOf(id).length === 0;
|
|
2172
3753
|
}
|
|
2173
3754
|
/** Delegated history deletion is synchronous and checkpointed; any failed byte/index removal remains retryable. */
|
|
2174
3755
|
deleteDelegatedRun(id) {
|
|
2175
|
-
const run = this.
|
|
3756
|
+
const run = this.peek(id);
|
|
2176
3757
|
try {
|
|
2177
3758
|
if (run.delegation?.role === 'worker') {
|
|
2178
|
-
const parent = this.
|
|
3759
|
+
const parent = this.peek(run.delegation.parentRunId);
|
|
2179
3760
|
if (parent.delegation?.role !== 'root')
|
|
2180
3761
|
return false;
|
|
2181
3762
|
const receipt = parent.delegation.receipts.find(entry => entry.workerId === id);
|
|
@@ -2194,6 +3775,10 @@ export class RunStore extends EventEmitter {
|
|
|
2194
3775
|
} : result.artifacts,
|
|
2195
3776
|
}, this.readWorkerResultDiff(parent.id, id));
|
|
2196
3777
|
this.removeRunHistoryBytes(id);
|
|
3778
|
+
// Delete scratch while its generation evidence still exists; failed removal keeps it.
|
|
3779
|
+
removeAgentTmpDir(this.dataDir, id);
|
|
3780
|
+
if (agentTmpDirMayExist(this.dataDir, id))
|
|
3781
|
+
return false;
|
|
2197
3782
|
// Private process/account evidence is now replaced by the exact parent receipt.
|
|
2198
3783
|
rmSync(this.identityPath(id), { force: true });
|
|
2199
3784
|
rmSync(this.processesPath(id), { force: true });
|
|
@@ -2205,20 +3790,15 @@ export class RunStore extends EventEmitter {
|
|
|
2205
3790
|
items: retained.artifacts.items.map(item => ({ state: 'deleted', reason: 'missing', id: item.id, path: item.path })),
|
|
2206
3791
|
} : retained.artifacts,
|
|
2207
3792
|
}, this.readWorkerResultDiff(parent.id, id));
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
receipts: parent.delegation.receipts.map(entry => entry.workerId === id ? { ...entry, deletion: { ...deletion, phase: 'complete' } } : entry),
|
|
2212
|
-
} });
|
|
2213
|
-
this.commitIndex(proposed, new Set([parent.id, id]));
|
|
3793
|
+
this.commitIndex(new Map([[parent.id, { ...parent, delegation: { ...parent.delegation,
|
|
3794
|
+
receipts: parent.delegation.receipts.map(entry => entry.workerId === id ? { ...entry, deletion: { ...deletion, phase: 'complete' } } : entry),
|
|
3795
|
+
} }], [id, null]]));
|
|
2214
3796
|
}
|
|
2215
3797
|
else if (run.delegation?.role === 'root') {
|
|
2216
3798
|
this.commitDelegation([{ id, delegation: { ...run.delegation, historyDeletion: 'pending' } }]);
|
|
2217
3799
|
this.removeRunHistoryBytes(id);
|
|
2218
3800
|
rmSync(join(this.dataDir, 'runs', `${id}-worker-results`), { recursive: true, force: true });
|
|
2219
|
-
|
|
2220
|
-
proposed.delete(id);
|
|
2221
|
-
this.commitIndex(proposed, new Set([id]));
|
|
3801
|
+
this.commitIndex(new Map([[id, null]]));
|
|
2222
3802
|
}
|
|
2223
3803
|
else
|
|
2224
3804
|
return false;
|
|
@@ -2236,7 +3816,8 @@ export class RunStore extends EventEmitter {
|
|
|
2236
3816
|
const dir = join(this.dataDir, 'runs');
|
|
2237
3817
|
if (realpathSync(dir) !== resolve(dir))
|
|
2238
3818
|
throw new Error('History storage redirected');
|
|
2239
|
-
|
|
3819
|
+
this.facts.forget(id);
|
|
3820
|
+
removeHistory(this.dataDir, id);
|
|
2240
3821
|
rmSync(this.handoffPath(id), { force: true });
|
|
2241
3822
|
rmSync(this.imagesDir(id), { recursive: true, force: true });
|
|
2242
3823
|
removeArtifacts(this.dataDir, id);
|
|
@@ -2244,18 +3825,15 @@ export class RunStore extends EventEmitter {
|
|
|
2244
3825
|
deleteRun(id) {
|
|
2245
3826
|
if (!this.canDeleteRun(id))
|
|
2246
3827
|
return false;
|
|
2247
|
-
const run = this.
|
|
3828
|
+
const run = this.peek(id);
|
|
3829
|
+
if (run && this.db && (this.quarantined.has(id) || !this.claimFamilies([familyKey(run)]).size))
|
|
3830
|
+
return false;
|
|
2248
3831
|
if (run?.delegation?.role === 'worker' || run?.delegation?.role === 'root')
|
|
2249
3832
|
return this.deleteDelegatedRun(id);
|
|
2250
|
-
const existed =
|
|
3833
|
+
const existed = run !== undefined;
|
|
3834
|
+
this.held.delete(id);
|
|
2251
3835
|
if (existed) {
|
|
2252
|
-
|
|
2253
|
-
rmSync(this.eventsPath(id), { force: true });
|
|
2254
|
-
rmSync(this.handoffPath(id), { force: true });
|
|
2255
|
-
rmSync(this.imagesDir(id), { recursive: true, force: true });
|
|
2256
|
-
removeArtifacts(this.dataDir, id);
|
|
2257
|
-
}
|
|
2258
|
-
catch { /* Ordinary run deletion preserves its existing best-effort behavior. */ }
|
|
3836
|
+
this.markDeleted(id, familyKey(run));
|
|
2259
3837
|
removeAgentTmpDir(this.dataDir, id);
|
|
2260
3838
|
this.seqs.delete(id);
|
|
2261
3839
|
this.scheduleSave();
|
|
@@ -2263,7 +3841,7 @@ export class RunStore extends EventEmitter {
|
|
|
2263
3841
|
}
|
|
2264
3842
|
return existed;
|
|
2265
3843
|
}
|
|
2266
|
-
/** Write the
|
|
3844
|
+
/** Write the pending rows out now (used on shutdown). */
|
|
2267
3845
|
flush() {
|
|
2268
3846
|
if (this.saveTimer) {
|
|
2269
3847
|
clearTimeout(this.saveTimer);
|
|
@@ -2284,15 +3862,10 @@ export class RunStore extends EventEmitter {
|
|
|
2284
3862
|
* resumed event, even across a reload (the frozen-transcript symptom class
|
|
2285
3863
|
* of #424). One file read on the first post-restart append per run. */
|
|
2286
3864
|
rehydrateSeq(runId) {
|
|
2287
|
-
|
|
2288
|
-
for (const event of this.readEvents(runId)) {
|
|
2289
|
-
if (typeof event.seq === 'number' && event.seq > max)
|
|
2290
|
-
max = event.seq;
|
|
2291
|
-
}
|
|
2292
|
-
return max;
|
|
3865
|
+
return this.transcriptFacts(runId).lastSeq;
|
|
2293
3866
|
}
|
|
2294
3867
|
eventsPath(runId) {
|
|
2295
|
-
return
|
|
3868
|
+
return historyPaths(this.dataDir, runId).plain;
|
|
2296
3869
|
}
|
|
2297
3870
|
/** Same location `handoffPath()` in handoff.ts produces — inlined to keep
|
|
2298
3871
|
* the store free of upward imports. */
|
|
@@ -2304,38 +3877,90 @@ export class RunStore extends EventEmitter {
|
|
|
2304
3877
|
return join(this.dataDir, 'runs', `${runId}-images`);
|
|
2305
3878
|
}
|
|
2306
3879
|
touch(run) {
|
|
3880
|
+
this.dirty.add(run.id);
|
|
2307
3881
|
this.scheduleSave();
|
|
2308
3882
|
this.emit('run', run);
|
|
3883
|
+
this.maybeEnqueueHistoryCompress(run);
|
|
3884
|
+
}
|
|
3885
|
+
/** Form follows archived: a terminal archived run with a plain transcript is queued again. */
|
|
3886
|
+
maybeEnqueueHistoryCompress(run) {
|
|
3887
|
+
if (run.archived && !isLiveStatus(run.status) && hasPlainHistory(this.dataDir, run.id)) {
|
|
3888
|
+
this.compressor.enqueue(run.id);
|
|
3889
|
+
}
|
|
3890
|
+
}
|
|
3891
|
+
/** The held record is gone; the next save or commit deletes its row, fenced by `family`'s claim
|
|
3892
|
+
* and by the revision this store last saw (kept in `base` until the delete is written). */
|
|
3893
|
+
markDeleted(id, family) {
|
|
3894
|
+
const read = this.coldBase.get(id);
|
|
3895
|
+
if (!this.base.has(id) && read)
|
|
3896
|
+
this.base.set(id, storedRow(read));
|
|
3897
|
+
this.dirty.delete(id);
|
|
3898
|
+
this.deleted.add(id);
|
|
3899
|
+
this.deletedFamilies.set(id, family);
|
|
3900
|
+
this.historyOwed.add(id);
|
|
3901
|
+
}
|
|
3902
|
+
/** A deleted run's history files, once the delete of its row has committed. Best effort, as
|
|
3903
|
+
* deletion always was. */
|
|
3904
|
+
removeOwedHistory(id) {
|
|
3905
|
+
if (!this.historyOwed.delete(id))
|
|
3906
|
+
return;
|
|
3907
|
+
try {
|
|
3908
|
+
this.facts.forget(id);
|
|
3909
|
+
removeHistory(this.dataDir, id);
|
|
3910
|
+
rmSync(this.handoffPath(id), { force: true });
|
|
3911
|
+
rmSync(this.imagesDir(id), { recursive: true, force: true });
|
|
3912
|
+
removeArtifacts(this.dataDir, id);
|
|
3913
|
+
}
|
|
3914
|
+
catch { /* best effort */ }
|
|
2309
3915
|
}
|
|
3916
|
+
/**
|
|
3917
|
+
* Count-based history retention: of the runs past the newest `MAX_RUNS_KEPT` unarchived ones,
|
|
3918
|
+
* archive each finished run without delegation that is not pinned and whose family this store
|
|
3919
|
+
* can claim. Ranked over the row keys (the `created_at` index, no record decoded) with memory
|
|
3920
|
+
* laid over them, since a run created or archived since the last save has no row yet; the rows
|
|
3921
|
+
* past the cut are then filtered in `runs.db`, so only real candidates decode. Archived runs
|
|
3922
|
+
* are never deleted by retention — only an explicit delete, worker destruction, or history
|
|
3923
|
+
* deletion removes a run.
|
|
3924
|
+
*/
|
|
2310
3925
|
pruneOldRuns() {
|
|
2311
|
-
const
|
|
2312
|
-
const
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
for (const
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
3926
|
+
const ranked = new Map();
|
|
3927
|
+
for (const key of this.db?.listKeysWhere('archived = ?', [0]) ?? []) {
|
|
3928
|
+
if (!this.held.has(key.id) && !this.deleted.has(key.id))
|
|
3929
|
+
ranked.set(key.id, key);
|
|
3930
|
+
}
|
|
3931
|
+
for (const run of this.held.values())
|
|
3932
|
+
if (!run.archived)
|
|
3933
|
+
ranked.set(run.id, { id: run.id, ...this.listOrder(run) });
|
|
3934
|
+
const overflow = [...ranked.values()].sort(newestFirst).slice(MAX_RUNS_KEPT).map((key) => key.id);
|
|
3935
|
+
if (overflow.length === 0)
|
|
3936
|
+
return;
|
|
3937
|
+
const stale = overflow.flatMap((id) => {
|
|
3938
|
+
const run = this.held.get(id);
|
|
3939
|
+
// Retention must not archive a live task between turns, a pinned run, or a delegation
|
|
3940
|
+
// family whose history is promised until explicit deletion.
|
|
3941
|
+
return run && !isLiveStatus(run.status) && !run.delegation && !run.pinned ? [id] : [];
|
|
3942
|
+
});
|
|
3943
|
+
const cold = overflow.filter((id) => !this.held.has(id));
|
|
3944
|
+
if (cold.length > 0) {
|
|
3945
|
+
// Another process's runs are its own to keep or archive (#779, plan step 3).
|
|
3946
|
+
const rows = this.db.listWhere(RETENTION_CANDIDATES_SQL, [JSON.stringify(cold)]);
|
|
3947
|
+
const writable = this.claimFamilies(rows.map(rowFamily));
|
|
3948
|
+
for (const row of rows)
|
|
3949
|
+
if (writable.has(rowFamily(row)))
|
|
3950
|
+
stale.push(row.id);
|
|
3951
|
+
}
|
|
3952
|
+
for (const id of stale) {
|
|
3953
|
+
const run = this.record(id);
|
|
3954
|
+
if (!run || run.archived)
|
|
2322
3955
|
continue;
|
|
2323
|
-
this.
|
|
2324
|
-
|
|
2325
|
-
rmSync(this.eventsPath(stale.id), { force: true });
|
|
2326
|
-
rmSync(this.handoffPath(stale.id), { force: true });
|
|
2327
|
-
rmSync(this.imagesDir(stale.id), { recursive: true, force: true });
|
|
2328
|
-
removeArtifacts(this.dataDir, stale.id);
|
|
2329
|
-
}
|
|
2330
|
-
catch {
|
|
2331
|
-
// best effort
|
|
2332
|
-
}
|
|
2333
|
-
removeAgentTmpDir(this.dataDir, stale.id);
|
|
3956
|
+
this.applyArchived(run, true);
|
|
3957
|
+
removeAgentTmpDir(this.dataDir, id);
|
|
2334
3958
|
}
|
|
2335
3959
|
}
|
|
2336
|
-
/** Debounced so token-usage updates don't rewrite
|
|
3960
|
+
/** Debounced so token-usage updates don't rewrite a row per event. Nothing to schedule without
|
|
3961
|
+
* a database: a `RunStore.unavailable` store, or a closed one, writes nothing. */
|
|
2337
3962
|
scheduleSave() {
|
|
2338
|
-
if (this.saveTimer)
|
|
3963
|
+
if (this.saveTimer || !this.db)
|
|
2339
3964
|
return;
|
|
2340
3965
|
this.saveTimer = setTimeout(() => {
|
|
2341
3966
|
this.saveTimer = null;
|
|
@@ -2343,30 +3968,234 @@ export class RunStore extends EventEmitter {
|
|
|
2343
3968
|
}, 300);
|
|
2344
3969
|
this.saveTimer.unref?.();
|
|
2345
3970
|
}
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
writeFileSync(tmpPath, JSON.stringify(runs, null, 2), 'utf8');
|
|
2350
|
-
renameSync(tmpPath, indexPath);
|
|
3971
|
+
/** The single database write, so a test can fail it the way a full disk would. */
|
|
3972
|
+
writeIndex(changes) {
|
|
3973
|
+
return this.db.transaction(changes);
|
|
2351
3974
|
}
|
|
2352
|
-
|
|
3975
|
+
/**
|
|
3976
|
+
* Write `staged` plus everything still pending — dirty rows from memory, deletions — in one
|
|
3977
|
+
* transaction, then clear the pending marks. Throws with nothing cleared when it fails.
|
|
3978
|
+
* Never copies or sorts the whole store: the cost is the rows that changed.
|
|
3979
|
+
*
|
|
3980
|
+
* Fenced (#779, plan step 3): the transaction commits only if every family it writes still
|
|
3981
|
+
* carries this store's claim and every row is at the revision this store last saw. Otherwise
|
|
3982
|
+
* nothing is written and `handleConflicts` deals with the rows that changed under it.
|
|
3983
|
+
*/
|
|
3984
|
+
persist(staged) {
|
|
3985
|
+
const upserts = [];
|
|
3986
|
+
const deletes = [];
|
|
3987
|
+
const families = new Map();
|
|
3988
|
+
const extras = new Map();
|
|
3989
|
+
const encode = (run) => {
|
|
3990
|
+
const encoded = encodeRawRecord(run, (this.base.get(run.id) ?? this.coldBase.get(run.id))?.extras);
|
|
3991
|
+
extras.set(run.id, encoded.extras);
|
|
3992
|
+
upserts.push(encodeRunRow(run, encoded.data));
|
|
3993
|
+
};
|
|
3994
|
+
for (const [id, next] of staged) {
|
|
3995
|
+
if (next)
|
|
3996
|
+
encode(next);
|
|
3997
|
+
else
|
|
3998
|
+
deletes.push(id);
|
|
3999
|
+
families.set(id, next ? familyKey(next) : this.familyOf(id));
|
|
4000
|
+
}
|
|
4001
|
+
for (const id of this.dirty) {
|
|
4002
|
+
const run = this.held.get(id);
|
|
4003
|
+
if (!run || staged.has(id))
|
|
4004
|
+
continue;
|
|
4005
|
+
encode(run);
|
|
4006
|
+
families.set(id, familyKey(run));
|
|
4007
|
+
}
|
|
4008
|
+
for (const id of this.deleted) {
|
|
4009
|
+
if (staged.has(id))
|
|
4010
|
+
continue;
|
|
4011
|
+
deletes.push(id);
|
|
4012
|
+
families.set(id, this.familyOf(id));
|
|
4013
|
+
}
|
|
4014
|
+
// A row's insertion order (`seq`) is the order this write inserts it in, and it is where the
|
|
4015
|
+
// run lists among runs created in the same millisecond. Insert new runs in creation order (the
|
|
4016
|
+
// held set's, then staged runs not held yet), whether staged or dirty brought them here.
|
|
4017
|
+
if (upserts.filter((row) => !this.base.has(row.id)).length > 1) {
|
|
4018
|
+
const created = new Map();
|
|
4019
|
+
for (const id of [...this.held.keys(), ...staged.keys()])
|
|
4020
|
+
if (!created.has(id))
|
|
4021
|
+
created.set(id, created.size);
|
|
4022
|
+
upserts.sort((a, b) => created.get(a.id) - created.get(b.id));
|
|
4023
|
+
}
|
|
4024
|
+
// A held run whose write moved it into a new family (a worker quarantined to `invalid`) takes
|
|
4025
|
+
// that family's claim first; one it cannot take fails the fence below as claim-lost.
|
|
4026
|
+
const moved = [...new Set(families.values())].filter((family) => !this.claimed.has(family) && !this.pendingClaims.has(family));
|
|
4027
|
+
if (moved.length > 0)
|
|
4028
|
+
this.claimFamilies(moved, { allowLive: true });
|
|
4029
|
+
let commit;
|
|
2353
4030
|
try {
|
|
2354
|
-
this.writeIndex(this.
|
|
4031
|
+
commit = this.writeIndex({ upserts, deletes, ...(this.owner ? { fence: this.fence(families) } : {}) });
|
|
2355
4032
|
}
|
|
2356
|
-
catch (
|
|
2357
|
-
if (
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
4033
|
+
catch (error) {
|
|
4034
|
+
if (error instanceof RunConflictError)
|
|
4035
|
+
this.handleConflicts(error, staged);
|
|
4036
|
+
throw error;
|
|
4037
|
+
}
|
|
4038
|
+
for (const [family, generation] of commit.claims) {
|
|
4039
|
+
this.claimed.set(family, generation);
|
|
4040
|
+
this.pendingClaims.delete(family);
|
|
4041
|
+
}
|
|
4042
|
+
for (const row of upserts) {
|
|
4043
|
+
this.base.set(row.id, { revision: commit.revisions.get(row.id), seq: commit.seqs.get(row.id), data: row.data, extras: extras.get(row.id) });
|
|
4044
|
+
}
|
|
4045
|
+
for (const id of deletes) {
|
|
4046
|
+
this.base.delete(id);
|
|
4047
|
+
this.deletedFamilies.delete(id);
|
|
4048
|
+
this.removeOwedHistory(id);
|
|
4049
|
+
}
|
|
4050
|
+
this.dirty.clear();
|
|
4051
|
+
this.deleted.clear();
|
|
4052
|
+
if (this.saveTimer) {
|
|
4053
|
+
clearTimeout(this.saveTimer);
|
|
4054
|
+
this.saveTimer = null;
|
|
4055
|
+
}
|
|
4056
|
+
}
|
|
4057
|
+
/** What a write of these rows (id → family) must still find: this store's claims, at the
|
|
4058
|
+
* generations it took them, and each row at the revision it last saw (none for a new row). */
|
|
4059
|
+
fence(families) {
|
|
4060
|
+
const claims = new Map();
|
|
4061
|
+
const rows = new Map();
|
|
4062
|
+
for (const [id, family] of families) {
|
|
4063
|
+
const pending = this.pendingClaims.get(family);
|
|
4064
|
+
// Generations start at 1, so an unclaimed family's 0 fails the fence as claim-lost.
|
|
4065
|
+
claims.set(family, pending !== undefined ? { take: pending } : { generation: this.claimed.get(family) ?? 0 });
|
|
4066
|
+
rows.set(id, { revision: (this.base.get(id) ?? this.coldBase.get(id))?.revision ?? null, family });
|
|
4067
|
+
}
|
|
4068
|
+
return { owner: this.owner, claims, rows };
|
|
4069
|
+
}
|
|
4070
|
+
/**
|
|
4071
|
+
* A fenced write found rows that changed under this store (#779, plan step 3); the write itself
|
|
4072
|
+
* has already failed with nothing changed, and its caller sees that error.
|
|
4073
|
+
*
|
|
4074
|
+
* A conflict is UNEXPECTED when this store owned the row's family when it last read or wrote it:
|
|
4075
|
+
* a held or pending row, or one read inside a family it claimed. Another writer then broke the
|
|
4076
|
+
* claim. For each such row the original (what this store last saw), local (what it meant to
|
|
4077
|
+
* write, or that it meant to delete it) and current versions go to `run_conflicts` first. Only
|
|
4078
|
+
* once that evidence is stored does the row leave the pending set, so no retry rewrites it; this
|
|
4079
|
+
* store then refuses to write it again, and its held copy is replaced by the current row (or
|
|
4080
|
+
* dropped when deleted) and announced. That refusal lasts as long as this store: the explicit
|
|
4081
|
+
* recovery is a restart, which reads the row as it now stands, and the evidence row stays for a
|
|
4082
|
+
* person to compare. Other pending rows stay pending and the next save writes them.
|
|
4083
|
+
*
|
|
4084
|
+
* If the evidence cannot be stored, nothing changes: the rows stay pending, and the next write
|
|
4085
|
+
* meets the same conflict and tries again. A row read before its family was claimed may simply
|
|
4086
|
+
* have been finished by its previous owner in between: a stale read, not a conflict, so only the
|
|
4087
|
+
* write fails.
|
|
4088
|
+
*/
|
|
4089
|
+
handleConflicts(error, staged) {
|
|
4090
|
+
const unexpected = error.conflicts.filter((conflict) => this.held.has(conflict.id) || this.deleted.has(conflict.id) || this.coldBase.get(conflict.id)?.owned === true);
|
|
4091
|
+
if (unexpected.length === 0 || !this.db || !this.owner)
|
|
4092
|
+
return;
|
|
4093
|
+
const evidence = unexpected.map((conflict) => {
|
|
4094
|
+
const seen = this.base.get(conflict.id) ?? this.coldBase.get(conflict.id);
|
|
4095
|
+
const local = staged.has(conflict.id) ? staged.get(conflict.id) : this.deleted.has(conflict.id) ? null : this.held.get(conflict.id) ?? null;
|
|
4096
|
+
return {
|
|
4097
|
+
runId: conflict.id, reason: conflict.reason,
|
|
4098
|
+
baseRevision: seen?.revision ?? null, baseData: seen?.data ?? null,
|
|
4099
|
+
localData: local === null ? null : JSON.stringify(local), localDeleted: local === null,
|
|
4100
|
+
currentRevision: conflict.current?.revision ?? null, currentData: conflict.current?.data ?? null,
|
|
4101
|
+
};
|
|
4102
|
+
});
|
|
4103
|
+
let seqs;
|
|
4104
|
+
try {
|
|
4105
|
+
seqs = this.db.recordConflicts(this.owner.session, evidence);
|
|
4106
|
+
}
|
|
4107
|
+
catch (evidenceError) {
|
|
4108
|
+
const message = evidenceError instanceof Error ? evidenceError.message : String(evidenceError);
|
|
4109
|
+
console.error(`[cez] ${RUNS_DB_FILE}: could not record the conflicting write on ${unexpected.map((c) => c.id).join(', ')}; it stays pending: ${message}`);
|
|
4110
|
+
return;
|
|
4111
|
+
}
|
|
4112
|
+
unexpected.forEach((conflict, index) => {
|
|
4113
|
+
const family = this.familyOf(conflict.id);
|
|
4114
|
+
this.quarantined.add(conflict.id);
|
|
4115
|
+
this.dirty.delete(conflict.id);
|
|
4116
|
+
this.deleted.delete(conflict.id);
|
|
4117
|
+
this.deletedFamilies.delete(conflict.id);
|
|
4118
|
+
// The delete did not happen, so the history stays with the row.
|
|
4119
|
+
this.historyOwed.delete(conflict.id);
|
|
4120
|
+
if (conflict.reason === 'claim-lost') {
|
|
4121
|
+
this.claimed.delete(family);
|
|
4122
|
+
this.pendingClaims.delete(family);
|
|
4123
|
+
}
|
|
4124
|
+
const decoded = conflict.current ? decodeRunRow(conflict.current.data) : undefined;
|
|
4125
|
+
const current = decoded?.run;
|
|
4126
|
+
const held = this.held.get(conflict.id);
|
|
4127
|
+
if (current && conflict.current) {
|
|
4128
|
+
// The other writer's record as it stands, normalized only as a live owner's run would be.
|
|
4129
|
+
reconcileLoadedRun(current, { keepLive: true });
|
|
4130
|
+
rescopeRun(current, this.repoHandle);
|
|
4131
|
+
if (held)
|
|
4132
|
+
replaceRecord(held, current);
|
|
4133
|
+
this.base.set(conflict.id, { revision: conflict.current.revision, seq: conflict.current.seq, data: conflict.current.data, extras: decoded.extras });
|
|
4134
|
+
this.emit('run', held ?? current);
|
|
4135
|
+
}
|
|
4136
|
+
else {
|
|
4137
|
+
this.held.delete(conflict.id);
|
|
4138
|
+
this.base.delete(conflict.id);
|
|
4139
|
+
this.emit('deleted', conflict.id);
|
|
4140
|
+
}
|
|
4141
|
+
console.error(`[cez] run ${conflict.id} was changed by another writer (${conflict.reason}); this process will not write it again until cezar restarts. Evidence: ${join(this.dataDir, RUNS_DB_FILE)}, table run_conflicts, seq ${seqs[index]}.`);
|
|
4142
|
+
});
|
|
4143
|
+
this.scheduleSave();
|
|
4144
|
+
}
|
|
4145
|
+
/** Write what is pending, then (unless closing) let every run whose write settled leave memory. */
|
|
4146
|
+
saveNow(evict = true) {
|
|
4147
|
+
if (!this.db)
|
|
4148
|
+
return;
|
|
4149
|
+
// A second attempt only after a conflict excluded its rows: the unrelated rest goes out at once.
|
|
4150
|
+
for (let attempt = 0; attempt < 2 && (this.dirty.size > 0 || this.deleted.size > 0); attempt++) {
|
|
4151
|
+
const quarantined = this.quarantined.size;
|
|
4152
|
+
try {
|
|
4153
|
+
this.persist(new Map());
|
|
4154
|
+
}
|
|
4155
|
+
catch (err) {
|
|
4156
|
+
if (err instanceof RunConflictError && this.quarantined.size > quarantined)
|
|
4157
|
+
continue;
|
|
4158
|
+
// A pending save may outlive its directory; never recreate it or hide live-directory
|
|
4159
|
+
// failures. The rows stay pending (and held), so the next save or commit retries them.
|
|
4160
|
+
if (existsSync(this.dataDir)) {
|
|
4161
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4162
|
+
console.error(`[cez] failed to save ${RUNS_DB_FILE}: ${message}`);
|
|
2365
4163
|
}
|
|
4164
|
+
break;
|
|
4165
|
+
}
|
|
4166
|
+
}
|
|
4167
|
+
if (evict)
|
|
4168
|
+
this.evictSettled();
|
|
4169
|
+
}
|
|
4170
|
+
/**
|
|
4171
|
+
* Write what is pending and release the database (store disposal, process shutdown).
|
|
4172
|
+
* Idempotent. A closed store keeps answering reads for what it holds — the final save evicts
|
|
4173
|
+
* nothing — but cannot read `runs.db` any more, so a run that had already left memory reads as
|
|
4174
|
+
* absent. It saves nothing more, and a durable commit on it throws.
|
|
4175
|
+
*/
|
|
4176
|
+
close() {
|
|
4177
|
+
this.compressor.stop();
|
|
4178
|
+
this.facts.stop();
|
|
4179
|
+
this.facts.flush();
|
|
4180
|
+
if (this.saveTimer) {
|
|
4181
|
+
clearTimeout(this.saveTimer);
|
|
4182
|
+
this.saveTimer = null;
|
|
4183
|
+
}
|
|
4184
|
+
this.saveNow(false);
|
|
4185
|
+
if (this.db && this.owner) {
|
|
4186
|
+
// Every claim goes with the store. Should the release fail, the claims stay until this
|
|
4187
|
+
// process exits; from then on they are provably dead and the next opener takes them.
|
|
4188
|
+
try {
|
|
4189
|
+
this.db.releaseClaims(this.owner.session);
|
|
2366
4190
|
}
|
|
2367
|
-
|
|
2368
|
-
console.error(`[cez] failed to save runs.json: ${message}`);
|
|
4191
|
+
catch { /* see above */ }
|
|
2369
4192
|
}
|
|
4193
|
+
this.claimed.clear();
|
|
4194
|
+
this.pendingClaims.clear();
|
|
4195
|
+
if (this.owner)
|
|
4196
|
+
closeClaimSession(this.owner.session);
|
|
4197
|
+
this.db?.close();
|
|
4198
|
+
this.db = null;
|
|
2370
4199
|
}
|
|
2371
4200
|
}
|
|
2372
4201
|
//# sourceMappingURL=store.js.map
|