@humwick/server 0.130.0 → 0.130.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/docs/guides/tools-administration.md +6 -1
- package/docs/guides/tools-coordination.md +1 -0
- package/docs/guides/tools-tasks.md +8 -0
- package/docs/guides/workflows.md +58 -0
- package/docs/openapi.json +414 -1
- package/package.json +8 -17
- package/public/assets/{MessageBubble-BWpoBjl4.js → MessageBubble-BWLqMGKG.js} +2 -2
- package/public/assets/{PlanGraphPanel-C0auck3n.js → PlanGraphPanel-3TlIQpjk.js} +2 -2
- package/public/assets/{TraceView-DBnOx7AQ.js → TraceView-Wimm0Czv.js} +2 -2
- package/public/assets/{VaultDecisionCard-0kp2oNBC.js → VaultDecisionCard-Bau_vNax.js} +2 -2
- package/public/assets/{activity-yLztXEBt.js → activity-CrSurn-r.js} +2 -2
- package/public/assets/{agent-lineage-BbJDR1dW.js → agent-lineage-CknAJKcb.js} +2 -2
- package/public/assets/agent-liveness-CaNwK0Zf.js +2 -0
- package/public/assets/{agent-name-CTZBrSD9.js → agent-name-DKbiIhBo.js} +2 -2
- package/public/assets/agent-profiles-DwF6EQ4R.js +2 -0
- package/public/assets/{agent-project-badge-9TqzUUYE.js → agent-project-badge-Diz4Sjjb.js} +2 -2
- package/public/assets/{agent-tools-D4n40pn8.js → agent-tools-CRhpRQza.js} +2 -2
- package/public/assets/{agent-type-icon-D6FUarOf.js → agent-type-icon-LrUEU3lB.js} +2 -2
- package/public/assets/{agents-BYDslkJc.js → agents-B-v66C-4.js} +2 -2
- package/public/assets/{analytics-B-XSbEMC.js → analytics-BqCb-LKi.js} +2 -2
- package/public/assets/{artifact-viewer-DdWIfV6M.js → artifact-viewer-kf4pGuAR.js} +2 -2
- package/public/assets/{artifacts-oBOqks-2.js → artifacts-D32zyw4Y.js} +2 -2
- package/public/assets/{artifacts-DPN1q4pA.js → artifacts-DJevXcvg.js} +2 -2
- package/public/assets/{automation-4KcRFXEu.js → automation-BpRajx_5.js} +2 -2
- package/public/assets/{backlog-Cw9xPTCF.js → backlog-D_UjTdzX.js} +2 -2
- package/public/assets/{busy-health-CTu32MZt.js → busy-health-N3VEh4ao.js} +2 -2
- package/public/assets/{channels-BiXzg1G9.js → channels-9jopungq.js} +2 -2
- package/public/assets/{chat-Ctk2qwvA.js → chat-CSXrTB3C.js} +2 -2
- package/public/assets/{code-preview-Bm1wWqAp.js → code-preview-p1NnmqK0.js} +2 -2
- package/public/assets/{connectors-C36ax4Vw.js → connectors-CZdqlDNL.js} +2 -2
- package/public/assets/{cosmos-Jj9LK2Wf.js → cosmos-BgdsE4dG.js} +2 -2
- package/public/assets/{crate-agent-H6S7-B9T.js → crate-agent-30aXTKva.js} +2 -2
- package/public/assets/{dependency-graph-Cd1E23wa.js → dependency-graph-B_jvw2GE.js} +2 -2
- package/public/assets/{deploys-Bwfq146q.js → deploys-DoN6bAb5.js} +2 -2
- package/public/assets/{display-CfO7rTuF.js → display-BrKWFGJz.js} +2 -2
- package/public/assets/{files-DhhZI2Ih.js → files-D6XEu8Ng.js} +2 -2
- package/public/assets/fleet-C7jQWbw2.js +2 -0
- package/public/assets/{fleet-metrics-BkHP-P3R.js → fleet-metrics-BiDs3MLx.js} +2 -2
- package/public/assets/{formatted-body-Cm1RrzmN.js → formatted-body-CQBwc1fb.js} +3 -3
- package/public/assets/{formatted-body-impl-DQiOuOD1.js → formatted-body-impl-DyG0WsQS.js} +2 -2
- package/public/assets/{host-providers-dialog-CHqwXTov.js → host-providers-dialog-DKaW31dN.js} +2 -2
- package/public/assets/index-CRHep3M9.js +15 -0
- package/public/assets/index-MYAIwYTX.css +2 -0
- package/public/assets/{insights-fH1EL-qZ.js → insights-CzrtkpOE.js} +2 -2
- package/public/assets/{integrations-C5Oix6ng.js → integrations-aVsUmigt.js} +2 -2
- package/public/assets/{issue-ref-chip-ODyondIa.js → issue-ref-chip-BnSlM_95.js} +2 -2
- package/public/assets/{log-viewer-Cvfy6PW7.js → log-viewer-qe7Ee47Z.js} +2 -2
- package/public/assets/{maintenance-D8CDH5wZ.js → maintenance-NfjqFZAk.js} +2 -2
- package/public/assets/{managed-agents-egllbpij.js → managed-agents-BiH0HgaE.js} +2 -2
- package/public/assets/{markdown-preview-CYSFgVgK.js → markdown-preview-BixaT3Xr.js} +3 -3
- package/public/assets/{markdown-preview-impl-BteVIjX4.js → markdown-preview-impl-DNGlWIS3.js} +2 -2
- package/public/assets/{memory-C8WmXsyr.js → memory-DNsCl6UX.js} +2 -2
- package/public/assets/{messages-BzQdqlO6.js → messages-DopU1L1X.js} +2 -2
- package/public/assets/{orchestrators-B1HxzkNw.js → orchestrators-Cq7ckLQT.js} +2 -2
- package/public/assets/{overview-DCUnI4Z7.js → overview-C1DCTzzV.js} +2 -2
- package/public/assets/{pairs-ftCd2XcD.js → pairs-DE_m6rrE.js} +2 -2
- package/public/assets/{profile-icon-B2zH8DMH.js → profile-icon-B3rfh77F.js} +2 -2
- package/public/assets/project-profile-instructions-DQWZE0kA.js +2 -0
- package/public/assets/projects-Br2Bd2h1.js +2 -0
- package/public/assets/{prompt-settings-Bftq9czX.js → prompt-settings-BKvFsvGD.js} +2 -2
- package/public/assets/provider-display-DsuWMvVL.js +2 -0
- package/public/assets/{registry-DWCU5c8q.js → registry-Cw38_6gW.js} +2 -2
- package/public/assets/run-card-CpKqRbhg.js +2 -0
- package/public/assets/scroll-text-BKXS1ahr.js +2 -0
- package/public/assets/{secrets-C7GIUATz.js → secrets-DxI-vzSq.js} +2 -2
- package/public/assets/{service-access-settings-C3aujqTG.js → service-access-settings-D9s_VJbd.js} +2 -2
- package/public/assets/{settings-C6sZ9eBa.js → settings-CKZTVGIx.js} +2 -2
- package/public/assets/{shipping-Bu8p1gEw.js → shipping-B9YtnwtG.js} +2 -2
- package/public/assets/{stats-V9Z-GxUd.js → stats-DyUBKyWI.js} +2 -2
- package/public/assets/{status-dot-CVv0iGHL.js → status-dot-DYqlEUhJ.js} +2 -2
- package/public/assets/{store-BvtFdlGX.js → store-DcClXLH0.js} +3 -3
- package/public/assets/{tasks-CXU75HRF.js → tasks-Bh7Chr4-.js} +2 -2
- package/public/assets/{teams-BPf976zb.js → teams-bowI7qVY.js} +2 -2
- package/public/assets/{terminal-viewer-DE9BOe-8.js → terminal-viewer-BgVLHdsU.js} +3 -3
- package/public/assets/{terminal-viewer-impl-D04pjmGh.js → terminal-viewer-impl-Dz22UPy6.js} +2 -2
- package/public/assets/token-estimate-Co-Jdphh.js +2 -0
- package/public/assets/{use-agent-terminal-B1yR456R.js → use-agent-terminal-DSQQ-iY7.js} +2 -2
- package/public/assets/{use-decisions-D-S4tpDf.js → use-decisions-samMQhQf.js} +2 -2
- package/public/assets/use-project-list-59LheVP7.js +2 -0
- package/public/assets/{use-selectors-CnrAk2YP.js → use-selectors-BZa25xEB.js} +2 -2
- package/public/assets/{user-avatar-nBFIcR4M.js → user-avatar-DHxlvUvV.js} +2 -2
- package/public/assets/versioned-prompt-editor-BmED5DnJ.js +8 -0
- package/public/assets/{work-queue-TCjJxeT-.js → work-queue-CSvpuNLz.js} +2 -2
- package/public/assets/{workflow-marker-Cf3Hf2fN.js → workflow-marker-DAZcj5qt.js} +2 -2
- package/public/assets/{workflows-CXQX20Qj.js → workflows-DdmxRSyg.js} +2 -2
- package/public/assets/{workspaces-qSYzKYMA.js → workspaces-BxeKfVuF.js} +2 -2
- package/public/cosmos.html +4 -4
- package/public/index.html +31 -29
- package/runner/plugins/claude/.claude-plugin/plugin.json +1 -1
- package/runner/plugins/claude/hooks/gate-run-guard.provisioned.mjs +31 -1
- package/runner/src/adapters/claude-effort.ts +2 -4
- package/runner/src/adapters/codex-launch-config.ts +4 -5
- package/runner/src/adapters/codex.ts +1 -1
- package/runner/src/adapters/opencode-provider.ts +7 -22
- package/runner/src/adapters/opencode.ts +4 -4
- package/runner/src/agent-runner/credentials.ts +42 -12
- package/runner/src/agent-runner/inbox.ts +27 -0
- package/runner/src/agent-runner/lifecycle-milestones.ts +0 -2
- package/runner/src/bin/claude.ts +4 -0
- package/runner/src/bin/codex.ts +4 -0
- package/runner/src/bin/opencode.ts +4 -0
- package/runner/src/index.ts +5 -2
- package/runner/src/provider-setup.ts +34 -0
- package/runner/src/providers.ts +2 -1
- package/runner/src/runner-core.ts +18 -23
- package/runner/src/spawn-input.ts +2 -1
- package/runner/src/test-run-dedup.ts +59 -56
- package/scripts/cd-config.ts +24 -3
- package/scripts/gate-run-lease.ts +10 -7
- package/scripts/gate.sh +5 -1
- package/src/branch-land-notice.ts +93 -0
- package/src/branch-landed.ts +7 -76
- package/src/bus.ts +19 -20
- package/src/cd-deploy-notify.ts +30 -2
- package/src/cd-host-lag-notify.ts +7 -6
- package/src/ci/incidents.ts +2 -2
- package/src/ci/monitor.ts +5 -8
- package/src/ci/root-missing.ts +2 -7
- package/src/ci/transition-notify.ts +7 -39
- package/src/cli/cd-pauses.ts +19 -0
- package/src/cli/orchestrator.ts +1 -1
- package/src/cli/state.ts +2 -19
- package/src/cli/task-edit.ts +22 -0
- package/src/cli/task.ts +3 -1
- package/src/cli/workflow.ts +8 -2
- package/src/commands-db.ts +10 -6
- package/src/commands-write.ts +8 -0
- package/src/config-store-write.ts +13 -1
- package/src/config-store.ts +25 -14
- package/src/db/activity-async.ts +6 -1
- package/src/db/activity-write-ops.ts +13 -1
- package/src/db/activity.ts +1 -14
- package/src/db/agent-lifecycle-write-ops.ts +1 -0
- package/src/db/agent-meta.ts +2 -2
- package/src/db/agent-ownership-write-ops.ts +4 -0
- package/src/db/agent-search.ts +1 -15
- package/src/db/agents-async.ts +58 -7
- package/src/db/agents-write-ops.ts +28 -5
- package/src/db/agents.ts +81 -54
- package/src/db/artifacts-write-ops.ts +8 -3
- package/src/db/bus-outbox-write-ops.ts +1 -0
- package/src/db/channels-write-ops.ts +4 -0
- package/src/db/ci-incidents-async.ts +5 -5
- package/src/db/ci-incidents-write-ops.ts +2 -2
- package/src/db/claude-api-budget-ops.ts +5 -5
- package/src/db/claude-app-oauth.ts +4 -4
- package/src/db/codex-reset-consumes-write-ops.ts +3 -0
- package/src/db/command-row.ts +10 -2
- package/src/db/commands-list-query.ts +2 -0
- package/src/db/commands-spawn-read.ts +1 -1
- package/src/db/commands-write-ops.ts +21 -0
- package/src/db/config-store-write-ops.ts +9 -2
- package/src/db/continuation-archives-async.ts +5 -0
- package/src/db/continuation-archives-write-ops.ts +9 -0
- package/src/db/continuations.ts +2 -2
- package/src/db/db-writer-worker.ts +8 -1
- package/src/db/delivery-write-ops.ts +5 -0
- package/src/db/deploy-events-async.ts +1 -0
- package/src/db/deploy-events-read-ops.ts +2 -1
- package/src/db/deploy-events.ts +35 -4
- package/src/db/hosts-registration.ts +10 -2
- package/src/db/hosts-write-ops.ts +14 -0
- package/src/db/hosts.ts +9 -6
- package/src/db/inbox-write-ops.ts +4 -0
- package/src/db/inbox.ts +0 -14
- package/src/db/index.ts +2 -0
- package/src/db/insights-write-ops.ts +1 -0
- package/src/db/integrations-async.ts +8 -0
- package/src/db/integrations-ops.ts +15 -0
- package/src/db/integrations.ts +15 -21
- package/src/db/issue-lifecycle.ts +3 -4
- package/src/db/issue-refs-async.ts +20 -0
- package/src/db/issue-refs-write-ops.ts +29 -0
- package/src/db/issue-refs.ts +7 -8
- package/src/db/long-running-work.ts +2 -2
- package/src/db/macos-privacy.ts +1 -1
- package/src/db/main-connection-writes.ts +23 -2
- package/src/db/maintenance-job-reconcile.ts +61 -0
- package/src/db/maintenance-job-runs-async.ts +14 -0
- package/src/db/maintenance-job-write-ops.ts +9 -0
- package/src/db/mappers.ts +2 -14
- package/src/db/merge-lease.ts +1 -15
- package/src/db/message-reads-write-ops.ts +3 -0
- package/src/db/message-write-ops.ts +4 -0
- package/src/db/migrations.ts +7 -15
- package/src/db/orchestrators-async.ts +7 -3
- package/src/db/orchestrators-write-ops.ts +7 -1
- package/src/db/orchestrators.ts +21 -77
- package/src/db/pairs.ts +1 -13
- package/src/db/project-profile-instructions.ts +50 -0
- package/src/db/prompt-versions-ops.ts +1 -0
- package/src/db/prompt-versions.ts +9 -2
- package/src/db/provider-catalog-write-ops.ts +4 -4
- package/src/db/provider-config-history.ts +8 -3
- package/src/db/provider-flap-messages-write-ops.ts +1 -0
- package/src/db/provider-model-seed.ts +34 -0
- package/src/db/provider-quota-lease-write-ops.ts +1 -0
- package/src/db/provider-quota-write-ops.ts +2 -0
- package/src/db/provisioning-write-ops.ts +9 -8
- package/src/db/read-op-marshaling-fixtures.ts +5 -0
- package/src/db/read-ops.ts +4 -0
- package/src/db/routing-policy.ts +3 -4
- package/src/db/schema.ts +9 -3
- package/src/db/secrets-broker-write-ops.ts +5 -5
- package/src/db/secrets-vault-write.ts +7 -7
- package/src/db/services-write-ops.ts +3 -0
- package/src/db/spawn-delegation-state.ts +11 -4
- package/src/db/spawn-tree-cleanup.ts +3 -0
- package/src/db/stats-async.ts +4 -3
- package/src/db/task-entity.ts +23 -2
- package/src/db/task-list.ts +3 -1
- package/src/db/tasks.ts +1 -17
- package/src/db/team-spawn-delegation.ts +1 -1
- package/src/db/teams-write-ops.ts +4 -4
- package/src/db/timer-cutover-write-ops.ts +7 -0
- package/src/db/timer-ingress-write-ops.ts +9 -0
- package/src/db/timer-ingress.ts +7 -1
- package/src/db/timer-operator-write-ops.ts +14 -0
- package/src/db/timer-transfer-write-ops.ts +4 -0
- package/src/db/timer-write-ops.ts +14 -0
- package/src/db/token-db-write-ops.ts +21 -0
- package/src/db/token-standing.ts +15 -11
- package/src/db/user-settings-write-ops.ts +1 -0
- package/src/db/vault-backup.ts +2 -2
- package/src/db/vault-expiry.ts +2 -2
- package/src/db/vault-health.ts +3 -3
- package/src/db/vault-token-ops.ts +8 -8
- package/src/db/web-tool-call-ops.ts +3 -3
- package/src/db/workflow-contact.ts +1 -1
- package/src/db/workflow-task-read-ops.ts +30 -0
- package/src/db/write-ops.ts +15 -1
- package/src/db/writer-fault-seam.ts +5 -2
- package/src/db/writer-gateway.ts +21 -12
- package/src/dispatch-availability.ts +10 -6
- package/src/ephemeral/{landing-ingress-proof-worker.ts → landing-ingress-proof.test-fixture.ts} +4 -2
- package/src/host/disk-alert.ts +121 -0
- package/src/lifecycle-manager.ts +3 -2
- package/src/maintenance/cd-watcher.ts +3 -3
- package/src/maintenance/critical-job-watchdog.ts +16 -17
- package/src/maintenance/index.ts +1 -0
- package/src/maintenance/jobs.ts +3 -2
- package/src/maintenance/scheduler.ts +84 -128
- package/src/maintenance/stalled-delivery-pager.ts +6 -5
- package/src/maintenance/workspaces.ts +1 -1
- package/src/managed-policy.ts +31 -27
- package/src/mcp/index.ts +5 -2
- package/src/mcp/output-descriptions.ts +8 -8
- package/src/mcp/presentation.ts +4 -4
- package/src/mcp/project-profile-instructions.ts +32 -0
- package/src/mcp/tool-groups.ts +1 -1
- package/src/mcp/tools-spawn.ts +4 -3
- package/src/mcp-agent-action.ts +1 -2
- package/src/mcp-host-tools.ts +1 -1
- package/src/mcp-issue-ref-tools.ts +12 -11
- package/src/mcp-maintenance-tools.ts +2 -2
- package/src/mcp-profile-trust-tools.ts +4 -4
- package/src/mcp-provider-catalog.ts +2 -2
- package/src/mcp-task-tools.ts +11 -5
- package/src/mcp-team-template-tools.ts +3 -3
- package/src/mcp-workflow-tools.ts +10 -3
- package/src/notification-types.ts +2 -1
- package/src/operator-alarm.ts +26 -11
- package/src/package-pack.ts +42 -39
- package/src/project-instructions.ts +4 -0
- package/src/project-profile-instruction-text.ts +6 -0
- package/src/project-profile-instructions.ts +20 -0
- package/src/provider-catalog-store.ts +3 -49
- package/src/provider-config-store.ts +12 -11
- package/src/provider-model-seed.ts +1 -18
- package/src/provider-quota-config-store.ts +5 -4
- package/src/provider-secrets.test-fixture.ts +3 -3
- package/src/recipe-runner.ts +2 -2
- package/src/repo-config-source.ts +10 -4
- package/src/routes/agent-profiles.ts +6 -6
- package/src/routes/agents-spawn.ts +1 -1
- package/src/routes/agents.ts +6 -6
- package/src/routes/commands-listing.ts +4 -0
- package/src/routes/commands.ts +2 -2
- package/src/routes/continuation-archives.ts +2 -2
- package/src/routes/host-resource-control.ts +4 -1
- package/src/routes/index.ts +10 -1
- package/src/routes/integrations.ts +24 -50
- package/src/routes/issue-refs.ts +7 -7
- package/src/routes/orchestrator-bootstrap.ts +3 -3
- package/src/routes/orchestrator-host-registration.ts +2 -2
- package/src/routes/orchestrator-host-utilization.ts +3 -1
- package/src/routes/orchestrator-proxy.ts +2 -2
- package/src/routes/orchestrator.ts +4 -4
- package/src/routes/project-profile-instructions.ts +27 -0
- package/src/routes/project-trust.ts +2 -2
- package/src/routes/provider-config.ts +9 -8
- package/src/routes/provider-quota-config.ts +2 -2
- package/src/routes/settings.ts +5 -5
- package/src/routes/stats.ts +6 -6
- package/src/routes/task-http.ts +15 -1
- package/src/routes/task-issues.ts +11 -26
- package/src/routes/tasks.ts +22 -53
- package/src/routes/team-templates.ts +5 -5
- package/src/routes/teams-config.ts +2 -2
- package/src/routes/tokens.ts +12 -5
- package/src/routes/workflows.ts +20 -5
- package/src/routing-policy-normalize.ts +1 -2
- package/src/routing-step-defaults.ts +0 -2
- package/src/runner-token-mint-limit.ts +63 -0
- package/src/runtime-grant.ts +4 -2
- package/src/runtime-tokens.ts +50 -9
- package/src/security.ts +13 -20
- package/src/server-background.ts +1 -1
- package/src/server.ts +5 -2
- package/src/services/host-utilization.ts +23 -0
- package/src/services/idle-epics.ts +5 -6
- package/src/services/issue-lifecycle.ts +57 -61
- package/src/services/issue-refs.ts +9 -13
- package/src/services/kernel-timers-cutover-chaos-worker.ts +12 -7
- package/src/services/kernel-timers-ingress.ts +1 -1
- package/src/services/kernel-timers-request-key-migration-chaos-worker.ts +4 -1
- package/src/services/landed-ancestry.ts +19 -3
- package/src/services/memory-areas.ts +15 -51
- package/src/services/plan-ci-notice.ts +57 -0
- package/src/services/plan-ci-recovery.ts +2 -55
- package/src/services/plan-wake-digest.ts +1 -0
- package/src/services/profile-trust.ts +4 -3
- package/src/services/project-config-read.ts +70 -0
- package/src/services/project-profile-instructions.ts +54 -0
- package/src/services/provisioning-harness/budget-baseline.json +4 -4
- package/src/services/restart-spawn.ts +61 -49
- package/src/services/self-care-state.ts +2 -1
- package/src/services/self-effort.ts +5 -5
- package/src/services/send-message.ts +5 -1
- package/src/services/spawn-agent.ts +13 -2
- package/src/services/spawn-chain-inspector.ts +6 -13
- package/src/services/spawn-policy.ts +68 -0
- package/src/services/spawn-tree-cleanup.ts +2 -2
- package/src/services/task-dispatch.ts +3 -0
- package/src/services/task-state-reconcile.ts +6 -14
- package/src/services/team-templates.ts +9 -8
- package/src/services/transient-agent-reaper.ts +3 -1
- package/src/services/workflow-auto-start.ts +25 -13
- package/src/services/workflow-ci-main.ts +1 -1
- package/src/services/workflow-deferred-cleanup.ts +4 -2
- package/src/services/workflow-effect-driver.ts +2 -0
- package/src/services/workflow-effects.ts +18 -11
- package/src/services/workflow-land-notice.ts +10 -0
- package/src/services/workflow-pause-text.ts +43 -0
- package/src/services/workflow-reconcile.ts +13 -11
- package/src/services/workflow-run-view.ts +2 -0
- package/src/services/workflow-runs.ts +29 -13
- package/src/services/workflow-supervisor-state.ts +5 -0
- package/src/services/workflow-supervisor.ts +7 -3
- package/src/settings/registry.ts +1 -2
- package/src/spawn-command.ts +4 -4
- package/src/spawn-models.ts +10 -12
- package/src/spawn-placement.ts +1 -1
- package/src/spawn-targets.ts +1 -1
- package/src/startup-context-injection.ts +1 -0
- package/src/state-adoption/cd-cutover.ts +53 -7
- package/src/state-adoption/fleet-cutover.ts +35 -0
- package/src/state-adoption/fleet-plan.ts +36 -0
- package/src/state-adoption/index.ts +2 -2
- package/src/state-adoption/linux-plan.ts +2 -2
- package/src/state-adoption/linux.ts +2 -2
- package/src/state-adoption/macos-plan.ts +2 -2
- package/src/state-adoption/macos.ts +1 -1
- package/src/steward.ts +7 -6
- package/src/task-dispatch-live.ts +1 -1
- package/src/task-dispatcher.ts +1 -1
- package/src/teams-config-store.ts +4 -3
- package/src/token-db-write.ts +10 -0
- package/src/workspace-command-settle.ts +2 -1
- package/src/workspace-steward-wake.ts +2 -2
- package/node_modules/@humwick/kernel/LICENSE +0 -167
- package/node_modules/@humwick/kernel/async-entity-persistence.ts +0 -344
- package/node_modules/@humwick/kernel/async-event-log.ts +0 -107
- package/node_modules/@humwick/kernel/automations.pilot.ts +0 -147
- package/node_modules/@humwick/kernel/config.ts +0 -111
- package/node_modules/@humwick/kernel/contract.ts +0 -572
- package/node_modules/@humwick/kernel/control-authz.ts +0 -281
- package/node_modules/@humwick/kernel/control.ts +0 -645
- package/node_modules/@humwick/kernel/effects.ts +0 -493
- package/node_modules/@humwick/kernel/event-log.ts +0 -319
- package/node_modules/@humwick/kernel/http-transport.ts +0 -384
- package/node_modules/@humwick/kernel/identity-grammar.ts +0 -20
- package/node_modules/@humwick/kernel/identity.ts +0 -248
- package/node_modules/@humwick/kernel/index.ts +0 -28
- package/node_modules/@humwick/kernel/kernel-persistence.ts +0 -181
- package/node_modules/@humwick/kernel/landing-incident.ts +0 -306
- package/node_modules/@humwick/kernel/landing.ts +0 -953
- package/node_modules/@humwick/kernel/manifest-validation.ts +0 -460
- package/node_modules/@humwick/kernel/mcp-transport.ts +0 -109
- package/node_modules/@humwick/kernel/message-authority-chaos-worker.ts +0 -200
- package/node_modules/@humwick/kernel/message-authority.ts +0 -281
- package/node_modules/@humwick/kernel/module-host.ts +0 -722
- package/node_modules/@humwick/kernel/openapi-init.ts +0 -14
- package/node_modules/@humwick/kernel/package.json +0 -47
- package/node_modules/@humwick/kernel/persistence-schema.ts +0 -248
- package/node_modules/@humwick/kernel/persistence.ts +0 -510
- package/node_modules/@humwick/kernel/registry.ts +0 -60
- package/node_modules/@humwick/kernel/sqlite-kernel-persistence.ts +0 -499
- package/node_modules/@humwick/kernel/sqlite-module-host.ts +0 -87
- package/node_modules/@humwick/kernel/timers-chaos-worker.ts +0 -186
- package/node_modules/@humwick/kernel/timers.pilot.ts +0 -1411
- package/node_modules/@humwick/kernel/workflow-activation.ts +0 -25
- package/node_modules/@humwick/kernel/workflow-backstop.ts +0 -40
- package/node_modules/@humwick/kernel/workflow-cancel-tree.ts +0 -20
- package/node_modules/@humwick/kernel/workflow-chaos-worker.ts +0 -213
- package/node_modules/@humwick/kernel/workflow-composition.ts +0 -231
- package/node_modules/@humwick/kernel/workflow-container.ts +0 -72
- package/node_modules/@humwick/kernel/workflow-control.test-fixture.ts +0 -42
- package/node_modules/@humwick/kernel/workflow-control.ts +0 -178
- package/node_modules/@humwick/kernel/workflow-definition.ts +0 -507
- package/node_modules/@humwick/kernel/workflow-dispatch.ts +0 -60
- package/node_modules/@humwick/kernel/workflow-foreach.ts +0 -167
- package/node_modules/@humwick/kernel/workflow-journal-codec.ts +0 -339
- package/node_modules/@humwick/kernel/workflow-journal.ts +0 -922
- package/node_modules/@humwick/kernel/workflow-lane.ts +0 -48
- package/node_modules/@humwick/kernel/workflow-lease-owner.ts +0 -60
- package/node_modules/@humwick/kernel/workflow-recovery.test-fixture.ts +0 -19
- package/node_modules/@humwick/kernel/workflow-runtime.ts +0 -938
- package/node_modules/@humwick/kernel/workflow-script.ts +0 -11
- package/node_modules/@humwick/kernel/workflow-state.ts +0 -299
- package/node_modules/@humwick/kernel/workflow-step-commit.ts +0 -195
- package/node_modules/@humwick/kernel/workflow-transitions.ts +0 -317
- package/node_modules/@humwick/kernel/workflow-values.ts +0 -153
- package/node_modules/@humwick/kernel/workflow.pilot.ts +0 -376
- package/node_modules/@humwick/offload/LICENSE +0 -167
- package/node_modules/@humwick/offload/README.md +0 -57
- package/node_modules/@humwick/offload/dist/async-runner.d.ts +0 -18
- package/node_modules/@humwick/offload/dist/async-runner.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/async-runner.js +0 -158
- package/node_modules/@humwick/offload/dist/async-runner.js.map +0 -1
- package/node_modules/@humwick/offload/dist/envelope.d.ts +0 -25
- package/node_modules/@humwick/offload/dist/envelope.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/envelope.js +0 -80
- package/node_modules/@humwick/offload/dist/envelope.js.map +0 -1
- package/node_modules/@humwick/offload/dist/fixtures/worker-task.d.ts +0 -10
- package/node_modules/@humwick/offload/dist/fixtures/worker-task.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/fixtures/worker-task.js +0 -59
- package/node_modules/@humwick/offload/dist/fixtures/worker-task.js.map +0 -1
- package/node_modules/@humwick/offload/dist/index.d.ts +0 -5
- package/node_modules/@humwick/offload/dist/index.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/index.js +0 -5
- package/node_modules/@humwick/offload/dist/index.js.map +0 -1
- package/node_modules/@humwick/offload/dist/subprocess-runner.d.ts +0 -62
- package/node_modules/@humwick/offload/dist/subprocess-runner.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/subprocess-runner.js +0 -295
- package/node_modules/@humwick/offload/dist/subprocess-runner.js.map +0 -1
- package/node_modules/@humwick/offload/dist/types.d.ts +0 -111
- package/node_modules/@humwick/offload/dist/types.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/types.js +0 -176
- package/node_modules/@humwick/offload/dist/types.js.map +0 -1
- package/node_modules/@humwick/offload/dist/worker-pool.d.ts +0 -23
- package/node_modules/@humwick/offload/dist/worker-pool.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/worker-pool.js +0 -435
- package/node_modules/@humwick/offload/dist/worker-pool.js.map +0 -1
- package/node_modules/@humwick/offload/dist/worker-runtime.d.ts +0 -2
- package/node_modules/@humwick/offload/dist/worker-runtime.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/worker-runtime.js +0 -27
- package/node_modules/@humwick/offload/dist/worker-runtime.js.map +0 -1
- package/node_modules/@humwick/offload/dist/worker.d.ts +0 -5
- package/node_modules/@humwick/offload/dist/worker.d.ts.map +0 -1
- package/node_modules/@humwick/offload/dist/worker.js +0 -1
- package/node_modules/@humwick/offload/dist/worker.js.map +0 -1
- package/node_modules/@humwick/offload/package.json +0 -46
- package/node_modules/@humwick/offload/src/async-runner.ts +0 -170
- package/node_modules/@humwick/offload/src/envelope.ts +0 -92
- package/node_modules/@humwick/offload/src/fixtures/worker-task.ts +0 -49
- package/node_modules/@humwick/offload/src/index.ts +0 -4
- package/node_modules/@humwick/offload/src/subprocess-runner.ts +0 -334
- package/node_modules/@humwick/offload/src/types.ts +0 -235
- package/node_modules/@humwick/offload/src/worker-pool.ts +0 -463
- package/node_modules/@humwick/offload/src/worker-runtime.ts +0 -32
- package/node_modules/@humwick/offload/src/worker.ts +0 -5
- package/public/assets/agent-liveness-DyqxfB-t.js +0 -2
- package/public/assets/agent-profiles-BpI_jTxq.js +0 -2
- package/public/assets/fleet-BL4UXS_k.js +0 -2
- package/public/assets/index-B7ZerHLY.css +0 -2
- package/public/assets/index-BK0WV6hF.js +0 -15
- package/public/assets/projects-B_lmcqfO.js +0 -2
- package/public/assets/provider-display-DbLWpP3j.js +0 -2
- package/public/assets/run-card-CO_bguQf.js +0 -2
- package/public/assets/use-project-list-B-T2Bnnn.js +0 -2
- package/public/assets/versioned-prompt-editor-Diu8BIaM.js +0 -8
- package/runner/src/gate-run-lease-probe.ts +0 -38
- package/src/db/claude-catalog-reconcile.ts +0 -27
- package/src/routing-model-aliases.ts +0 -21
|
@@ -1,1411 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import { type EntityStore, type Grant, type HostCommandRun, type ModuleCtx, type StepResult, OpError, defineModule } from "./contract";
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* The timers pilot module: a Timers CRUD (`create`/`get`/`list`/`cancel`) served through the module
|
|
6
|
-
* host — default-deny authz, I/O validation, scope-tagged persistence and events, and atomic
|
|
7
|
-
* commit/rollback — plus DURABLE FIRING riding on the K7 workflow runtime (M3). Creating a timer
|
|
8
|
-
* enqueues a one-shot `fire` workflow that sleeps until `fireAt` and then, on a wakeup pass, flips
|
|
9
|
-
* the timer `scheduled → fired` and emits `timers.fired`. Firing is a `sleeping` journal row a
|
|
10
|
-
* wakeup pass claims — never a live `setTimeout` — so it is crash-durable and clock-deterministic.
|
|
11
|
-
*
|
|
12
|
-
* ── Command timers (Phase-2 P2-2, #1409) ────────────────────────────────────────────────────────
|
|
13
|
-
* A timer may carry a `command` spec. Instead of the plain `scheduled → fired` flip, its firing
|
|
14
|
-
* workflow issues a durable P3 command (`ctx.commands.issue("run-command", …)`) at the deadline and
|
|
15
|
-
* parks `waiting` on that command's terminal status (`waitFor:{kind:"command"}`). The orchestrator —
|
|
16
|
-
* NOT this process — execs the shell; the kernel only dispatches an authz-gated command and awaits
|
|
17
|
-
* it. The command's `idempotencyKey` is the timer id, so a re-driven `fire` step (a crash after the
|
|
18
|
-
* issue but before the journal memo) dedupes onto the SAME orchestrator command and never
|
|
19
|
-
* double-dispatches. The completion relay message rides the P2-3 outbox — see the `await` step.
|
|
20
|
-
*
|
|
21
|
-
* ── Relay-dispatch transactional outbox (Phase-2 P2-3, #1409) ────────────────────────────────────
|
|
22
|
-
* A timer that carries a `message` spec dispatches a Relay message on fire. The firing step does NOT
|
|
23
|
-
* call `sendMessageService` (the kernel must not import src, and a raw send would be a non-durable
|
|
24
|
-
* dual-write); instead it writes a `relaySend` OUTBOX row INSIDE its atomic step-commit. That single
|
|
25
|
-
* transaction commits the fired flip + `timers.fired` event + outbox row together or not at all, so the
|
|
26
|
-
* DECISION to send is journal-atomic exactly-once. A src-side drainer (`kernel-timers-outbox.ts`) then
|
|
27
|
-
* scans pending rows, calls `sendMessageService`, and marks each sent — at-least-once delivery deduped
|
|
28
|
-
* to effectively-once by the run-scoped `base:run:N` idempotency key `sendMessageService` already
|
|
29
|
-
* honors, identical to legacy `fireClaimedTimer`. Closes BOTH parity seams: the message-timer send and
|
|
30
|
-
* the command-timer completion send.
|
|
31
|
-
*
|
|
32
|
-
* SECURITY (R1): the command runs under the timer's FROZEN create-time identity snapshot
|
|
33
|
-
* (`command.identity`), reconstructed src-side into a cwd-constrained component token that
|
|
34
|
-
* `dispatchCommand`'s #968 gate binds the command cwd to. The kernel carries the snapshot opaquely;
|
|
35
|
-
* it never fabricates authority. See `src/services/kernel-timers-command.ts`.
|
|
36
|
-
*
|
|
37
|
-
* ── Recurring/interval + failure backoff (Phase-2 P2-5, #1409) ───────────────────────────────────
|
|
38
|
-
* A timer carrying `everyMs` is RECURRING: firing is a LOOPING workflow (arm→fire→arm) that reschedules
|
|
39
|
-
* the next deadline deterministically from durable state — `max(now+everyMs, prevFireAt+everyMs)`, no
|
|
40
|
-
* drift, no live `setTimeout` — and disables at `maxRuns`/`expiresAt` (parity with legacy
|
|
41
|
-
* `completeScheduledTimerRun`). The run number `N` a fire dispatches under lives in the firing workflow's
|
|
42
|
-
* MEMOIZED state (`state.run`, reconstructed by the journal fold), so a crash-recovery re-drive of run N
|
|
43
|
-
* recomputes the SAME N → the `base:run:N` outbox dedup holds → exactly-once send (G3). A fire that
|
|
44
|
-
* ERRORS is a timer-run-level failure: it bumps `consecutiveFailures`, reschedules at `now+retryMs`, and
|
|
45
|
-
* disables after `maxFailures` (parity with legacy `recordScheduledTimerFailure`) — distinct from the M3
|
|
46
|
-
* per-step retry.
|
|
47
|
-
*
|
|
48
|
-
* Cutover note (#1408): this proves kernel-served firing at the module's own semantic level
|
|
49
|
-
* (schedule/cancel/list/fire/command/relay-send/recurring). It is NOT yet a full functional replacement
|
|
50
|
-
* for the legacy prod timers in `src/db/scheduled-timers.ts`: the MCP write surface for recurring params
|
|
51
|
-
* remains deferred create-path parity work (P2-7) before a real prod cutover. Relay message dispatch
|
|
52
|
-
* (message + command-completion sends) is covered by the P2-3 outbox above.
|
|
53
|
-
*/
|
|
54
|
-
|
|
55
|
-
// `disabled` (P2-5) is the terminal state a recurring/one-shot timer lands in when it stops WITHOUT a
|
|
56
|
-
// clean fire — it expired before delivery, or its FIRE failed `SCHEDULER_MAX_FAILURES` times (the
|
|
57
|
-
// legacy `enabled=0 + last_error` disable). `fired` remains "completed a run" (one-shot fired, or a
|
|
58
|
-
// recurring timer reached `maxRuns` / would run past `expiresAt` AFTER a successful run). `cancelled`
|
|
59
|
-
// stays user-initiated. Only `scheduled` is active/armed — the fire step's cancel-safe guard keys on it.
|
|
60
|
-
const statusSchema = z.enum(["scheduled", "fired", "cancelled", "disabled"]);
|
|
61
|
-
|
|
62
|
-
/** Ceiling on a command-timer shell string, mirroring the legacy `cleanCommand` 8000-char cap
|
|
63
|
-
* (`scheduler-command.ts`). Bounds the durable-command payload the kernel path can persist. */
|
|
64
|
-
const MAX_COMMAND_CHARS = 8_000;
|
|
65
|
-
|
|
66
|
-
/** The grant a command timer's creator must hold — the kernel spelling of the legacy `command:write`
|
|
67
|
-
* scope `normalizeScheduleCommand` requires. `command:write` is a valid flat {@link Grant}, so the op
|
|
68
|
-
* can enforce it from `ctx.principal.grants` (the invoker-resolved principal) with no contract change. */
|
|
69
|
-
const COMMAND_WRITE_GRANT: Grant = "command:write";
|
|
70
|
-
|
|
71
|
-
/** The admin grant that lets a principal cross creators on `list`/`cancel` (mirrors the legacy
|
|
72
|
-
* `canAdministerTimers`: `actor.kind === "system" || hasScope("system:admin")`). The kernel Principal
|
|
73
|
-
* carries no actor-kind, so the MCP binding maps a system/admin token to this grant. */
|
|
74
|
-
const TIMERS_ADMIN_GRANT: Grant = "system:admin";
|
|
75
|
-
|
|
76
|
-
/** The grant the P2-3 relay-outbox drainer holds. A system-only capability distinct from the
|
|
77
|
-
* creator-facing `timers:read`/`timers:write` — the drain ops (`drainRelaySends`/`markRelaySendSent`)
|
|
78
|
-
* cross creators to see every pending send, so an ordinary timer creator must NOT be able to invoke
|
|
79
|
-
* them. The src supervisor injects an identity carrying exactly this grant; the ops are never exposed
|
|
80
|
-
* over any transport (see `expose`). */
|
|
81
|
-
const TIMERS_DRAIN_GRANT: Grant = "timers:admin";
|
|
82
|
-
|
|
83
|
-
/** The grant that lets a principal create an UNBOUNDED recurring timer (no `maxRuns`/`expiresAt`) —
|
|
84
|
-
* the kernel spelling of the legacy `schedule:persistent` scope `canCreatePersistentTimer` honors. A
|
|
85
|
-
* `system:admin` principal also qualifies (mirrors `canAdministerTimers`). The scope→grant mapping the
|
|
86
|
-
* MCP binding performs is the deferred P2-7 create-path seam (gates G1/G2); the `create` op enforces
|
|
87
|
-
* the RULE from `ctx.principal.grants` here, with no contract change. */
|
|
88
|
-
const SCHEDULE_PERSISTENT_GRANT: Grant = "schedule:admin";
|
|
89
|
-
|
|
90
|
-
/** The frozen create-time creator identity a timer's external effects fire under (R1). Mirrors the
|
|
91
|
-
* legacy `persistedAuthContext` snapshot (createdBy/kind/scopes/constraints). BOTH parity seams carry
|
|
92
|
-
* it: a command timer's `run-command` exec (P2-2) AND a message/command timer's relay-send outbox
|
|
93
|
-
* (P2-3), so the src side reconstructs the SAME `ComponentToken` the creator held — never a widened
|
|
94
|
-
* one. The kernel treats `constraints` as an OPAQUE JSON blob — it never imports the src
|
|
95
|
-
* `TokenConstraints` type — and hands it verbatim to the src translator/drainer.
|
|
96
|
-
*
|
|
97
|
-
* SECURITY (P2-4 / C2 residual #1): this snapshot is the SERVER-resolved authority — the `create` op
|
|
98
|
-
* builds it from `ctx.principal` (createdBy = the authenticated actor; createdByScopes = the invoker
|
|
99
|
-
* -resolved grants) + the token's real cwd constraint, NEVER from client input. See `create.handler`. */
|
|
100
|
-
const frozenIdentitySchema = z.object({
|
|
101
|
-
createdBy: z.string(),
|
|
102
|
-
createdByKind: z.string(),
|
|
103
|
-
createdByScopes: z.array(z.string()),
|
|
104
|
-
createdByConstraints: z.record(z.string(), z.unknown()).optional(),
|
|
105
|
-
createdAt: z.number().int(),
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
/** A command timer's persisted payload: the shell command + where/how it runs + the FROZEN creator
|
|
109
|
-
* identity. This is the ENTITY / `run-command` shape — the server-built `identity` is present. The
|
|
110
|
-
* client-facing create INPUT uses {@link commandRequestSchema} instead (no server-owned identity). */
|
|
111
|
-
const commandSpecSchema = z.object({
|
|
112
|
-
command: z.string().min(1).max(MAX_COMMAND_CHARS),
|
|
113
|
-
cwd: z.string().min(1),
|
|
114
|
-
env: z.record(z.string(), z.string()).optional(),
|
|
115
|
-
timeoutMs: z.number().int().positive(),
|
|
116
|
-
orchestratorId: z.string().min(1),
|
|
117
|
-
identity: frozenIdentitySchema,
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
/** The client-facing command spec on the `create` op input: the exec request WITHOUT any creator
|
|
121
|
-
* identity. G1 (#1409): NO identity field is accepted from op input at all — createdBy / kind / scopes
|
|
122
|
-
* / constraints are ALL server-resolved by the op from `ctx.principal` (its `actor`, `grants`, and the
|
|
123
|
-
* trusted `resolvedIdentity` the host binding attaches from the authenticated token) + the kernel clock.
|
|
124
|
-
* A client thus cannot name a creator, broaden its scopes, or supply a wider cwd constraint — even a
|
|
125
|
-
* hostile `identity`/`createdByConstraints` key on the wire is an unknown Zod strips before the handler
|
|
126
|
-
* runs. See `freezeIdentity` / `freezeCommandTimer`. */
|
|
127
|
-
const commandRequestSchema = z.object({
|
|
128
|
-
command: z.string().min(1).max(MAX_COMMAND_CHARS),
|
|
129
|
-
cwd: z.string().min(1),
|
|
130
|
-
env: z.record(z.string(), z.string()).optional(),
|
|
131
|
-
timeoutMs: z.number().int().positive(),
|
|
132
|
-
orchestratorId: z.string().min(1),
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
/** ── Relay-dispatch outbox (P2-3, #1409) ─────────────────────────────────────────────────────────
|
|
136
|
-
* The fully-resolved relay send a timer fires, persisted so a MESSAGE timer (and a command timer's
|
|
137
|
-
* completion) can dispatch a Relay message at parity with legacy `sendMessageService`. The kernel
|
|
138
|
-
* NEVER sends: it writes the send as an outbox row IN the atomic step-commit tx (journal-atomic
|
|
139
|
-
* exactly-once on the DECISION to send), and a src-side drainer performs the effect, deduped by the
|
|
140
|
-
* run-scoped `idempotencyKey` (`base:run:N`) `sendMessageService` already honors. See the `fire`/
|
|
141
|
-
* `await` steps + `src/services/kernel-timers-outbox.ts`. */
|
|
142
|
-
|
|
143
|
-
/** The client-facing message spec on `create`: WHERE/WHAT to send, plus the same server-resolved
|
|
144
|
-
* authority hints the command request carries. `from` defaults to the resolved creator; `body` is
|
|
145
|
-
* required for a pure message timer (validated in the handler) but optional for a command timer, whose
|
|
146
|
-
* completion body defaults from the command outcome. `idempotencyKey` is the OPTIONAL delivery-dedup
|
|
147
|
-
* BASE — the outbox stamps `base:run:N` from it (falling back to `scheduler:<id>`), matching legacy. */
|
|
148
|
-
const messageRequestSchema = z.object({
|
|
149
|
-
to: z.string().min(1),
|
|
150
|
-
from: z.string().min(1).optional(),
|
|
151
|
-
kind: z.string().min(1).optional(),
|
|
152
|
-
channel: z.string().min(1).optional(),
|
|
153
|
-
subject: z.string().min(1).optional(),
|
|
154
|
-
body: z.string().min(1).optional(),
|
|
155
|
-
claimable: z.boolean().optional(),
|
|
156
|
-
payload: z.record(z.string(), z.unknown()).optional(),
|
|
157
|
-
meta: z.record(z.string(), z.unknown()).optional(),
|
|
158
|
-
idempotencyKey: z.string().min(1).optional(),
|
|
159
|
-
// G1 (#1409): NO creator identity is accepted from op input — the send fires under the server-resolved
|
|
160
|
-
// frozen identity (`freezeIdentity` from `ctx.principal`), never a client-named one.
|
|
161
|
-
});
|
|
162
|
-
|
|
163
|
-
/** A timer's persisted relay-send spec: the resolved send + the FROZEN creator identity the drainer
|
|
164
|
-
* fires under (R1 — mirrors legacy `persistedAuthContext`, so the component send gate runs under the
|
|
165
|
-
* creator's real reach, not a widened one). `from` and `identity` are server-resolved. */
|
|
166
|
-
const messageSpecSchema = z.object({
|
|
167
|
-
from: z.string().min(1),
|
|
168
|
-
to: z.string().min(1),
|
|
169
|
-
kind: z.string().min(1).optional(),
|
|
170
|
-
channel: z.string().min(1).optional(),
|
|
171
|
-
subject: z.string().min(1).optional(),
|
|
172
|
-
body: z.string().min(1).optional(),
|
|
173
|
-
claimable: z.boolean().optional(),
|
|
174
|
-
payload: z.record(z.string(), z.unknown()).optional(),
|
|
175
|
-
meta: z.record(z.string(), z.unknown()).optional(),
|
|
176
|
-
idempotencyKey: z.string().min(1).optional(),
|
|
177
|
-
/** D1 (#1409): the legacy `scheduler.correlationId` (`timer:<id>`) the fired message's payload/meta
|
|
178
|
-
* `scheduler` block carries, so a kernel-fired message is byte-identical to a legacy-fired one for a
|
|
179
|
-
* consumer keying off it. Server-derived at create (`timer:<id>`); the migrate op carries the legacy
|
|
180
|
-
* row's persisted value verbatim. Absent → the outbox falls back to `timer:<timerId>` (see
|
|
181
|
-
* `messageOutboxInput`), so a correlationId is ALWAYS stamped. */
|
|
182
|
-
correlationId: z.string().min(1).optional(),
|
|
183
|
-
identity: frozenIdentitySchema,
|
|
184
|
-
});
|
|
185
|
-
|
|
186
|
-
const relaySendStatusSchema = z.enum(["pending", "sent"]);
|
|
187
|
-
|
|
188
|
-
/** ── T-0 apply fencing (#1409 hardening) ─────────────────────────────────────────────────────────
|
|
189
|
-
* The durable per-timer LAST-APPLIED-FENCE record for the ingress apply seam. The fence token is the
|
|
190
|
-
* ingress log's `seq` — globally monotone and never reused, hence STRICTLY monotone per timer. A
|
|
191
|
-
* fenced `migrate`/`cancelMirror` whose `fence` is LOWER than this row's is refused AT THE KERNEL
|
|
192
|
-
* WRITE (`fenced: true`, no entity mutation, no workflow start, no event): a live-but-lease-expired
|
|
193
|
-
* applier resuming a stale create after a newer cancel already applied can never arm the timer.
|
|
194
|
-
* A fenced cancel of an ABSENT entity still writes this row — the durable KERNEL TOMBSTONE that
|
|
195
|
-
* refuses the stale create ("a cancel has already been applied for this timerId at fence ≥ mine").
|
|
196
|
-
* Cross-process safety: the row is read-checked in the handler and (re-)written in the op's atomic
|
|
197
|
-
* commit, whose optimistic-version CAS / PK-conflict aborts a raced flush — the loser retries,
|
|
198
|
-
* re-reads the newer fence, and refuses cleanly. Equal fences (a same-intent crash re-drive)
|
|
199
|
-
* proceed idempotently. Unfenced calls (no `fence` input — the pre-activation shadow mirror and the
|
|
200
|
-
* P2-6 migration script, which run under LEGACY authority) neither check nor touch this record.
|
|
201
|
-
*
|
|
202
|
-
* SECOND-PASS HARDENING (#1409, T1175): the CAS covers ALL side effects of an apply, not just the
|
|
203
|
-
* entity — `migrate`'s firing-workflow enqueue is ENLISTED in the same atomic op commit
|
|
204
|
-
* (`enlist: "op-commit"`), so the instance seed sits in the very transaction this fence's
|
|
205
|
-
* CAS/PK-conflict aborts. A migrate that read a stale fence and loses the flush race commits
|
|
206
|
-
* NOTHING (no entity, no event, no fence, no workflow instance); a later same-id create therefore
|
|
207
|
-
* has no orphan to dedupe onto and can never fire at a rolled-back migrate's stale deadline. */
|
|
208
|
-
const applyFenceSchema = z.object({
|
|
209
|
-
/** The timer id this fence guards (the entity key). */
|
|
210
|
-
id: z.string(),
|
|
211
|
-
/** The highest ingress fence (log seq) whose apply committed for this timer. */
|
|
212
|
-
fence: z.number().int().positive(),
|
|
213
|
-
/** Which op applied at that fence — `cancel` on an absent entity IS the durable tombstone. */
|
|
214
|
-
op: z.enum(["create", "cancel"]),
|
|
215
|
-
updatedAt: z.number().int(),
|
|
216
|
-
});
|
|
217
|
-
|
|
218
|
-
/** The transactional-outbox entity: one fully-resolved, ready-to-dispatch Relay send. Written INSIDE
|
|
219
|
-
* the firing step's atomic commit (so the decision to send is journal-atomic exactly-once) and drained
|
|
220
|
-
* by the src side, which flips `status` to `sent` after `sendMessageService` returns. Delivery is
|
|
221
|
-
* at-least-once (a crash between send and mark re-drains) but deduped to effectively-once by
|
|
222
|
-
* `idempotencyKey` (`base:run:N`) — the exact key `sendMessageService` dedupes on. `identity` is the
|
|
223
|
-
* frozen creator snapshot the drainer reconstructs the send AuthContext from. */
|
|
224
|
-
const relaySendSchema = z.object({
|
|
225
|
-
id: z.string(),
|
|
226
|
-
/** The timer that produced this send — for tracing/observability + the drainer's per-row context. */
|
|
227
|
-
timerId: z.string(),
|
|
228
|
-
/** The run-scoped delivery-dedup key `sendMessageService` honors: a re-drain re-presents the SAME
|
|
229
|
-
* key, so a crash between the send and the `sent` mark never doubles the message. */
|
|
230
|
-
idempotencyKey: z.string(),
|
|
231
|
-
status: relaySendStatusSchema.default("pending"),
|
|
232
|
-
from: z.string(),
|
|
233
|
-
to: z.string(),
|
|
234
|
-
kind: z.string().optional(),
|
|
235
|
-
channel: z.string().optional(),
|
|
236
|
-
subject: z.string().optional(),
|
|
237
|
-
body: z.string(),
|
|
238
|
-
claimable: z.boolean().optional(),
|
|
239
|
-
payload: z.record(z.string(), z.unknown()).optional(),
|
|
240
|
-
meta: z.record(z.string(), z.unknown()).optional(),
|
|
241
|
-
/** The frozen creator identity the drainer fires the send under (R1). */
|
|
242
|
-
identity: frozenIdentitySchema,
|
|
243
|
-
/** Set by the drainer once the send lands — observability only; dedup is by `idempotencyKey`. */
|
|
244
|
-
sentMessageId: z.number().int().optional(),
|
|
245
|
-
createdAt: z.number().int(),
|
|
246
|
-
});
|
|
247
|
-
|
|
248
|
-
const timerSchema = z.object({
|
|
249
|
-
id: z.string(),
|
|
250
|
-
label: z.string(),
|
|
251
|
-
/** The actor id (from `ctx.principal.actor`) that created the timer — the per-creator ownership key
|
|
252
|
-
* `list`/`cancel`/`get` filter on for a non-admin principal (C2). Server-stamped; never client input. */
|
|
253
|
-
createdBy: z.string(),
|
|
254
|
-
/** Epoch milliseconds at which the timer is due to fire. For a recurring timer this is the CURRENT
|
|
255
|
-
* run's deadline; the fire step advances it deterministically for each subsequent run. */
|
|
256
|
-
fireAt: z.number().int(),
|
|
257
|
-
status: statusSchema.default("scheduled"),
|
|
258
|
-
/** ── Recurring/interval (P2-5) ─────────────────────────────────────────────────────────────────
|
|
259
|
-
* Present iff this is a RECURRING timer: the fixed interval between runs. Firing loops arm→fire→arm
|
|
260
|
-
* (a looping workflow), rescheduling `fireAt` to `max(now+everyMs, prevFireAt+everyMs)` — computed
|
|
261
|
-
* from durable state, never the wall clock, so it cannot drift. Command timers may NOT be recurring
|
|
262
|
-
* (legacy parity); enforced at `create`. */
|
|
263
|
-
everyMs: z.number().int().positive().optional(),
|
|
264
|
-
/** A recurring timer disables (terminal `fired`) once it has completed this many runs. */
|
|
265
|
-
maxRuns: z.number().int().positive().optional(),
|
|
266
|
-
/** Epoch ms after which the timer stops: it disables before firing a run past this, and a fire whose
|
|
267
|
-
* deadline is already past `expiresAt` disables WITHOUT firing (legacy `fireClaimedTimer` expiry). */
|
|
268
|
-
expiresAt: z.number().int().optional(),
|
|
269
|
-
/** Runs COMPLETED so far (bumped on each successful fire) — the observability counter, parity with
|
|
270
|
-
* legacy `run_count`. NOTE: the durable, re-drive-stable run NUMBER a fire dispatches under lives in
|
|
271
|
-
* the firing workflow's memoized `state.run` (the journal fold), NOT here — this is a projection. */
|
|
272
|
-
runCount: z.number().int().nonnegative().default(0),
|
|
273
|
-
/** Consecutive FIRE failures (parity with legacy `consecutive_failures`): bumped when a fire errors,
|
|
274
|
-
* reset to 0 on a successful run. The timer disables after `maxFailures` consecutive failures. */
|
|
275
|
-
consecutiveFailures: z.number().int().nonnegative().default(0),
|
|
276
|
-
/** The last fire error (parity with legacy `last_error`); cleared on a successful run. */
|
|
277
|
-
lastError: z.string().optional(),
|
|
278
|
-
/** Present iff this is a command timer: firing dispatches this command and awaits it (see above). */
|
|
279
|
-
command: commandSpecSchema.optional(),
|
|
280
|
-
/** Present iff the timer dispatches a Relay message on fire (P2-3): the resolved send + frozen
|
|
281
|
-
* creator identity. A MESSAGE timer carries it as its whole effect; a COMMAND timer may carry it as
|
|
282
|
-
* the completion send composed once the command settles. Absent → firing is a pure status flip
|
|
283
|
-
* (the pre-P2-3 pilot behavior, preserved). */
|
|
284
|
-
message: messageSpecSchema.optional(),
|
|
285
|
-
});
|
|
286
|
-
|
|
287
|
-
const configSchema = z.object({
|
|
288
|
-
/** Per-creator active (scheduled) timer quota — the kernel spelling of legacy
|
|
289
|
-
* `SCHEDULER_MAX_ACTIVE_TIMERS` (default 100). env-backed, so it must coerce from a string. */
|
|
290
|
-
maxTimers: z.coerce.number().int().positive().default(100),
|
|
291
|
-
/** Max lead time a timer may be scheduled ahead — the kernel spelling of the legacy command
|
|
292
|
-
* `afterMs ≤ 24h` ceiling, applied to EVERY timer so the kernel path can't evade it. */
|
|
293
|
-
maxFireAheadMs: z.coerce.number().int().positive().default(24 * 60 * 60 * 1000),
|
|
294
|
-
/** Minimum interval a RECURRING timer may specify — the kernel spelling of legacy
|
|
295
|
-
* `SCHEDULER_MIN_RECURRING_INTERVAL_MS` (default 60s). Enforced at `create` (P2-5 SCOPE 4). */
|
|
296
|
-
minRecurringIntervalMs: z.coerce.number().int().positive().default(60 * 1000),
|
|
297
|
-
/** Backoff before a FAILED fire's next attempt — the kernel spelling of legacy `SCHEDULER_RETRY_MS`
|
|
298
|
-
* (default 60s). A fire error reschedules the SAME run at `now + retryMs` (P2-5 SCOPE 3). */
|
|
299
|
-
retryMs: z.coerce.number().int().positive().default(60 * 1000),
|
|
300
|
-
/** Consecutive fire failures that disable a timer — the kernel spelling of legacy
|
|
301
|
-
* `SCHEDULER_MAX_FAILURES` (default 3) (P2-5 SCOPE 3). */
|
|
302
|
-
maxFailures: z.coerce.number().int().positive().default(3),
|
|
303
|
-
/** Label applied to a created timer when the caller supplies none. */
|
|
304
|
-
defaultLabel: z.string().min(1).default("timer"),
|
|
305
|
-
});
|
|
306
|
-
|
|
307
|
-
export type Timer = z.output<typeof timerSchema>;
|
|
308
|
-
export type TimerCommandSpec = z.output<typeof commandSpecSchema>;
|
|
309
|
-
export type TimerMessageSpec = z.output<typeof messageSpecSchema>;
|
|
310
|
-
export type RelaySend = z.output<typeof relaySendSchema>;
|
|
311
|
-
export type TimerApplyFence = z.output<typeof applyFenceSchema>;
|
|
312
|
-
export type TimersConfig = z.output<typeof configSchema>;
|
|
313
|
-
|
|
314
|
-
/** Does the opaque constraint blob pin the cwd(s) a command may run in? Mirrors the bridge's
|
|
315
|
-
* `hasCwdConstraint` / `dispatch-command`'s #968 precondition: an UNCONSTRAINED creator identity would
|
|
316
|
-
* clear the fire-time authz WITHOUT the cwd gate ever running, so a command timer must carry one at
|
|
317
|
-
* create. Kept structural (no src `TokenConstraints` import) — checks the two known cwd axes. */
|
|
318
|
-
function hasCwdConstraint(constraints: Record<string, unknown> | undefined): boolean {
|
|
319
|
-
if (!constraints) return false;
|
|
320
|
-
const cwd = constraints.cwd;
|
|
321
|
-
const prefixes = constraints.cwdPrefixes;
|
|
322
|
-
return (typeof cwd === "string" && cwd.length > 0) || (Array.isArray(prefixes) && prefixes.length > 0);
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
/** A principal may cross creators (see/cancel any timer) iff it holds the admin grant (C2). Mirrors
|
|
326
|
-
* legacy `canAdministerTimers`; the MCP binding maps a `system` actor / `system:admin` scope onto it. */
|
|
327
|
-
function isTimersAdmin(ctx: ModuleCtx): boolean {
|
|
328
|
-
return ctx.principal.grants.has(TIMERS_ADMIN_GRANT);
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
/** The per-creator ownership filter a non-admin op applies from the authenticated actor (C2). An admin
|
|
332
|
-
* returns `undefined` (no restriction). This is the op-level `where: {createdBy}` the design mandates. */
|
|
333
|
-
function creatorFilter(ctx: ModuleCtx): { createdBy: string } | undefined {
|
|
334
|
-
return isTimersAdmin(ctx) ? undefined : { createdBy: ctx.principal.actor };
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
/** May this principal create an UNBOUNDED recurring timer (no `maxRuns`/`expiresAt`)? Mirrors legacy
|
|
338
|
-
* `canCreatePersistentTimer`: an admin, or a holder of the persistent grant (P2-5 SCOPE 4). */
|
|
339
|
-
function canCreatePersistentTimer(ctx: ModuleCtx): boolean {
|
|
340
|
-
return isTimersAdmin(ctx) || ctx.principal.grants.has(SCHEDULE_PERSISTENT_GRANT);
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
/** Human-readable message from any thrown value (inlined; the kernel never imports the src SDK). */
|
|
344
|
-
function errMessage(error: unknown): string {
|
|
345
|
-
return error instanceof Error ? error.message : String(error);
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
/**
|
|
349
|
-
* Server-resolve + FREEZE a creator identity from the authenticated principal (G1/G2, #1409). Every
|
|
350
|
-
* field is server-sourced — NONE from client op input:
|
|
351
|
-
* • `createdBy` = `ctx.principal.actor` (the authenticated actor);
|
|
352
|
-
* • `createdAt` = the kernel clock;
|
|
353
|
-
* • `createdByKind` / `createdByScopes` / `createdByConstraints` = the trusted
|
|
354
|
-
* `ctx.principal.resolvedIdentity` the host binding attached from the authenticated TOKEN.
|
|
355
|
-
* So a client can neither name a different creator, broaden its scopes, nor widen its cwd constraint
|
|
356
|
-
* (G1 — a hostile `createdByConstraints:["/"] ` on the wire never reaches here). The frozen snapshot
|
|
357
|
-
* reconstructs the SAME create-time authority legacy `persistedAuthContext` did — shared by both parity
|
|
358
|
-
* seams (command exec P2-2 and relay send P2-3).
|
|
359
|
-
*
|
|
360
|
-
* G2 (#1409): `createdByScopes` is the creator's REAL transport scopes (`resolvedIdentity.scopes`,
|
|
361
|
-
* e.g. `message:send`) — NOT the flat kernel grants, which the grammar cannot spell `message:send` in.
|
|
362
|
-
* That is what lets the P2-3 outbox drainer's `sendMessageService` authorize under the creator's
|
|
363
|
-
* message-send scope instead of failing closed at fire time. When no `resolvedIdentity` rides the
|
|
364
|
-
* principal (a system/test caller that mints no durable send), it falls back to the principal grants.
|
|
365
|
-
*/
|
|
366
|
-
function freezeIdentity(ctx: ModuleCtx, createdBy: string, now: number): z.output<typeof frozenIdentitySchema> {
|
|
367
|
-
const resolved = ctx.principal.resolvedIdentity;
|
|
368
|
-
const constraints = resolved?.constraints;
|
|
369
|
-
return {
|
|
370
|
-
createdBy,
|
|
371
|
-
createdByKind: resolved?.kind ?? "component",
|
|
372
|
-
createdByScopes: resolved?.scopes ? [...resolved.scopes] : ([...ctx.principal.grants] as string[]),
|
|
373
|
-
...(constraints ? { createdByConstraints: { ...constraints } } : {}),
|
|
374
|
-
createdAt: now,
|
|
375
|
-
};
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
/**
|
|
379
|
-
* Server-resolve + FREEZE a command timer's exec spec + creator identity. Preconditions mirror the
|
|
380
|
-
* legacy `normalizeScheduleCommand` + `assertScheduleCommandAuthorized`:
|
|
381
|
-
* • the caller MUST hold the `command:write` grant (else no command timer at all), and
|
|
382
|
-
* • the frozen identity MUST carry a cwd constraint (else the fire-time #968 gate would not fire).
|
|
383
|
-
* Throws a typed {@link OpError} (fail-closed) on either.
|
|
384
|
-
*/
|
|
385
|
-
function freezeCommandTimer(
|
|
386
|
-
ctx: ModuleCtx,
|
|
387
|
-
request: z.output<typeof commandRequestSchema>,
|
|
388
|
-
createdBy: string,
|
|
389
|
-
now: number,
|
|
390
|
-
): z.output<typeof commandSpecSchema> {
|
|
391
|
-
if (!ctx.principal.grants.has(COMMAND_WRITE_GRANT)) {
|
|
392
|
-
throw new OpError("unauthorized", 403, "command timers require the 'command:write' grant");
|
|
393
|
-
}
|
|
394
|
-
// The cwd constraint is read from the SERVER-resolved identity (`ctx.principal.resolvedIdentity`),
|
|
395
|
-
// never from client input (G1) — so an unconstrained creator cannot create a command timer, and a
|
|
396
|
-
// client cannot fabricate a wide constraint to clear this gate.
|
|
397
|
-
if (!hasCwdConstraint(ctx.principal.resolvedIdentity?.constraints)) {
|
|
398
|
-
throw new OpError("unauthorized", 403, "command timers require a cwd-constrained creator identity");
|
|
399
|
-
}
|
|
400
|
-
return {
|
|
401
|
-
command: request.command,
|
|
402
|
-
cwd: request.cwd,
|
|
403
|
-
...(request.env ? { env: request.env } : {}),
|
|
404
|
-
timeoutMs: request.timeoutMs,
|
|
405
|
-
orchestratorId: request.orchestratorId,
|
|
406
|
-
identity: freezeIdentity(ctx, createdBy, now),
|
|
407
|
-
};
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
/**
|
|
411
|
-
* Server-resolve + FREEZE a timer's relay-send spec (P2-3). `from` resolves server-side (the wire
|
|
412
|
-
* `from`, else the resolved creator) and `identity` is frozen from the authenticated principal, so the
|
|
413
|
-
* drainer fires the send under the creator's REAL reach — the component send gate runs exactly as it
|
|
414
|
-
* did for legacy `persistedAuthContext`. A message timer (no command) additionally REQUIRES a body; a
|
|
415
|
-
* command timer's completion body may be absent (it defaults from the command outcome at settle).
|
|
416
|
-
*/
|
|
417
|
-
function freezeMessageTimer(
|
|
418
|
-
ctx: ModuleCtx,
|
|
419
|
-
request: z.output<typeof messageRequestSchema>,
|
|
420
|
-
createdBy: string,
|
|
421
|
-
now: number,
|
|
422
|
-
requireBody: boolean,
|
|
423
|
-
correlationId: string,
|
|
424
|
-
): z.output<typeof messageSpecSchema> {
|
|
425
|
-
if (requireBody && (request.body === undefined || request.body.length === 0)) {
|
|
426
|
-
throw new OpError("invalid_input", 400, "a message timer requires message.body");
|
|
427
|
-
}
|
|
428
|
-
return {
|
|
429
|
-
from: request.from ?? createdBy,
|
|
430
|
-
to: request.to,
|
|
431
|
-
...(request.kind ? { kind: request.kind } : {}),
|
|
432
|
-
...(request.channel ? { channel: request.channel } : {}),
|
|
433
|
-
...(request.subject ? { subject: request.subject } : {}),
|
|
434
|
-
...(request.body ? { body: request.body } : {}),
|
|
435
|
-
...(request.claimable !== undefined ? { claimable: request.claimable } : {}),
|
|
436
|
-
...(request.payload ? { payload: request.payload } : {}),
|
|
437
|
-
...(request.meta ? { meta: request.meta } : {}),
|
|
438
|
-
...(request.idempotencyKey ? { idempotencyKey: request.idempotencyKey } : {}),
|
|
439
|
-
// D1 (#1409): stamp legacy's `timer:<id>` correlation id so the fired message is byte-identical.
|
|
440
|
-
correlationId,
|
|
441
|
-
identity: freezeIdentity(ctx, createdBy, now),
|
|
442
|
-
};
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
/** ── Recurring reschedule + failure backoff math (P2-5) — PURE, mirroring legacy exactly ───────────
|
|
446
|
-
* These derive the NEXT deadline from DURABLE state (the timer entity + the workflow's memoized run
|
|
447
|
-
* number), never the wall clock, so a re-drive recomputes an identical result. Exported for direct
|
|
448
|
-
* unit tests (parity with `completeScheduledTimerRun` / `recordScheduledTimerFailure`). */
|
|
449
|
-
|
|
450
|
-
/** The next recurring run's absolute deadline: `max(now+everyMs, prevFireAt+everyMs)`. Mirrors legacy
|
|
451
|
-
* `completeScheduledTimerRun`'s `candidateNextRunAt` — a late wakeup schedules the next run `everyMs`
|
|
452
|
-
* after NOW (self-correcting), never in the past, and an on-time run stays perfectly on the grid (no
|
|
453
|
-
* drift, since it derives from `prevFireAt`, not the actual — possibly late — wakeup time). */
|
|
454
|
-
export function nextRecurringFireAt(everyMs: number, now: number, prevFireAt: number): number {
|
|
455
|
-
return Math.max(now + everyMs, prevFireAt + everyMs);
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
/** Whether a recurring timer keeps looping after completing run `runsCompleted`, and its next deadline.
|
|
459
|
-
* Mirrors legacy `completeScheduledTimerRun`'s disable logic: a non-recurring timer is always terminal;
|
|
460
|
-
* a recurring one disables at `maxRuns` or when its next deadline would fall past `expiresAt`. */
|
|
461
|
-
export type RecurringContinuation =
|
|
462
|
-
| { readonly kind: "continue"; readonly nextFireAt: number; readonly nextRun: number }
|
|
463
|
-
| { readonly kind: "terminal" };
|
|
464
|
-
|
|
465
|
-
export function recurringContinuation(input: {
|
|
466
|
-
everyMs: number | undefined;
|
|
467
|
-
now: number;
|
|
468
|
-
prevFireAt: number;
|
|
469
|
-
runsCompleted: number;
|
|
470
|
-
maxRuns: number | undefined;
|
|
471
|
-
expiresAt: number | undefined;
|
|
472
|
-
}): RecurringContinuation {
|
|
473
|
-
if (input.everyMs === undefined) return { kind: "terminal" };
|
|
474
|
-
const nextFireAt = nextRecurringFireAt(input.everyMs, input.now, input.prevFireAt);
|
|
475
|
-
const reachedMaxRuns = input.maxRuns !== undefined && input.runsCompleted >= input.maxRuns;
|
|
476
|
-
const expiredBeforeNext = input.expiresAt !== undefined && nextFireAt > input.expiresAt;
|
|
477
|
-
if (reachedMaxRuns || expiredBeforeNext) return { kind: "terminal" };
|
|
478
|
-
return { kind: "continue", nextFireAt, nextRun: input.runsCompleted + 1 };
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
/** The outcome of a FAILED fire: retry after `retryMs` (same run number), or disable once
|
|
482
|
-
* `consecutiveFailures` reaches `maxFailures`. Mirrors legacy `recordScheduledTimerFailure`. */
|
|
483
|
-
export type FailureOutcome =
|
|
484
|
-
| { readonly kind: "retry"; readonly nextFireAt: number; readonly failures: number }
|
|
485
|
-
| { readonly kind: "disable"; readonly failures: number };
|
|
486
|
-
|
|
487
|
-
export function failureBackoff(input: {
|
|
488
|
-
now: number;
|
|
489
|
-
consecutiveFailures: number;
|
|
490
|
-
retryMs: number;
|
|
491
|
-
maxFailures: number;
|
|
492
|
-
}): FailureOutcome {
|
|
493
|
-
const failures = input.consecutiveFailures + 1;
|
|
494
|
-
if (failures >= input.maxFailures) return { kind: "disable", failures };
|
|
495
|
-
return { kind: "retry", nextFireAt: input.now + input.retryMs, failures };
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
/** The run-scoped delivery-dedup key `sendMessageService` honors (`base:run:N`). Mirrors legacy
|
|
499
|
-
* `timerRunIdempotencyKey`: base is the caller's `idempotencyKey`, else `scheduler:<timerId>`. */
|
|
500
|
-
function relaySendKey(message: z.output<typeof messageSpecSchema>, timerId: string, run: number): string {
|
|
501
|
-
const base = message.idempotencyKey ?? `scheduler:${timerId}`;
|
|
502
|
-
return `${base}:run:${run}`;
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
/** Stamp the legacy `scheduler` correlation block onto a send's payload/meta so a drained message is
|
|
506
|
-
* observably a timer fire (parity with `fireClaimedTimer`). Non-mutating. */
|
|
507
|
-
function withScheduler(
|
|
508
|
-
base: Record<string, unknown> | undefined,
|
|
509
|
-
scheduler: Record<string, unknown>,
|
|
510
|
-
): Record<string, unknown> {
|
|
511
|
-
return { ...(base ?? {}), scheduler };
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
/** A durable command result the completion send summarizes (the `run-command` decl's result shape). */
|
|
515
|
-
type CommandRunResult = { exitCode: number | null; timedOut: boolean; stdoutTail: string; stderrTail: string };
|
|
516
|
-
|
|
517
|
-
/** A human completion line for a command timer whose configured `message.body` was omitted. Mirrors
|
|
518
|
-
* the shape of legacy `commandSummary` (status + exit code) over the kernel command result. */
|
|
519
|
-
function commandCompletionBody(result: CommandRunResult | undefined): string {
|
|
520
|
-
if (!result) return "Background command completed";
|
|
521
|
-
const label = result.timedOut ? "timed out" : result.exitCode === 0 ? "succeeded" : "failed";
|
|
522
|
-
const code = result.exitCode === null || result.exitCode === undefined ? "" : ` (exit ${result.exitCode})`;
|
|
523
|
-
return `Background command ${label}${code}`;
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
/** Build the fully-resolved outbox row for a MESSAGE timer's send. `run` is the DURABLE, re-drive-stable
|
|
527
|
-
* run number (`state.run` from the firing workflow's journal fold — 1, 2, 3, … for a recurring timer);
|
|
528
|
-
* the `scheduler` stamp + `base:run:N` key mirror legacy `fireClaimedTimer`. */
|
|
529
|
-
function messageOutboxInput(
|
|
530
|
-
ctx: ModuleCtx,
|
|
531
|
-
timer: Timer,
|
|
532
|
-
message: z.output<typeof messageSpecSchema>,
|
|
533
|
-
run: number,
|
|
534
|
-
): z.input<typeof relaySendSchema> {
|
|
535
|
-
// D1 (#1409): stamp `correlationId` (`timer:<id>`) alongside `timerId`/`run` so the delivered
|
|
536
|
-
// message's `scheduler` block is byte-identical to legacy `fireClaimedTimer`. Always present: the
|
|
537
|
-
// frozen `message.correlationId` when set (native create / migrate), else derived from the timer id.
|
|
538
|
-
const scheduler = { timerId: timer.id, correlationId: message.correlationId ?? `timer:${timer.id}`, run };
|
|
539
|
-
return {
|
|
540
|
-
id: ctx.rng.uuid(),
|
|
541
|
-
timerId: timer.id,
|
|
542
|
-
idempotencyKey: relaySendKey(message, timer.id, run),
|
|
543
|
-
status: "pending",
|
|
544
|
-
from: message.from,
|
|
545
|
-
to: message.to,
|
|
546
|
-
...(message.kind ? { kind: message.kind } : {}),
|
|
547
|
-
...(message.channel ? { channel: message.channel } : {}),
|
|
548
|
-
...(message.subject ? { subject: message.subject } : {}),
|
|
549
|
-
body: message.body ?? "",
|
|
550
|
-
...(message.claimable !== undefined ? { claimable: message.claimable } : {}),
|
|
551
|
-
payload: withScheduler(message.payload, scheduler),
|
|
552
|
-
meta: withScheduler(message.meta, scheduler),
|
|
553
|
-
identity: message.identity,
|
|
554
|
-
createdAt: ctx.clock.now(),
|
|
555
|
-
};
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
/** Project a pure MESSAGE send into the shared immutable fire/outbox payload. */
|
|
559
|
-
function authoritySendPayload(timer: Timer, message: z.output<typeof messageSpecSchema>, run: number) {
|
|
560
|
-
const scheduler = { timerId: timer.id, correlationId: message.correlationId ?? `timer:${timer.id}`, run };
|
|
561
|
-
const idempotencyKey = relaySendKey(message, timer.id, run);
|
|
562
|
-
return {
|
|
563
|
-
idempotencyKey,
|
|
564
|
-
sendPayload: {
|
|
565
|
-
input: {
|
|
566
|
-
from: message.from,
|
|
567
|
-
to: message.to,
|
|
568
|
-
...(message.kind ? { kind: message.kind } : {}),
|
|
569
|
-
...(message.channel ? { channel: message.channel } : {}),
|
|
570
|
-
...(message.subject ? { subject: message.subject } : {}),
|
|
571
|
-
body: message.body ?? "",
|
|
572
|
-
...(message.claimable !== undefined ? { claimable: message.claimable } : {}),
|
|
573
|
-
payload: withScheduler(message.payload, scheduler),
|
|
574
|
-
meta: withScheduler(message.meta, scheduler),
|
|
575
|
-
idempotencyKey,
|
|
576
|
-
},
|
|
577
|
-
identity: { ...message.identity, deliveryFrom: message.from },
|
|
578
|
-
},
|
|
579
|
-
};
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
/** Build the fully-resolved outbox row for a COMMAND timer's completion send. The body defaults from
|
|
583
|
-
* the command outcome when the timer configured none; the `scheduler`/`commandResult` blocks carry the
|
|
584
|
-
* result so the message is observably a settled background command (parity with legacy
|
|
585
|
-
* `fireScheduledCommandWake`). */
|
|
586
|
-
function commandCompletionOutboxInput(
|
|
587
|
-
ctx: ModuleCtx,
|
|
588
|
-
timer: Timer,
|
|
589
|
-
message: z.output<typeof messageSpecSchema>,
|
|
590
|
-
run: number,
|
|
591
|
-
commandId: string | null,
|
|
592
|
-
result: CommandRunResult | undefined,
|
|
593
|
-
): z.input<typeof relaySendSchema> {
|
|
594
|
-
const scheduler = { timerId: timer.id, correlationId: message.correlationId ?? `timer:${timer.id}`, run, ...(commandId ? { commandId } : {}) };
|
|
595
|
-
return {
|
|
596
|
-
id: ctx.rng.uuid(),
|
|
597
|
-
timerId: timer.id,
|
|
598
|
-
idempotencyKey: relaySendKey(message, timer.id, run),
|
|
599
|
-
status: "pending",
|
|
600
|
-
from: message.from,
|
|
601
|
-
to: message.to,
|
|
602
|
-
...(message.kind ? { kind: message.kind } : {}),
|
|
603
|
-
...(message.channel ? { channel: message.channel } : {}),
|
|
604
|
-
...(message.subject ? { subject: message.subject } : {}),
|
|
605
|
-
body: message.body ?? commandCompletionBody(result),
|
|
606
|
-
...(message.claimable !== undefined ? { claimable: message.claimable } : {}),
|
|
607
|
-
payload: withScheduler(message.payload, { ...scheduler, command: result ?? null }),
|
|
608
|
-
meta: withScheduler(message.meta, scheduler),
|
|
609
|
-
identity: message.identity,
|
|
610
|
-
createdAt: ctx.clock.now(),
|
|
611
|
-
};
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
/** The firing workflow state. `commandId` is null until a command timer's `fire` step has issued its
|
|
615
|
-
* P3 command; the `await` step reads it back to settle firing.
|
|
616
|
-
*
|
|
617
|
-
* `run` is the CRUX of P2-5's exactly-once (G3): the run number the CURRENT iteration fires under. It is
|
|
618
|
-
* carried in the workflow state — which is memoized in the durable journal and reconstructed by the
|
|
619
|
-
* fold — so a crash-recovery RE-DRIVE of a given run recomputes the SAME `run`, hence the SAME
|
|
620
|
-
* `base:run:N` outbox dedup key, hence NO double-send. It is NOT a live counter: the `fire` step reads
|
|
621
|
-
* `run` from folded state to stamp the send, and only the COMMITTED transition to the next `arm`
|
|
622
|
-
* advances it to `run+1` for the next iteration. A one-shot timer simply never advances past run 1. */
|
|
623
|
-
type FireState = {
|
|
624
|
-
id: string;
|
|
625
|
-
/** F is null for native/shadow workflows; authority generations carry exact F/G/R. */
|
|
626
|
-
fence: number | null;
|
|
627
|
-
generation: number;
|
|
628
|
-
fireAt: number;
|
|
629
|
-
commandId: string | null;
|
|
630
|
-
run: number;
|
|
631
|
-
};
|
|
632
|
-
|
|
633
|
-
/** The manifest handler receives the un-parameterized `ModuleCtx` (the manifest type is not yet
|
|
634
|
-
* available at its own declaration site), so a module narrows its own entities/config here. */
|
|
635
|
-
function timerStore(ctx: ModuleCtx): EntityStore<{ schema: typeof timerSchema }> {
|
|
636
|
-
return (ctx.entities as unknown as { readonly timer: EntityStore<{ schema: typeof timerSchema }> }).timer;
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
/** The relay-send outbox store (P2-3). Written by the firing steps in-tx; read/marked by the src drainer. */
|
|
640
|
-
function relaySendStore(ctx: ModuleCtx): EntityStore<{ schema: typeof relaySendSchema }> {
|
|
641
|
-
return (ctx.entities as unknown as { readonly relaySend: EntityStore<{ schema: typeof relaySendSchema }> }).relaySend;
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
/** The per-timer apply-fence store (T-0 hardening). Read-checked + upserted by `migrate`/`cancelMirror`. */
|
|
645
|
-
function applyFenceStore(ctx: ModuleCtx): EntityStore<{ schema: typeof applyFenceSchema }> {
|
|
646
|
-
return (ctx.entities as unknown as { readonly applyFence: EntityStore<{ schema: typeof applyFenceSchema }> }).applyFence;
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
/** Raise the durable fence to `fence` for `op` — a no-op at equality (same-intent re-drive). Runs in
|
|
650
|
-
* the op's atomic commit: a concurrent raced write aborts the WHOLE op via the flush CAS/PK conflict,
|
|
651
|
-
* so the fence can never regress and a stale write can never slip past a newer one. */
|
|
652
|
-
async function raiseApplyFence(
|
|
653
|
-
ctx: ModuleCtx,
|
|
654
|
-
gate: TimerApplyFence | undefined,
|
|
655
|
-
id: string,
|
|
656
|
-
fence: number,
|
|
657
|
-
op: "create" | "cancel",
|
|
658
|
-
): Promise<void> {
|
|
659
|
-
const fences = applyFenceStore(ctx);
|
|
660
|
-
if (!gate) await fences.create({ id, fence, op, updatedAt: ctx.clock.now() });
|
|
661
|
-
else if (fence > gate.fence) await fences.update(id, { fence, op, updatedAt: ctx.clock.now() });
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
function timersConfig(ctx: ModuleCtx): TimersConfig {
|
|
665
|
-
return ctx.config as unknown as TimersConfig;
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
/**
|
|
669
|
-
* Handle a FAILED fire at the TIMER-RUN level (SCOPE 3) — distinct from the M3 per-step retry. Bumps
|
|
670
|
-
* `consecutiveFailures` on the entity (read fresh, so a re-drive after a rolled-back commit recomputes
|
|
671
|
-
* the SAME count), and either reschedules the SAME run at `now + retryMs` (looping back to `arm`) or
|
|
672
|
-
* disables the timer once `maxFailures` consecutive failures accumulate. The failure bookkeeping commits
|
|
673
|
-
* atomically with the step's journal memo. The run number is NOT advanced — a retry re-fires under the
|
|
674
|
-
* same `state.run`, so its eventual send keeps the same `base:run:N` key (parity with legacy, which
|
|
675
|
-
* leaves `run_count` untouched on `recordScheduledTimerFailure`).
|
|
676
|
-
*/
|
|
677
|
-
async function applyFireFailure(
|
|
678
|
-
ctx: ModuleCtx,
|
|
679
|
-
timer: Timer,
|
|
680
|
-
state: FireState,
|
|
681
|
-
now: number,
|
|
682
|
-
error: unknown,
|
|
683
|
-
): Promise<StepResult<FireState>> {
|
|
684
|
-
const cfg = timersConfig(ctx);
|
|
685
|
-
const outcome = failureBackoff({
|
|
686
|
-
now,
|
|
687
|
-
consecutiveFailures: timer.consecutiveFailures,
|
|
688
|
-
retryMs: cfg.retryMs,
|
|
689
|
-
maxFailures: cfg.maxFailures,
|
|
690
|
-
});
|
|
691
|
-
const store = timerStore(ctx);
|
|
692
|
-
if (outcome.kind === "disable") {
|
|
693
|
-
await store.update(state.id, { status: "disabled", consecutiveFailures: outcome.failures, lastError: errMessage(error) });
|
|
694
|
-
return { state, next: "done" };
|
|
695
|
-
}
|
|
696
|
-
await store.update(state.id, { consecutiveFailures: outcome.failures, lastError: errMessage(error) });
|
|
697
|
-
return { state: { ...state, fireAt: outcome.nextFireAt }, next: "arm" };
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
export const timers = defineModule({
|
|
701
|
-
id: "timers",
|
|
702
|
-
title: "Timers & Schedules",
|
|
703
|
-
description: "Kernel-served timers pilot: durable message + command firing on the K7 workflow runtime.",
|
|
704
|
-
config: {
|
|
705
|
-
schema: configSchema,
|
|
706
|
-
env: {
|
|
707
|
-
maxTimers: "TIMERS_MAX_TIMERS",
|
|
708
|
-
minRecurringIntervalMs: "TIMERS_MIN_RECURRING_INTERVAL_MS",
|
|
709
|
-
retryMs: "TIMERS_RETRY_MS",
|
|
710
|
-
maxFailures: "TIMERS_MAX_FAILURES",
|
|
711
|
-
},
|
|
712
|
-
overridableAt: {
|
|
713
|
-
maxTimers: ["host"],
|
|
714
|
-
minRecurringIntervalMs: ["host"],
|
|
715
|
-
retryMs: ["host"],
|
|
716
|
-
maxFailures: ["host"],
|
|
717
|
-
defaultLabel: ["instance", "host"],
|
|
718
|
-
},
|
|
719
|
-
},
|
|
720
|
-
entities: {
|
|
721
|
-
timer: {
|
|
722
|
-
schema: timerSchema,
|
|
723
|
-
// `createdBy` is indexed so the per-creator ownership filter (C2) on list/cancel/get is a keyed
|
|
724
|
-
// lookup, not a scan; `status` backs the active-quota count + status filter.
|
|
725
|
-
indexes: ["status", "createdBy"],
|
|
726
|
-
},
|
|
727
|
-
// The relay-dispatch transactional outbox (P2-3). A firing step writes one `pending` row IN its
|
|
728
|
-
// atomic step-commit (journal-atomic exactly-once on the send DECISION); the src drainer scans
|
|
729
|
-
// `pending` rows, calls `sendMessageService`, and flips them `sent`. `status` is indexed so the
|
|
730
|
-
// drainer's pending scan is a keyed lookup, not a table scan.
|
|
731
|
-
relaySend: {
|
|
732
|
-
schema: relaySendSchema,
|
|
733
|
-
indexes: ["status"],
|
|
734
|
-
},
|
|
735
|
-
// The per-timer durable last-applied-fence / tombstone record for the T-0 ingress apply seam
|
|
736
|
-
// (see `applyFenceSchema`). Point-read by id only — no secondary indexes.
|
|
737
|
-
applyFence: {
|
|
738
|
-
schema: applyFenceSchema,
|
|
739
|
-
},
|
|
740
|
-
},
|
|
741
|
-
events: {
|
|
742
|
-
"timers.created": z.object({ id: z.string(), label: z.string(), fireAt: z.number().int() }),
|
|
743
|
-
"timers.fired": z.object({ id: z.string(), firedAt: z.number().int(), run: z.number().int() }),
|
|
744
|
-
"timers.cancelled": z.object({ id: z.string() }),
|
|
745
|
-
},
|
|
746
|
-
commands: {
|
|
747
|
-
// The durable P3 shell command a command timer fires. The relay process NEVER execs — this
|
|
748
|
-
// dispatches onto the orchestrator command bus (src `dispatchCommand`), which the runner claims
|
|
749
|
-
// and runs as `bash -lc` at `cwd`, cwd-pinned + cgroup-reaped + timeout-bounded (the pre-existing
|
|
750
|
-
// legacy exec surface; the kernel neither moves nor widens it). `grant:"command:run-command"`
|
|
751
|
-
// maps src-side to — and never widens beyond — the legacy `command:write` + cwd-constrained authz
|
|
752
|
-
// (`dispatchCommand`'s #968 gate). `idempotencyKey` is the TIMER ID: a re-driven `fire` step
|
|
753
|
-
// dedupes onto the same command instead of double-dispatching (exactly-once boundary, #1408).
|
|
754
|
-
"run-command": {
|
|
755
|
-
input: z.object({
|
|
756
|
-
timerId: z.string().min(1),
|
|
757
|
-
command: z.string().min(1),
|
|
758
|
-
cwd: z.string().min(1),
|
|
759
|
-
env: z.record(z.string(), z.string()).optional(),
|
|
760
|
-
timeoutMs: z.number().int().positive(),
|
|
761
|
-
orchestratorId: z.string().min(1),
|
|
762
|
-
identity: frozenIdentitySchema,
|
|
763
|
-
}),
|
|
764
|
-
result: z.object({
|
|
765
|
-
exitCode: z.number().nullable(),
|
|
766
|
-
timedOut: z.boolean(),
|
|
767
|
-
stdoutTail: z.string(),
|
|
768
|
-
stderrTail: z.string(),
|
|
769
|
-
}),
|
|
770
|
-
target: "runner",
|
|
771
|
-
grant: "command:run-command",
|
|
772
|
-
idempotencyKey: (input) => input.timerId,
|
|
773
|
-
// Static declared ceiling (≤ 24h, matching the legacy COMMAND_MAX_MS). The ACTUAL per-run exec
|
|
774
|
-
// timeout rides in `input.timeoutMs`; this is the coarse manifest-level bound.
|
|
775
|
-
timeoutMs: 24 * 60 * 60 * 1000,
|
|
776
|
-
whenDegraded: "queue",
|
|
777
|
-
},
|
|
778
|
-
},
|
|
779
|
-
workflows: {
|
|
780
|
-
// Durable firing for a single timer — a LOOPING workflow (P2-5). `create` starts it with the timer
|
|
781
|
-
// id + fireAt + run 1.
|
|
782
|
-
// arm → parks the instance `sleeping` until the absolute `fireAt` (in state), never a setTimeout.
|
|
783
|
-
// fire → on the wakeup pass at/after fireAt, reads the timer and — only if still `scheduled`:
|
|
784
|
-
// • past its `expiresAt` → disables WITHOUT firing (legacy expiry-before-delivery).
|
|
785
|
-
// • a COMMAND timer issues its P3 `run-command` and parks `waiting` on it (→ `await`); a
|
|
786
|
-
// dispatch ERROR is a timer-run failure → backoff/disable (SCOPE 3). Command timers are
|
|
787
|
-
// NEVER recurring (legacy parity), so they always settle to `done` via `await`.
|
|
788
|
-
// • a MESSAGE/plain timer flips status + emits `timers.fired` (+ writes the P2-3 outbox
|
|
789
|
-
// row) in ONE atomic step commit, then — if it is a RECURRING interval still under
|
|
790
|
-
// `maxRuns`/`expiresAt` — loops back to `arm` with the NEXT deadline and run number `N+1`;
|
|
791
|
-
// otherwise it disables (terminal `fired`).
|
|
792
|
-
// Cancel-safe + idempotent by construction: the timer entity is the source of truth, so a timer
|
|
793
|
-
// cancelled (or already terminal) before a deadline simply no-ops when its firing workflow wakes.
|
|
794
|
-
//
|
|
795
|
-
// G3 (exactly-once): the run number `N` a fire dispatches under is `state.run` — carried in the
|
|
796
|
-
// DURABLE workflow state (memoized in the journal, reconstructed by the fold), NOT a live counter.
|
|
797
|
-
// A re-drive of run N recomputes the SAME N (its `arm` memo pins `state.run` before `fire` re-runs),
|
|
798
|
-
// so the `base:run:N` outbox dedup holds → no double-send. A fresh run only ever gets `N` from the
|
|
799
|
-
// PREVIOUS fire's COMMITTED transition to `run+1`, so distinct committed runs get distinct N.
|
|
800
|
-
fire: {
|
|
801
|
-
// `initialRun` (optional, default 1) is the run number the FIRST fire dispatches under. The
|
|
802
|
-
// create path never supplies it (→ run 1, unchanged behavior). The P2-6 migrate-at-cutover op
|
|
803
|
-
// seeds it to `legacyRunCount + 1` so a mid-stream recurring timer's kernel run numbering
|
|
804
|
-
// CONTINUES the legacy sequence instead of restarting at 1 — that alignment is what makes the
|
|
805
|
-
// §4.2 dual-read `base:run:N` dedup hold for a migrated recurring timer during the bake window.
|
|
806
|
-
input: z.object({
|
|
807
|
-
id: z.string(),
|
|
808
|
-
fireAt: z.number().int(),
|
|
809
|
-
initialRun: z.number().int().positive().optional(),
|
|
810
|
-
fence: z.number().int().nonnegative().nullable().optional(),
|
|
811
|
-
generation: z.number().int().nonnegative().optional(),
|
|
812
|
-
run: z.number().int().positive().optional(),
|
|
813
|
-
}),
|
|
814
|
-
state: z.object({
|
|
815
|
-
id: z.string(),
|
|
816
|
-
fence: z.number().int().nonnegative().nullable(),
|
|
817
|
-
generation: z.number().int().nonnegative(),
|
|
818
|
-
fireAt: z.number().int(),
|
|
819
|
-
commandId: z.string().nullable(),
|
|
820
|
-
run: z.number().int().positive(),
|
|
821
|
-
}),
|
|
822
|
-
init: (input) => ({
|
|
823
|
-
id: input.id,
|
|
824
|
-
fence: input.fence ?? null,
|
|
825
|
-
generation: input.generation ?? 0,
|
|
826
|
-
fireAt: input.fireAt,
|
|
827
|
-
commandId: null,
|
|
828
|
-
run: input.run ?? input.initialRun ?? 1,
|
|
829
|
-
}),
|
|
830
|
-
start: "arm",
|
|
831
|
-
steps: {
|
|
832
|
-
arm: {
|
|
833
|
-
run: async (_ctx, state: FireState): Promise<StepResult<FireState>> => {
|
|
834
|
-
// Deterministic: the deadline comes from state (the current run's fireAt), never the clock.
|
|
835
|
-
return { state, next: "fire", sleepUntil: state.fireAt };
|
|
836
|
-
},
|
|
837
|
-
},
|
|
838
|
-
fire: {
|
|
839
|
-
run: async (ctx, state: FireState): Promise<StepResult<FireState>> => {
|
|
840
|
-
const store = timerStore(ctx);
|
|
841
|
-
const timer = await store.get(state.id);
|
|
842
|
-
// Cancelled or already terminal before the deadline: no-op (cancel-safe firing).
|
|
843
|
-
if (!timer || timer.status !== "scheduled") return { state, next: "done" };
|
|
844
|
-
const now = ctx.clock.now();
|
|
845
|
-
// Expiry-before-delivery (legacy `fireClaimedTimer`): a timer past its expiry disables WITHOUT
|
|
846
|
-
// firing this run — no flip to `fired`, no event, no send.
|
|
847
|
-
if (timer.expiresAt !== undefined && timer.expiresAt <= now) {
|
|
848
|
-
await store.update(state.id, { status: "disabled", lastError: "timer expired before delivery" });
|
|
849
|
-
if (state.fence !== null) {
|
|
850
|
-
if (!timer.message) throw new TypeError("authority workflow requires a pure MESSAGE timer");
|
|
851
|
-
ctx.messageAuthority.decideFire({
|
|
852
|
-
timer: { timerId: timer.id, message: timer.message },
|
|
853
|
-
fence: state.fence,
|
|
854
|
-
generation: state.generation,
|
|
855
|
-
run: state.run,
|
|
856
|
-
engine: "kernel",
|
|
857
|
-
decision: "expired",
|
|
858
|
-
fireAt: state.fireAt,
|
|
859
|
-
lifecycle: "disabled",
|
|
860
|
-
nextRun: null,
|
|
861
|
-
nextRunAt: null,
|
|
862
|
-
decidedAt: now,
|
|
863
|
-
sendPayload: null,
|
|
864
|
-
idempotencyKey: null,
|
|
865
|
-
});
|
|
866
|
-
}
|
|
867
|
-
return { state, next: "done" };
|
|
868
|
-
}
|
|
869
|
-
if (timer.command) {
|
|
870
|
-
// Command timer: dispatch the authz-gated P3 command and park on its terminal status. The
|
|
871
|
-
// issue is idempotency-keyed on the timer id (see the decl), so a crash-and-re-drive of THIS
|
|
872
|
-
// step dedupes onto the same orchestrator command — never a second dispatch. A dispatch
|
|
873
|
-
// ERROR is a TIMER-RUN failure (distinct from M3 per-step retry): back off + disable after
|
|
874
|
-
// `maxFailures` (SCOPE 3), rather than terminating the whole firing workflow.
|
|
875
|
-
let run: HostCommandRun;
|
|
876
|
-
try {
|
|
877
|
-
run = await ctx.commands.issue(
|
|
878
|
-
"run-command",
|
|
879
|
-
{
|
|
880
|
-
timerId: timer.id,
|
|
881
|
-
command: timer.command.command,
|
|
882
|
-
cwd: timer.command.cwd,
|
|
883
|
-
...(timer.command.env ? { env: timer.command.env } : {}),
|
|
884
|
-
timeoutMs: timer.command.timeoutMs,
|
|
885
|
-
orchestratorId: timer.command.orchestratorId,
|
|
886
|
-
identity: timer.command.identity,
|
|
887
|
-
},
|
|
888
|
-
{ host: timer.command.orchestratorId },
|
|
889
|
-
);
|
|
890
|
-
} catch (error) {
|
|
891
|
-
return applyFireFailure(ctx, timer, state, now, error);
|
|
892
|
-
}
|
|
893
|
-
// A successful dispatch clears any prior failure streak (parity: success resets it).
|
|
894
|
-
if (timer.consecutiveFailures > 0) {
|
|
895
|
-
await store.update(state.id, { consecutiveFailures: 0, lastError: undefined });
|
|
896
|
-
}
|
|
897
|
-
return { state: { ...state, commandId: run.id }, next: "await", waitFor: { kind: "command", commandId: run.id } };
|
|
898
|
-
}
|
|
899
|
-
// Message/plain timer: decide the recurring continuation from DURABLE state (this run number
|
|
900
|
-
// = runs completed once it lands), then flip + emit AND — if it carries a resolved send —
|
|
901
|
-
// write the relay-dispatch outbox row (P2-3), all in ONE atomic step commit. The outbox
|
|
902
|
-
// `create` buffers into the same K4 session `commitStepAttempt` flushes with the journal memo,
|
|
903
|
-
// so the DECISION to send is journal-atomic exactly-once: a fault between the step effects and
|
|
904
|
-
// the commit rolls the whole tx back (no flip, no event, NO outbox row, NO run advance); a
|
|
905
|
-
// re-drive re-runs THIS run (same `state.run`) atomically. The src drainer performs the actual
|
|
906
|
-
// send, deduped effectively-once by `base:run:N`.
|
|
907
|
-
const cont = recurringContinuation({
|
|
908
|
-
everyMs: timer.everyMs,
|
|
909
|
-
now,
|
|
910
|
-
prevFireAt: state.fireAt,
|
|
911
|
-
runsCompleted: state.run,
|
|
912
|
-
maxRuns: timer.maxRuns,
|
|
913
|
-
expiresAt: timer.expiresAt,
|
|
914
|
-
});
|
|
915
|
-
await store.update(state.id, {
|
|
916
|
-
status: cont.kind === "terminal" ? "fired" : "scheduled",
|
|
917
|
-
...(cont.kind === "continue" ? { fireAt: cont.nextFireAt } : {}),
|
|
918
|
-
runCount: state.run,
|
|
919
|
-
consecutiveFailures: 0,
|
|
920
|
-
lastError: undefined,
|
|
921
|
-
});
|
|
922
|
-
await ctx.events.emit("timers.fired", { id: state.id, firedAt: now, run: state.run });
|
|
923
|
-
if (timer.message) {
|
|
924
|
-
if (state.fence === null) {
|
|
925
|
-
await relaySendStore(ctx).create(messageOutboxInput(ctx, timer, timer.message, state.run));
|
|
926
|
-
} else {
|
|
927
|
-
const send = authoritySendPayload(timer, timer.message, state.run);
|
|
928
|
-
ctx.messageAuthority.decideFire({
|
|
929
|
-
timer: { timerId: timer.id, message: timer.message },
|
|
930
|
-
fence: state.fence,
|
|
931
|
-
generation: state.generation,
|
|
932
|
-
run: state.run,
|
|
933
|
-
engine: "kernel",
|
|
934
|
-
decision: "dispatch",
|
|
935
|
-
fireAt: state.fireAt,
|
|
936
|
-
lifecycle: cont.kind === "continue" ? "scheduled" : "fired",
|
|
937
|
-
nextRun: cont.kind === "continue" ? cont.nextRun : null,
|
|
938
|
-
nextRunAt: cont.kind === "continue" ? cont.nextFireAt : null,
|
|
939
|
-
decidedAt: now,
|
|
940
|
-
sendPayload: send.sendPayload,
|
|
941
|
-
idempotencyKey: send.idempotencyKey,
|
|
942
|
-
});
|
|
943
|
-
}
|
|
944
|
-
}
|
|
945
|
-
// Recurring: loop back to `arm` with the next deadline + the NEXT run number. The advance to
|
|
946
|
-
// `run+1` is only durable once THIS transition commits, so a re-drive of this run never sees it.
|
|
947
|
-
if (cont.kind === "continue") {
|
|
948
|
-
return { state: { ...state, fireAt: cont.nextFireAt, run: cont.nextRun }, next: "arm" };
|
|
949
|
-
}
|
|
950
|
-
return { state, next: "done" };
|
|
951
|
-
},
|
|
952
|
-
},
|
|
953
|
-
await: {
|
|
954
|
-
run: async (ctx, state: FireState): Promise<StepResult<FireState>> => {
|
|
955
|
-
// Reached only by a command timer (never recurring), once the wait evaluator reports terminal.
|
|
956
|
-
const store = timerStore(ctx);
|
|
957
|
-
const timer = await store.get(state.id);
|
|
958
|
-
// Still cancel-safe: a cancel between issue and settle → no fired flip, no event, no send.
|
|
959
|
-
if (timer && timer.status === "scheduled") {
|
|
960
|
-
await store.update(state.id, { status: "fired", runCount: state.run });
|
|
961
|
-
await ctx.events.emit("timers.fired", { id: state.id, firedAt: ctx.clock.now(), run: state.run });
|
|
962
|
-
// P2-3 (completion seam now CLOSED): if the command timer carries a completion send, write
|
|
963
|
-
// its outbox row IN this same atomic commit. The command result ({exitCode, timedOut,
|
|
964
|
-
// stdoutTail, stderrTail}) is durable and read back via `ctx.commands.status(commandId)`;
|
|
965
|
-
// the completion body/payload compose from it. Same journal-atomic exactly-once guarantee
|
|
966
|
-
// as the message path — the outbox row commits with the fired flip + event or not at all.
|
|
967
|
-
if (timer.message) {
|
|
968
|
-
const run = state.commandId ? await ctx.commands.status(state.commandId) : undefined;
|
|
969
|
-
const result = run?.result as CommandRunResult | undefined;
|
|
970
|
-
await relaySendStore(ctx).create(
|
|
971
|
-
commandCompletionOutboxInput(ctx, timer, timer.message, state.run, state.commandId, result),
|
|
972
|
-
);
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
return { state, next: "done", output: { id: state.id, commandId: state.commandId } };
|
|
976
|
-
},
|
|
977
|
-
},
|
|
978
|
-
},
|
|
979
|
-
},
|
|
980
|
-
},
|
|
981
|
-
ops: {
|
|
982
|
-
create: {
|
|
983
|
-
kind: "mutation",
|
|
984
|
-
idempotent: false,
|
|
985
|
-
grant: "timers:write",
|
|
986
|
-
summary: "Create a scheduled timer.",
|
|
987
|
-
emits: ["timers.created"],
|
|
988
|
-
http: { method: "POST", path: "/timers" },
|
|
989
|
-
// `command` (optional) makes this a COMMAND timer (see the module header + `run-command` decl).
|
|
990
|
-
// `message` (optional) makes firing dispatch a Relay message via the P2-3 outbox: on a plain
|
|
991
|
-
// timer it is the whole effect; on a command timer it is the completion send. Both freeze a
|
|
992
|
-
// server-resolved identity the drainer fires under.
|
|
993
|
-
// P2-4 (C2 residual #1): the create input carries NO client-owned frozen identity — the op
|
|
994
|
-
// SERVER-RESOLVES it from the authenticated principal (`ctx.principal`) + the token's real cwd
|
|
995
|
-
// constraint. A client cannot name/broaden its own authority; the fire-time reconstruction binds
|
|
996
|
-
// the command cwd to the creator's REAL constraint via `dispatchCommand`'s #968 gate (R1).
|
|
997
|
-
// `everyMs` (optional) makes this a RECURRING interval timer (P2-5): firing loops arm→fire→arm,
|
|
998
|
-
// rescheduling `fireAt` deterministically. `maxRuns`/`expiresAt` bound it. Command timers may NOT
|
|
999
|
-
// be recurring; a recurring timer needs a bound (`maxRuns`/`expiresAt`) or the persistent grant.
|
|
1000
|
-
input: z.object({
|
|
1001
|
-
label: z.string().min(1).optional(),
|
|
1002
|
-
fireAt: z.number().int(),
|
|
1003
|
-
everyMs: z.number().int().positive().optional(),
|
|
1004
|
-
maxRuns: z.number().int().positive().optional(),
|
|
1005
|
-
expiresAt: z.number().int().optional(),
|
|
1006
|
-
command: commandRequestSchema.optional(),
|
|
1007
|
-
message: messageRequestSchema.optional(),
|
|
1008
|
-
}),
|
|
1009
|
-
output: timerSchema,
|
|
1010
|
-
handler: async (ctx, input) => {
|
|
1011
|
-
const cfg = timersConfig(ctx);
|
|
1012
|
-
const now = ctx.clock.now();
|
|
1013
|
-
const createdBy = ctx.principal.actor;
|
|
1014
|
-
|
|
1015
|
-
// Quota / limit re-expression (C2 threat 3), so the kernel path can't be used to evade the
|
|
1016
|
-
// legacy scheduler's DoS ceilings:
|
|
1017
|
-
// • afterMs ≤ 24h → the timer may not be scheduled further than `maxFireAheadMs` ahead.
|
|
1018
|
-
if (input.fireAt > now + cfg.maxFireAheadMs) {
|
|
1019
|
-
throw new OpError("invalid_input", 400, `fireAt is more than ${cfg.maxFireAheadMs}ms ahead of now`);
|
|
1020
|
-
}
|
|
1021
|
-
|
|
1022
|
-
// SCOPE 4 — recurring/interval validation parity (legacy `scheduleTimer`):
|
|
1023
|
-
// • command timers may NOT be recurring;
|
|
1024
|
-
// • `everyMs >= minRecurringIntervalMs`;
|
|
1025
|
-
// • a recurring timer needs a bound (`maxRuns`/`expiresAt`) OR the persistent grant;
|
|
1026
|
-
// • `expiresAt` (any timer) must be after the first run.
|
|
1027
|
-
if (input.everyMs !== undefined) {
|
|
1028
|
-
if (input.command) {
|
|
1029
|
-
throw new OpError("invalid_input", 400, "everyMs is not supported with command timers");
|
|
1030
|
-
}
|
|
1031
|
-
if (input.everyMs < cfg.minRecurringIntervalMs) {
|
|
1032
|
-
throw new OpError("invalid_input", 400, `everyMs must be at least ${cfg.minRecurringIntervalMs}`);
|
|
1033
|
-
}
|
|
1034
|
-
if (input.maxRuns === undefined && input.expiresAt === undefined && !canCreatePersistentTimer(ctx)) {
|
|
1035
|
-
throw new OpError("unauthorized", 403, "recurring timers require maxRuns, expiresAt, or the schedule:persistent grant");
|
|
1036
|
-
}
|
|
1037
|
-
}
|
|
1038
|
-
if (input.expiresAt !== undefined && input.expiresAt <= input.fireAt) {
|
|
1039
|
-
throw new OpError("invalid_input", 400, "expiresAt must be after the first run");
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
// • SCHEDULER_MAX_ACTIVE_TIMERS per creator → count this creator's active (scheduled) timers.
|
|
1043
|
-
// The entity page caps at 200; `min` keeps the count within it (the default quota is 100).
|
|
1044
|
-
const active = await timerStore(ctx).list({ where: { status: "scheduled", createdBy }, limit: Math.min(cfg.maxTimers, 200) });
|
|
1045
|
-
if (active.items.length >= cfg.maxTimers) {
|
|
1046
|
-
throw new OpError("quota_exceeded", 429, `active timer quota exceeded (${cfg.maxTimers})`);
|
|
1047
|
-
}
|
|
1048
|
-
|
|
1049
|
-
// SCOPE 1 — server-resolve + FREEZE the command-timer identity from the authenticated principal.
|
|
1050
|
-
// A command timer additionally requires the `command:write` grant AND a cwd-constrained creator
|
|
1051
|
-
// identity at create (mirroring legacy `normalizeScheduleCommand` + `assertScheduleCommandAuthorized`),
|
|
1052
|
-
// so an unconstrained / non-command-write caller cannot create a command timer AT ALL.
|
|
1053
|
-
const command = input.command ? freezeCommandTimer(ctx, input.command, createdBy, now) : undefined;
|
|
1054
|
-
|
|
1055
|
-
// The timer id is generated up-front so the frozen message can carry the `timer:<id>`
|
|
1056
|
-
// correlation id (D1) — byte-parity with legacy `insertScheduledTimer`, which derives the same.
|
|
1057
|
-
const id = ctx.rng.uuid();
|
|
1058
|
-
|
|
1059
|
-
// SCOPE 1 (P2-3) — server-resolve + FREEZE the relay-send spec. A pure message timer REQUIRES a
|
|
1060
|
-
// body; a command timer's completion body may be omitted (it defaults from the command outcome).
|
|
1061
|
-
const message = input.message
|
|
1062
|
-
? freezeMessageTimer(ctx, input.message, createdBy, now, /*requireBody*/ !command, `timer:${id}`)
|
|
1063
|
-
: undefined;
|
|
1064
|
-
|
|
1065
|
-
const label = input.label ?? cfg.defaultLabel;
|
|
1066
|
-
const created = await timerStore(ctx).create({
|
|
1067
|
-
id,
|
|
1068
|
-
label,
|
|
1069
|
-
createdBy,
|
|
1070
|
-
fireAt: input.fireAt,
|
|
1071
|
-
...(input.everyMs !== undefined ? { everyMs: input.everyMs } : {}),
|
|
1072
|
-
...(input.maxRuns !== undefined ? { maxRuns: input.maxRuns } : {}),
|
|
1073
|
-
...(input.expiresAt !== undefined ? { expiresAt: input.expiresAt } : {}),
|
|
1074
|
-
...(command ? { command } : {}),
|
|
1075
|
-
...(message ? { message } : {}),
|
|
1076
|
-
});
|
|
1077
|
-
await ctx.events.emit("timers.created", { id, label, fireAt: input.fireAt });
|
|
1078
|
-
// Enqueue durable firing, ENLISTED in this op's atomic commit (T-0 #1409 finding 1): the
|
|
1079
|
-
// instance seed executes INSIDE the same transaction as the entity + event flush, so the
|
|
1080
|
-
// timer and its firing workflow commit together or not at all — an op that rolls back
|
|
1081
|
-
// (validation failure, version conflict) leaves NO orphaned instance keyed to this id for
|
|
1082
|
-
// any later create/migrate of the same id to dedupe onto. The drive runs after the commit;
|
|
1083
|
-
// a crash before it leaves a durable `pending` instance the recovery pass converges.
|
|
1084
|
-
// Idempotency key = timer id, so a retried create dedupes onto the same instance.
|
|
1085
|
-
await ctx.workflows.start("fire", { id, fireAt: input.fireAt }, { idempotencyKey: id, enlist: "op-commit" });
|
|
1086
|
-
return created;
|
|
1087
|
-
},
|
|
1088
|
-
},
|
|
1089
|
-
get: {
|
|
1090
|
-
kind: "query",
|
|
1091
|
-
grant: "timers:read",
|
|
1092
|
-
summary: "Fetch a timer by id.",
|
|
1093
|
-
http: { method: "GET", path: "/timers/{id}" },
|
|
1094
|
-
input: z.object({ id: z.string() }),
|
|
1095
|
-
output: timerSchema,
|
|
1096
|
-
handler: async (ctx, input) => {
|
|
1097
|
-
const timer = await timerStore(ctx).get(input.id);
|
|
1098
|
-
// Per-creator ownership (C2): a non-admin principal cannot even OBSERVE another creator's timer.
|
|
1099
|
-
// Report the same 404 as a genuinely-missing id, so a non-owner cannot probe existence.
|
|
1100
|
-
const owner = creatorFilter(ctx);
|
|
1101
|
-
if (!timer || (owner && timer.createdBy !== owner.createdBy)) {
|
|
1102
|
-
throw new OpError("not_found", 404, `timer '${input.id}' was not found`);
|
|
1103
|
-
}
|
|
1104
|
-
return timer;
|
|
1105
|
-
},
|
|
1106
|
-
},
|
|
1107
|
-
list: {
|
|
1108
|
-
kind: "query",
|
|
1109
|
-
grant: "timers:read",
|
|
1110
|
-
summary: "List timers, optionally filtered by status.",
|
|
1111
|
-
http: { method: "GET", path: "/timers" },
|
|
1112
|
-
input: z.object({
|
|
1113
|
-
status: statusSchema.optional(),
|
|
1114
|
-
// Coerced: over HTTP this arrives as a query-string value; in-process callers may still
|
|
1115
|
-
// pass a number. `z.coerce.number` accepts both, so the transport stays type-agnostic.
|
|
1116
|
-
limit: z.coerce.number().int().min(1).max(200).optional(),
|
|
1117
|
-
cursor: z.string().optional(),
|
|
1118
|
-
}),
|
|
1119
|
-
output: z.object({ items: z.array(timerSchema), cursor: z.string().optional() }),
|
|
1120
|
-
handler: async (ctx, input) => {
|
|
1121
|
-
// Per-creator ownership (C2): a non-admin principal only ever sees its OWN timers — the op
|
|
1122
|
-
// applies a `where: {createdBy}` filter from `ctx.principal.actor`; an admin (system:admin)
|
|
1123
|
-
// crosses creators. Merged with the optional status filter.
|
|
1124
|
-
const where = { ...creatorFilter(ctx), ...(input.status ? { status: input.status } : {}) };
|
|
1125
|
-
const page = await timerStore(ctx).list({
|
|
1126
|
-
limit: input.limit,
|
|
1127
|
-
cursor: input.cursor,
|
|
1128
|
-
...(Object.keys(where).length ? { where } : {}),
|
|
1129
|
-
});
|
|
1130
|
-
return { items: [...page.items], ...(page.cursor ? { cursor: page.cursor } : {}) };
|
|
1131
|
-
},
|
|
1132
|
-
},
|
|
1133
|
-
cancel: {
|
|
1134
|
-
kind: "mutation",
|
|
1135
|
-
idempotent: true,
|
|
1136
|
-
grant: "timers:write",
|
|
1137
|
-
summary: "Cancel a scheduled timer.",
|
|
1138
|
-
emits: ["timers.cancelled"],
|
|
1139
|
-
http: { method: "DELETE", path: "/timers/{id}" },
|
|
1140
|
-
input: z.object({ id: z.string() }),
|
|
1141
|
-
output: timerSchema,
|
|
1142
|
-
handler: async (ctx, input) => {
|
|
1143
|
-
const store = timerStore(ctx);
|
|
1144
|
-
const timer = await store.get(input.id);
|
|
1145
|
-
// Per-creator ownership (C2): a non-admin principal cannot cancel another creator's timer, and
|
|
1146
|
-
// gets the SAME 404 a missing id yields (no existence disclosure). Admin (system:admin) crosses.
|
|
1147
|
-
const owner = creatorFilter(ctx);
|
|
1148
|
-
if (!timer || (owner && timer.createdBy !== owner.createdBy)) {
|
|
1149
|
-
throw new OpError("not_found", 404, `timer '${input.id}' was not found`);
|
|
1150
|
-
}
|
|
1151
|
-
// Idempotent: a repeat cancel is a no-op that emits nothing and mutates nothing.
|
|
1152
|
-
if (timer.status === "cancelled") return timer;
|
|
1153
|
-
const cancelled = await store.update(input.id, { status: "cancelled" });
|
|
1154
|
-
await ctx.events.emit("timers.cancelled", { id: input.id });
|
|
1155
|
-
return cancelled;
|
|
1156
|
-
},
|
|
1157
|
-
},
|
|
1158
|
-
// ── P2-6 migrate-at-cutover surface (SYSTEM-only; the src migration script invokes this) ───────
|
|
1159
|
-
// The one-time forward migration (§4.2, RATIFIED disjoint-tables model) copies an ACTIVE legacy
|
|
1160
|
-
// `scheduled_timers` row into the kernel timer entity + enqueues its `fire` workflow. Like the drain
|
|
1161
|
-
// ops it is drainer-only (`timers:admin`) and never exposed over any transport, so it lands as a
|
|
1162
|
-
// prod no-op until the cutover script runs it. The src side (`kernel-timers-migrate.ts`) owns the
|
|
1163
|
-
// legacy→kernel field mapping; this op just faithfully UPSERTS the resolved entity.
|
|
1164
|
-
//
|
|
1165
|
-
// IDEMPOTENT (the crux, keyed on the LEGACY timer id): the kernel timer id IS the legacy id, so a
|
|
1166
|
-
// re-run finds the entity already present and does NOT create a duplicate. It ALWAYS (re-)starts the
|
|
1167
|
-
// fire workflow with `idempotencyKey = id`, which the runtime dedupes onto the existing instance —
|
|
1168
|
-
// so a crash BETWEEN the entity create and the workflow start converges on re-run (the entity is
|
|
1169
|
-
// there, the still-missing instance gets started). Safe to re-run / resume after a crash.
|
|
1170
|
-
//
|
|
1171
|
-
// FENCED (T-0 hardening, #1409): an optional `fence` (the caller's ingress seq) makes the write
|
|
1172
|
-
// refuse when stale — `fence <` the per-timer `applyFence` record (e.g. a cancel tombstone) returns
|
|
1173
|
-
// `{fenced: true}` with NO entity write, NO workflow start, NO event. Equal fence = same-intent
|
|
1174
|
-
// re-drive, proceeds idempotently. And when the staleness is only discovered AT FLUSH (both ops
|
|
1175
|
-
// read the old fence; the other committed first), the lost CAS aborts the WHOLE commit — including
|
|
1176
|
-
// the enlisted workflow seed — so the losing migrate leaves nothing at all behind (second-pass
|
|
1177
|
-
// hardening, T1175). See `applyFenceSchema` for the full contract.
|
|
1178
|
-
//
|
|
1179
|
-
// SECURITY (server-trusted snapshot — sidesteps G1's client-input concern): `command.identity` /
|
|
1180
|
-
// `message.identity` here are the LEGACY row's PERSISTED create-time identity (the server-resolved
|
|
1181
|
-
// snapshot `insertScheduledTimer` froze), carried verbatim — NOT client input at migrate time. So no
|
|
1182
|
-
// creator-facing re-validation (quota / persistent-grant / cwd-constraint) runs: those gated the
|
|
1183
|
-
// legacy row at ITS create, and this copies that already-trusted state. The fire-time #968 cwd gate
|
|
1184
|
-
// still binds a command's cwd to `command.identity` exactly as for a natively-created kernel timer.
|
|
1185
|
-
migrate: {
|
|
1186
|
-
kind: "mutation",
|
|
1187
|
-
idempotent: true,
|
|
1188
|
-
grant: TIMERS_DRAIN_GRANT,
|
|
1189
|
-
summary: "Forward-migrate one active legacy timer into the kernel entity + enqueue firing (P2-6; system-only).",
|
|
1190
|
-
emits: ["timers.created"],
|
|
1191
|
-
expose: { http: false, mcp: false, cli: false },
|
|
1192
|
-
input: z.object({
|
|
1193
|
-
id: z.string().min(1),
|
|
1194
|
-
label: z.string().min(1),
|
|
1195
|
-
createdBy: z.string().min(1),
|
|
1196
|
-
fireAt: z.number().int(),
|
|
1197
|
-
everyMs: z.number().int().positive().optional(),
|
|
1198
|
-
maxRuns: z.number().int().positive().optional(),
|
|
1199
|
-
expiresAt: z.number().int().optional(),
|
|
1200
|
-
// Runs the legacy timer already COMPLETED. The kernel entity carries it as its projection and
|
|
1201
|
-
// the fire workflow starts at `runCount + 1`, so the migrated timer's next run number CONTINUES
|
|
1202
|
-
// the legacy sequence (dual-read dedup parity for recurring timers).
|
|
1203
|
-
runCount: z.number().int().nonnegative().default(0),
|
|
1204
|
-
// Failure-backoff state must SURVIVE migration (#1409): a legacy timer mid-backoff (e.g. 2 of 3
|
|
1205
|
-
// consecutive failures) migrates with its streak intact, so the kernel keeps disabling it after
|
|
1206
|
-
// `maxFailures` rather than resetting to 0 and re-granting it a full retry budget.
|
|
1207
|
-
consecutiveFailures: z.number().int().nonnegative().default(0),
|
|
1208
|
-
lastError: z.string().optional(),
|
|
1209
|
-
command: commandSpecSchema.optional(),
|
|
1210
|
-
message: messageSpecSchema.optional(),
|
|
1211
|
-
// T-0 apply fence (#1409 hardening): the caller's ingress seq. When present, a stale write
|
|
1212
|
-
// (fence < the stored per-timer fence) is REFUSED before any effect — see `applyFenceSchema`.
|
|
1213
|
-
fence: z.number().int().positive().optional(),
|
|
1214
|
-
// Stage 5 authority snapshot. These are supplied together only when a real authority row
|
|
1215
|
-
// exists; shadow/fence-only ingress intentionally omits them and retains its old workflow.
|
|
1216
|
-
generation: z.number().int().nonnegative().optional(),
|
|
1217
|
-
run: z.number().int().positive().optional(),
|
|
1218
|
-
}),
|
|
1219
|
-
output: z.object({ timer: timerSchema.optional(), created: z.boolean(), fenced: z.boolean() }),
|
|
1220
|
-
handler: async (ctx, input) => {
|
|
1221
|
-
const authorityApply = input.generation !== undefined || input.run !== undefined;
|
|
1222
|
-
if (authorityApply && (input.fence === undefined || input.generation === undefined || input.run === undefined || !input.message || input.command)) {
|
|
1223
|
-
throw new OpError("invalid_input", 400, "authority migrate requires a pure MESSAGE timer with exact fence/generation/run");
|
|
1224
|
-
}
|
|
1225
|
-
// Fence check FIRST — before the entity write AND before the workflow start (which commits in
|
|
1226
|
-
// its own transaction): a stale fenced migrate must have NO effect at all. The read-check is
|
|
1227
|
-
// raced-safe because the fence upsert below flushes in this op's atomic commit (CAS-guarded);
|
|
1228
|
-
// a loser aborts wholesale and its retry re-reads the newer fence here.
|
|
1229
|
-
const gate = input.fence !== undefined ? await applyFenceStore(ctx).get(input.id) : undefined;
|
|
1230
|
-
if (input.fence !== undefined && gate !== undefined && input.fence < gate.fence) {
|
|
1231
|
-
return { created: false, fenced: true };
|
|
1232
|
-
}
|
|
1233
|
-
const store = timerStore(ctx);
|
|
1234
|
-
const existing = await store.get(input.id);
|
|
1235
|
-
let timer = existing;
|
|
1236
|
-
const created = existing === undefined;
|
|
1237
|
-
if (!existing) {
|
|
1238
|
-
timer = await store.create({
|
|
1239
|
-
id: input.id,
|
|
1240
|
-
label: input.label,
|
|
1241
|
-
createdBy: input.createdBy,
|
|
1242
|
-
fireAt: input.fireAt,
|
|
1243
|
-
runCount: input.runCount,
|
|
1244
|
-
consecutiveFailures: input.consecutiveFailures,
|
|
1245
|
-
...(input.lastError !== undefined ? { lastError: input.lastError } : {}),
|
|
1246
|
-
...(input.everyMs !== undefined ? { everyMs: input.everyMs } : {}),
|
|
1247
|
-
...(input.maxRuns !== undefined ? { maxRuns: input.maxRuns } : {}),
|
|
1248
|
-
...(input.expiresAt !== undefined ? { expiresAt: input.expiresAt } : {}),
|
|
1249
|
-
...(input.command ? { command: input.command } : {}),
|
|
1250
|
-
...(input.message ? { message: input.message } : {}),
|
|
1251
|
-
});
|
|
1252
|
-
await ctx.events.emit("timers.created", { id: input.id, label: input.label, fireAt: input.fireAt });
|
|
1253
|
-
} else if (input.fence !== undefined && (gate === undefined || input.fence > gate.fence)) {
|
|
1254
|
-
// Higher-F apply is convergence, not create-only mirroring. Write every timer field,
|
|
1255
|
-
// including explicit undefined values that clear fields removed by the new intent.
|
|
1256
|
-
timer = await store.update(input.id, {
|
|
1257
|
-
label: input.label,
|
|
1258
|
-
createdBy: input.createdBy,
|
|
1259
|
-
fireAt: input.fireAt,
|
|
1260
|
-
status: "scheduled",
|
|
1261
|
-
everyMs: input.everyMs,
|
|
1262
|
-
maxRuns: input.maxRuns,
|
|
1263
|
-
expiresAt: input.expiresAt,
|
|
1264
|
-
runCount: input.runCount,
|
|
1265
|
-
consecutiveFailures: input.consecutiveFailures,
|
|
1266
|
-
lastError: input.lastError,
|
|
1267
|
-
command: input.command,
|
|
1268
|
-
message: input.message,
|
|
1269
|
-
});
|
|
1270
|
-
}
|
|
1271
|
-
// ALWAYS (re-)enqueue firing, idempotently AND — the T-0 #1409 finding-1 crux — ENLISTED in
|
|
1272
|
-
// this op's atomic commit: the instance seed executes INSIDE the same transaction that
|
|
1273
|
-
// flushes the entity and the applyFence CAS below. A migrate that read a stale fence and
|
|
1274
|
-
// LOSES that CAS at flush (a concurrent newer cancel/create committed first) therefore
|
|
1275
|
-
// rolls back the workflow seed WITH the entity and the fence — a fence-losing migrate
|
|
1276
|
-
// commits NOTHING: no entity, no event, no fence write, no instance. That closes the
|
|
1277
|
-
// orphaned-workflow adoption hole: an instance keyed `idempotencyKey = id` can only exist
|
|
1278
|
-
// if the op that seeded it COMMITTED, i.e. its entity + fence exist at that fence — so a
|
|
1279
|
-
// later create can never dedupe onto a rolled-back migrate's workflow and fire at a stale
|
|
1280
|
-
// deadline. The drive runs post-commit; a crash before it leaves a durable `pending`
|
|
1281
|
-
// instance which the ingress re-drive (equal fence, idempotent) or `recover()` converges.
|
|
1282
|
-
// The first run number continues the legacy sequence (`runCount + 1`); the workflow input
|
|
1283
|
-
// is validated EAGERLY at this call (typed `invalid_input` on failure — finding 2), so a
|
|
1284
|
-
// deterministically-rejectable row aborts here before anything is buffered.
|
|
1285
|
-
await ctx.workflows.start(
|
|
1286
|
-
"fire",
|
|
1287
|
-
authorityApply
|
|
1288
|
-
? { id: input.id, fence: input.fence!, generation: input.generation!, fireAt: input.fireAt, run: input.run! }
|
|
1289
|
-
: { id: input.id, fireAt: input.fireAt, initialRun: input.runCount + 1 },
|
|
1290
|
-
{
|
|
1291
|
-
idempotencyKey: authorityApply ? `${input.id}:generation:${input.generation}` : input.id,
|
|
1292
|
-
enlist: "op-commit",
|
|
1293
|
-
},
|
|
1294
|
-
);
|
|
1295
|
-
if (input.fence !== undefined) {
|
|
1296
|
-
await raiseApplyFence(ctx, gate, input.id, input.fence, "create");
|
|
1297
|
-
if (authorityApply) {
|
|
1298
|
-
ctx.messageAuthority.applyFence({
|
|
1299
|
-
timer: { timerId: input.id, message: input.message! },
|
|
1300
|
-
fence: input.fence,
|
|
1301
|
-
updatedAt: ctx.clock.now(),
|
|
1302
|
-
});
|
|
1303
|
-
}
|
|
1304
|
-
}
|
|
1305
|
-
return { timer: timer!, created, fenced: false };
|
|
1306
|
-
},
|
|
1307
|
-
},
|
|
1308
|
-
// ── P2-7 live CANCEL ingress (#1414; SYSTEM-only) — the `migrate` (create) counterpart ─────────
|
|
1309
|
-
// When a legacy timer is CANCELLED via the legacy path (`cancelTimer`), the src ingress mirrors that
|
|
1310
|
-
// cancel into the kernel so the kernel copy STOPS firing — otherwise the mirror stays `scheduled` and
|
|
1311
|
-
// keeps firing autonomously until it independently hits `maxRuns`/`expiresAt` (the #1414 phantom-fire).
|
|
1312
|
-
// Like `migrate`/the drain ops it is drainer-only (`timers:admin`) and never exposed over any transport
|
|
1313
|
-
// (`expose` all false): the ordinary creator-facing `cancel` op requires `timers:write` AND applies the
|
|
1314
|
-
// C2 per-creator filter, so it cannot serve a cross-creator SYSTEM ingress. This op crosses creators by
|
|
1315
|
-
// construction (system-trusted) — it is the src→kernel cancel seam, mirroring how `migrate` is the
|
|
1316
|
-
// create seam.
|
|
1317
|
-
//
|
|
1318
|
-
// Only an ARMED (`scheduled`) timer transitions to `cancelled` (flipping the fire step's cancel-safe
|
|
1319
|
-
// guard, which keys on `status === "scheduled"`). Idempotent + terminal-safe: an already-`cancelled`,
|
|
1320
|
-
// `fired`, or `disabled` timer is a no-op (never resurrects a terminal row). A not-yet-mirrored id (a
|
|
1321
|
-
// create-mirror miss, or a command timer the create-mirror deliberately skipped) never MANUFACTURES a
|
|
1322
|
-
// kernel TIMER row — but a FENCED cancel of an absent entity DOES durably record its fence as the
|
|
1323
|
-
// kernel TOMBSTONE (T-0 hardening), so a stale lower-fence create can never arm the timer afterwards.
|
|
1324
|
-
// An unfenced absent-entity cancel stays a pure best-effort no-op (pre-activation seams).
|
|
1325
|
-
cancelMirror: {
|
|
1326
|
-
kind: "mutation",
|
|
1327
|
-
idempotent: true,
|
|
1328
|
-
grant: TIMERS_DRAIN_GRANT,
|
|
1329
|
-
summary: "Cancel a migrated timer's kernel mirror when its legacy row is cancelled (P2-7; system-only).",
|
|
1330
|
-
emits: ["timers.cancelled"],
|
|
1331
|
-
expose: { http: false, mcp: false, cli: false },
|
|
1332
|
-
input: z.object({
|
|
1333
|
-
id: z.string().min(1),
|
|
1334
|
-
// T-0 apply fence (#1409 hardening): the caller's ingress seq — see `migrate` and `applyFenceSchema`.
|
|
1335
|
-
fence: z.number().int().positive().optional(),
|
|
1336
|
-
}),
|
|
1337
|
-
output: z.object({ cancelled: z.boolean(), fenced: z.boolean() }),
|
|
1338
|
-
handler: async (ctx, input) => {
|
|
1339
|
-
// Stale-fence refusal first (same contract as `migrate`): a cancel that lost to a NEWER applied
|
|
1340
|
-
// intent must not touch the entity nor regress the fence.
|
|
1341
|
-
const gate = input.fence !== undefined ? await applyFenceStore(ctx).get(input.id) : undefined;
|
|
1342
|
-
if (input.fence !== undefined && gate !== undefined && input.fence < gate.fence) {
|
|
1343
|
-
return { cancelled: false, fenced: true };
|
|
1344
|
-
}
|
|
1345
|
-
const store = timerStore(ctx);
|
|
1346
|
-
const timer = await store.get(input.id);
|
|
1347
|
-
if (!timer) {
|
|
1348
|
-
// ABSENT entity: the fenced cancel's fence row IS the durable kernel tombstone — the whole
|
|
1349
|
-
// point of the T-0 hardening (an absent-entity cancel must NOT be a silent no-op).
|
|
1350
|
-
if (input.fence !== undefined) await raiseApplyFence(ctx, gate, input.id, input.fence, "cancel");
|
|
1351
|
-
return { cancelled: false, fenced: false };
|
|
1352
|
-
}
|
|
1353
|
-
if (timer.status !== "scheduled") {
|
|
1354
|
-
if (input.fence !== undefined) await raiseApplyFence(ctx, gate, input.id, input.fence, "cancel");
|
|
1355
|
-
return { cancelled: false, fenced: false };
|
|
1356
|
-
}
|
|
1357
|
-
await store.update(input.id, { status: "cancelled" });
|
|
1358
|
-
await ctx.events.emit("timers.cancelled", { id: input.id });
|
|
1359
|
-
if (input.fence !== undefined) await raiseApplyFence(ctx, gate, input.id, input.fence, "cancel");
|
|
1360
|
-
return { cancelled: true, fenced: false };
|
|
1361
|
-
},
|
|
1362
|
-
},
|
|
1363
|
-
// ── P2-3 relay-outbox drain surface (SYSTEM-only; the src drainer invokes these) ───────────────
|
|
1364
|
-
// The kernel emits the send INTENT (the in-tx outbox row); the src side performs the EFFECT. These
|
|
1365
|
-
// two ops are the narrow, authz-gated seam it does that through — no `kernel/`→`src/` import, and no
|
|
1366
|
-
// ad-hoc entity-store leak. They require the drainer-only `timers:admin` grant and are never exposed
|
|
1367
|
-
// over any transport (`expose` all false), so an ordinary caller can neither read nor settle the
|
|
1368
|
-
// outbox. Marking a row sent is idempotent, which is exactly what makes the at-least-once drain safe
|
|
1369
|
-
// (a crash between `sendMessageService` returning and the mark landing re-drains + re-sends, deduped
|
|
1370
|
-
// by `base:run:N`; the second mark is a no-op).
|
|
1371
|
-
drainRelaySends: {
|
|
1372
|
-
kind: "query",
|
|
1373
|
-
grant: TIMERS_DRAIN_GRANT,
|
|
1374
|
-
summary: "List pending relay-send outbox rows (P2-3 drainer; system-only).",
|
|
1375
|
-
expose: { http: false, mcp: false, cli: false },
|
|
1376
|
-
input: z.object({
|
|
1377
|
-
limit: z.coerce.number().int().min(1).max(200).optional(),
|
|
1378
|
-
cursor: z.string().optional(),
|
|
1379
|
-
}),
|
|
1380
|
-
output: z.object({ items: z.array(relaySendSchema), cursor: z.string().optional() }),
|
|
1381
|
-
handler: async (ctx, input) => {
|
|
1382
|
-
const page = await relaySendStore(ctx).list({
|
|
1383
|
-
where: { status: "pending" },
|
|
1384
|
-
...(input.limit !== undefined ? { limit: input.limit } : {}),
|
|
1385
|
-
...(input.cursor !== undefined ? { cursor: input.cursor } : {}),
|
|
1386
|
-
});
|
|
1387
|
-
return { items: [...page.items], ...(page.cursor ? { cursor: page.cursor } : {}) };
|
|
1388
|
-
},
|
|
1389
|
-
},
|
|
1390
|
-
markRelaySendSent: {
|
|
1391
|
-
kind: "mutation",
|
|
1392
|
-
idempotent: true,
|
|
1393
|
-
grant: TIMERS_DRAIN_GRANT,
|
|
1394
|
-
summary: "Mark a relay-send outbox row sent (P2-3 drainer; system-only).",
|
|
1395
|
-
expose: { http: false, mcp: false, cli: false },
|
|
1396
|
-
input: z.object({ id: z.string(), messageId: z.number().int().optional() }),
|
|
1397
|
-
output: relaySendSchema,
|
|
1398
|
-
handler: async (ctx, input) => {
|
|
1399
|
-
const store = relaySendStore(ctx);
|
|
1400
|
-
const row = await store.get(input.id);
|
|
1401
|
-
if (!row) throw new OpError("not_found", 404, `relay send '${input.id}' was not found`);
|
|
1402
|
-
// Idempotent: a re-mark (a re-drained row already settled, or a duplicate drain) is a no-op.
|
|
1403
|
-
if (row.status === "sent") return row;
|
|
1404
|
-
return store.update(input.id, {
|
|
1405
|
-
status: "sent",
|
|
1406
|
-
...(input.messageId !== undefined ? { sentMessageId: input.messageId } : {}),
|
|
1407
|
-
});
|
|
1408
|
-
},
|
|
1409
|
-
},
|
|
1410
|
-
},
|
|
1411
|
-
});
|