chainlesschain 0.163.6 → 0.163.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/src/assets/web-panel/assets/{AIOps-DDIUofA8.js → AIOps-CavM7W3k.js} +1 -1
- package/src/assets/web-panel/assets/{ActionButton-Bm-5xYRn.js → ActionButton-CUvZGabQ.js} +1 -1
- package/src/assets/web-panel/assets/{Analytics-D5UH4ljI.js → Analytics-DeBnWnbe.js} +3 -3
- package/src/assets/web-panel/assets/{AppLayout-D_ifjOOP.js → AppLayout-DbI5kxvd.js} +4 -4
- package/src/assets/web-panel/assets/{Artifacts-BY1mXdC7.js → Artifacts-D93bigKm.js} +1 -1
- package/src/assets/web-panel/assets/{Audit-Bujr4Ml3.js → Audit-CBNvfUZa.js} +1 -1
- package/src/assets/web-panel/assets/{BackgroundAgents-sioKSTcC.js → BackgroundAgents-CJLyh_YU.js} +1 -1
- package/src/assets/web-panel/assets/{Backup-DDOAK8RK.js → Backup-D_0yoehD.js} +1 -1
- package/src/assets/web-panel/assets/{BaseInput-CtCsVVke.js → BaseInput-TJxmbObW.js} +1 -1
- package/src/assets/web-panel/assets/{Chat-DpNj0vnd.js → Chat-CE0dto1q.js} +5 -5
- package/src/assets/web-panel/assets/{ChatBubbleRenderer-CZta3Ftr.js → ChatBubbleRenderer-CZ5-yYiD.js} +1 -1
- package/src/assets/web-panel/assets/{Checkbox-DoHVNSXq.js → Checkbox-Ck1RVIVo.js} +1 -1
- package/src/assets/web-panel/assets/{Codegen-DlVl7Qmv.js → Codegen-qpoyf6DK.js} +1 -1
- package/src/assets/web-panel/assets/{Col-GIqoPYty.js → Col-DUyPWjIE.js} +1 -1
- package/src/assets/web-panel/assets/{Community-CIAPEVji.js → Community-D_S1BemS.js} +1 -1
- package/src/assets/web-panel/assets/{Compact-BN7JiI_h.js → Compact-BjxeK-Gc.js} +1 -1
- package/src/assets/web-panel/assets/{Compliance-CqUWCM9B.js → Compliance-Dd3CEJsu.js} +1 -1
- package/src/assets/web-panel/assets/{Cowork-A4iPp7A4.js → Cowork-B4QG7-ad.js} +3 -3
- package/src/assets/web-panel/assets/{Cron-B2zeXXWb.js → Cron-BtZnids5.js} +2 -2
- package/src/assets/web-panel/assets/{Crosschain-BR2xmWr8.js → Crosschain-CHPfCBFD.js} +1 -1
- package/src/assets/web-panel/assets/{DID-BzFDjrIv.js → DID-DT8DCEOK.js} +2 -2
- package/src/assets/web-panel/assets/{Dashboard-DKtDo3yA.js → Dashboard-COQab5_U.js} +2 -2
- package/src/assets/web-panel/assets/{Dropdown-RNx43_YG.js → Dropdown-BY1PsQuU.js} +1 -1
- package/src/assets/web-panel/assets/{EmailListRenderer-SwMt4Rcy.js → EmailListRenderer-CsuAuaq3.js} +1 -1
- package/src/assets/web-panel/assets/{FamilyGuardDashboard-CsYebZ4X.js → FamilyGuardDashboard-BqpinD-j.js} +1 -1
- package/src/assets/web-panel/assets/{Federation-BmRKvBTy.js → Federation-D0gkfQFv.js} +1 -1
- package/src/assets/web-panel/assets/{FormItemContext-CVxGTA-0.js → FormItemContext-BqgE-axK.js} +1 -1
- package/src/assets/web-panel/assets/{GenericCardRenderer-x2iXpyB7.js → GenericCardRenderer-D0hJ2KvC.js} +1 -1
- package/src/assets/web-panel/assets/{Git-DfqbkiOP.js → Git-CmMB0uPY.js} +2 -2
- package/src/assets/web-panel/assets/{Governance-RuHyyj5O.js → Governance-BnVEI0Dk.js} +1 -1
- package/src/assets/web-panel/assets/{Inference-Bn7uZ7rI.js → Inference-YTAMihRP.js} +1 -1
- package/src/assets/web-panel/assets/{KnowledgeGraph-DlP7XefZ.js → KnowledgeGraph-0ax7VHop.js} +1 -1
- package/src/assets/web-panel/assets/{Logs-BYNdOiCd.js → Logs-B9TRcokL.js} +2 -2
- package/src/assets/web-panel/assets/MarkdownRenderer-yySN5YNM.js +1 -0
- package/src/assets/web-panel/assets/{Marketplace-JQYmDqM8.js → Marketplace-B6CSuD_H.js} +1 -1
- package/src/assets/web-panel/assets/{McpTools-Cyx4yZ-9.js → McpTools-CjMXvOVU.js} +4 -4
- package/src/assets/web-panel/assets/{Memory-VDuGZTN2.js → Memory-CXRF71U3.js} +2 -2
- package/src/assets/web-panel/assets/{MobileBridge-_yKo97gi.js → MobileBridge-DQgNp_oS.js} +1 -1
- package/src/assets/web-panel/assets/{MobileProjects-CXRXW3q7.js → MobileProjects-DlPwcCbq.js} +1 -1
- package/src/assets/web-panel/assets/{Mtc-acTGtfLH.js → Mtc-6hDTRPe1.js} +5 -5
- package/src/assets/web-panel/assets/{MtcAudit-ClpjXfkY.js → MtcAudit-BlVHU1DS.js} +4 -4
- package/src/assets/web-panel/assets/{Multisig-_6Tn3o96.js → Multisig-DP9pzEbV.js} +3 -3
- package/src/assets/web-panel/assets/{NLProgramming-CK7pbZzP.js → NLProgramming-Djorh8EV.js} +1 -1
- package/src/assets/web-panel/assets/{Notes-Czy6V_P8.js → Notes-oLZ-JkUg.js} +4 -4
- package/src/assets/web-panel/assets/{NotificationSettings-Cqx-UVv3.js → NotificationSettings-Cg288VH3.js} +1 -1
- package/src/assets/web-panel/assets/{OrderTableRenderer-C69WDxGy.js → OrderTableRenderer-Bb1O3Ofr.js} +1 -1
- package/src/assets/web-panel/assets/{Organization-DFnA9vLQ.js → Organization-CBdo-bHs.js} +4 -4
- package/src/assets/web-panel/assets/{Overflow-ReZkk2yT.js → Overflow-qZdnVumx.js} +1 -1
- package/src/assets/web-panel/assets/{P2P-DzjSI0aI.js → P2P-D4kNfGNQ.js} +2 -2
- package/src/assets/web-panel/assets/PdhVaultBrowser-DnlMemM7.js +7 -0
- package/src/assets/web-panel/assets/{Permissions-DhQ5klhR.js → Permissions-DHucHFBJ.js} +3 -3
- package/src/assets/web-panel/assets/{PersonalDataHub-CUL1iHrW.js → PersonalDataHub-BovlOIPR.js} +4 -4
- package/src/assets/web-panel/assets/{Pipeline-DU88iu4N.js → Pipeline-Deru3CQY.js} +1 -1
- package/src/assets/web-panel/assets/{Privacy-Bw-sYfyK.js → Privacy-Bq5oTTbU.js} +1 -1
- package/src/assets/web-panel/assets/{ProjectInit-Bq3Q5y0z.js → ProjectInit-BfzxVWAz.js} +2 -2
- package/src/assets/web-panel/assets/{ProjectSettings-C8spzpho.js → ProjectSettings-CKC6XBtH.js} +2 -2
- package/src/assets/web-panel/assets/Projects-CMolGhae.js +1 -0
- package/src/assets/web-panel/assets/{Providers-Bf4eTP9W.js → Providers-C_jx4TWX.js} +1 -1
- package/src/assets/web-panel/assets/{QrScannerModal-Dme9BDU1.js → QrScannerModal-BdIHeVGC.js} +1 -1
- package/src/assets/web-panel/assets/{QuickAsk-CIXfv1rE.js → QuickAsk-BwZwAcPD.js} +1 -1
- package/src/assets/web-panel/assets/{Recommend-CIxvdeZ-.js → Recommend-BNTt0L-m.js} +1 -1
- package/src/assets/web-panel/assets/{RemoteSession-C6k7MExI.js → RemoteSession-Biyn2-Ro.js} +2 -2
- package/src/assets/web-panel/assets/{Reputation-CISDz2bl.js → Reputation-D6VpG32B.js} +1 -1
- package/src/assets/web-panel/assets/{Row-CdnDxXyf.js → Row-D_74OvQQ.js} +1 -1
- package/src/assets/web-panel/assets/{RssFeed-C-QpRHsl.js → RssFeed-BTr0a-BQ.js} +3 -3
- package/src/assets/web-panel/assets/{Search-DBjaGzSn.js → Search-C3MCFoNU.js} +1 -1
- package/src/assets/web-panel/assets/{Security-CQKukOeE.js → Security-CqW9aJ6s.js} +3 -3
- package/src/assets/web-panel/assets/{Services-DZz86jWb.js → Services-Cw5GNaHF.js} +2 -2
- package/src/assets/web-panel/assets/{Skeleton-jb8RMK0d.js → Skeleton-DZps5qeY.js} +1 -1
- package/src/assets/web-panel/assets/{Skills-CdY76fZG.js → Skills-C6NQ1znT.js} +1 -1
- package/src/assets/web-panel/assets/{Sla-C_CSiZX9.js → Sla-DFP8G2Sh.js} +1 -1
- package/src/assets/web-panel/assets/SpeechSettings-ESt6MB6v.js +1 -0
- package/src/assets/web-panel/assets/{SyncSettings-qvimGO0O.js → SyncSettings-BHnN98aM.js} +2 -2
- package/src/assets/web-panel/assets/{Tasks-Dng5LPJC.js → Tasks-NiSCaOf-.js} +1 -1
- package/src/assets/web-panel/assets/{Templates-Bwqq5Wll.js → Templates-DsKFhW2C.js} +1 -1
- package/src/assets/web-panel/assets/{Tenant-DNWuwP7Z.js → Tenant-CQ0E0ze5.js} +1 -1
- package/src/assets/web-panel/assets/{Terminal-B-6Othpp.js → Terminal-D_rBu_by.js} +2 -2
- package/src/assets/web-panel/assets/{TimelineRenderer-DKpKZfzx.js → TimelineRenderer-D3Vm4Po4.js} +1 -1
- package/src/assets/web-panel/assets/{Tokens-DzfGMd2J.js → Tokens-DNTxRH6q.js} +1 -1
- package/src/assets/web-panel/assets/{Trigger-BXwUi74J.js → Trigger-Dgprz1ms.js} +1 -1
- package/src/assets/web-panel/assets/{Trust-BNzUq4ui.js → Trust-B2rUnVBD.js} +1 -1
- package/src/assets/web-panel/assets/{UkeySign-BYugxww4.js → UkeySign-DEGQNc6_.js} +1 -1
- package/src/assets/web-panel/assets/{VideoEditing-DdauhtSJ.js → VideoEditing-jlsAx3Eh.js} +1 -1
- package/src/assets/web-panel/assets/{Wallet-CSC9bCe_.js → Wallet-8jQih1ij.js} +4 -4
- package/src/assets/web-panel/assets/{WebAuthn-Q8AN3bTO.js → WebAuthn-XLeO6xIB.js} +5 -5
- package/src/assets/web-panel/assets/{WorkflowEditor-d-LBGCuA.js → WorkflowEditor-De_dgDAG.js} +1 -1
- package/src/assets/web-panel/assets/{chat-BLVDCgsZ.js → chat-CI19R_Sk.js} +1 -1
- package/src/assets/web-panel/assets/{colors-BZlPSEHl.js → colors-DsYTOYN3.js} +1 -1
- package/src/assets/web-panel/assets/{compact-item-IWseP6R6.js → compact-item-CM3-t_Qr.js} +1 -1
- package/src/assets/web-panel/assets/{createContext-DnVRV3Ao.js → createContext-BjxlBRZv.js} +1 -1
- package/src/assets/web-panel/assets/devWarning-S1ld2LwU.js +1 -0
- package/src/assets/web-panel/assets/{hasIn-DLQZA_Wn.js → hasIn-RDaWX_4i.js} +1 -1
- package/src/assets/web-panel/assets/{index-BSjilZ9L.js → index-9R6BxAr0.js} +1 -1
- package/src/assets/web-panel/assets/{index-B_N4-Gq9.js → index-B-Z8eFUJ.js} +1 -1
- package/src/assets/web-panel/assets/{index-Dy0R5ZED.js → index-B0PMlbMs.js} +1 -1
- package/src/assets/web-panel/assets/{index-C1ifZtXo.js → index-B5REnKt8.js} +1 -1
- package/src/assets/web-panel/assets/{index-CINJQEj6.js → index-B7WBt_4T.js} +1 -1
- package/src/assets/web-panel/assets/{index-DocEhinm.js → index-B9O9YyWE.js} +1 -1
- package/src/assets/web-panel/assets/{index-DjWnONdy.js → index-BCnZyIMA.js} +1 -1
- package/src/assets/web-panel/assets/{index-hwVDeoC0.js → index-BGqqJfL0.js} +1 -1
- package/src/assets/web-panel/assets/{index-Bbj2NS_I.js → index-BHNYPO0b.js} +3 -3
- package/src/assets/web-panel/assets/{index-C5dgnilC.js → index-BKJr-TZp.js} +1 -1
- package/src/assets/web-panel/assets/{index-CT9vVN-X.js → index-BN6UPD2E.js} +1 -1
- package/src/assets/web-panel/assets/{index-DpVrK6fr.js → index-Bi46Xu4u.js} +1 -1
- package/src/assets/web-panel/assets/{index-CpBaJ3TD.js → index-BlT2VAEK.js} +1 -1
- package/src/assets/web-panel/assets/{index-1jeAudHd.js → index-C0yx-jtQ.js} +1 -1
- package/src/assets/web-panel/assets/{index-DOoQpuQY.js → index-C4gkzxkw.js} +1 -1
- package/src/assets/web-panel/assets/{index-DZpYeqrh.js → index-CVQMeiwH.js} +1 -1
- package/src/assets/web-panel/assets/{index-BhA3QEni.js → index-Cd3euP_4.js} +1 -1
- package/src/assets/web-panel/assets/{index-N_GoWZX4.js → index-Cd9OJhHG.js} +1 -1
- package/src/assets/web-panel/assets/{index-BOZJBTWZ.js → index-CiECJi-Q.js} +1 -1
- package/src/assets/web-panel/assets/{index-CarMTIyQ.js → index-CjTrRt13.js} +1 -1
- package/src/assets/web-panel/assets/{index-CZDekj7T.js → index-CnHH7bnD.js} +1 -1
- package/src/assets/web-panel/assets/{index-M7wxfAbm.js → index-Cpqm1qfU.js} +1 -1
- package/src/assets/web-panel/assets/{index-Bxp7uDD8.js → index-CyOGN6a5.js} +1 -1
- package/src/assets/web-panel/assets/{index-DalA0TKC.js → index-D0zcy54x.js} +1 -1
- package/src/assets/web-panel/assets/{index-COev0W2g.js → index-D6sm5-Xq.js} +1 -1
- package/src/assets/web-panel/assets/{index-DUq0c7gq.js → index-DIUnNOjG.js} +1 -1
- package/src/assets/web-panel/assets/{index-BZs5bBnk.js → index-DX7tuWVJ.js} +1 -1
- package/src/assets/web-panel/assets/{index-CbQNL0ue.js → index-DXXtGMUJ.js} +1 -1
- package/src/assets/web-panel/assets/{index-DTbxp4UH.js → index-DcPFB8qJ.js} +1 -1
- package/src/assets/web-panel/assets/{index-YSw_gxlZ.js → index-DoQA0-52.js} +1 -1
- package/src/assets/web-panel/assets/{index-D_LQodeJ.js → index-Drlnwr8F.js} +1 -1
- package/src/assets/web-panel/assets/index-Drz6yq0m.js +1 -0
- package/src/assets/web-panel/assets/{index-jWb70ZgZ.js → index-Dsij_KsN.js} +1 -1
- package/src/assets/web-panel/assets/{index-DB0MJgF_.js → index-Dxge-xWl.js} +1 -1
- package/src/assets/web-panel/assets/{index-BEu2lMBy.js → index-DyIWssPC.js} +1 -1
- package/src/assets/web-panel/assets/{index-C2FqfMvU.js → index-IeiL9RKc.js} +1 -1
- package/src/assets/web-panel/assets/{index-Dkwsi1Kn.js → index-nAH87mQP.js} +1 -1
- package/src/assets/web-panel/assets/{index-gt8bNefW.js → index-pPL66J85.js} +1 -1
- package/src/assets/web-panel/assets/index-uWiUs7S_.js +1 -0
- package/src/assets/web-panel/assets/{initDefaultProps-Co67Kqmb.js → initDefaultProps-BTL88of4.js} +1 -1
- package/src/assets/web-panel/assets/{motion-BMOxQSF_.js → motion-BUB2pXwB.js} +1 -1
- package/src/assets/web-panel/assets/{move-C5fWDw-N.js → move-egCkkOMm.js} +1 -1
- package/src/assets/web-panel/assets/{mtc-parser-BJevpL0e.js → mtc-parser-DAWhZC8c.js} +1 -1
- package/src/assets/web-panel/assets/{omit-BCzNU8s6.js → omit-CvOxe_6d.js} +1 -1
- package/src/assets/web-panel/assets/{pickAttrs-CybWvDPk.js → pickAttrs-DZUSLZ5Z.js} +1 -1
- package/src/assets/web-panel/assets/{placementArrow-BMWS9HnW.js → placementArrow-BERxGNMY.js} +1 -1
- package/src/assets/web-panel/assets/{responsiveObserve-BoL2UnGY.js → responsiveObserve-DZuDfZ9n.js} +1 -1
- package/src/assets/web-panel/assets/{slide-DWNfXraU.js → slide-DO756sgI.js} +1 -1
- package/src/assets/web-panel/assets/{statusUtils-BquTM3tX.js → statusUtils-BRQlIz28.js} +1 -1
- package/src/assets/web-panel/assets/{styleChecker-BEkCleim.js → styleChecker-DCSfdybl.js} +1 -1
- package/src/assets/web-panel/assets/{useFlexGapSupport-Oecnq7EA.js → useFlexGapSupport-LD8dWVve.js} +1 -1
- package/src/assets/web-panel/assets/{useFs-CkDffJ4F.js → useFs-BX9uW2LK.js} +1 -1
- package/src/assets/web-panel/assets/{usePersonalDataHub-qbvKXesB.js → usePersonalDataHub-BPSlPuHU.js} +1 -1
- package/src/assets/web-panel/assets/{vnode-C6v8n6mA.js → vnode-BmOILC9V.js} +1 -1
- package/src/assets/web-panel/assets/{zoom-4CTeFILV.js → zoom-1iKowKGg.js} +1 -1
- package/src/assets/web-panel/index.html +1 -1
- package/src/command-help-index.json +5 -5
- package/src/commands/agent.js +90 -1
- package/src/commands/automation.js +103 -1
- package/src/commands/chat.js +11 -2
- package/src/commands/compact.js +106 -23
- package/src/commands/loop.js +28 -10
- package/src/commands/routine.js +1 -1
- package/src/commands/scheduler-daemon.js +359 -2
- package/src/commands/session-observability.js +305 -0
- package/src/commands/session.js +38 -0
- package/src/commands/team-merge-review.js +937 -0
- package/src/commands/team.js +2 -0
- package/src/data/changelog.json +27 -1
- package/src/gateways/ws/ws-agent-handler.js +439 -5
- package/src/harness/jsonl-session-store.js +463 -125
- package/src/harness/prompt-compressor.js +86 -13
- package/src/harness/provider-backed-compaction.js +100 -27
- package/src/lib/advisor-runtime.js +3 -1
- package/src/lib/agent-schedule-store.js +549 -28
- package/src/lib/agent-team/team-adjudication.js +40 -1
- package/src/lib/agent-team/team-merge-review-store.js +765 -0
- package/src/lib/agent-team/team-merge-review-transaction.js +1223 -0
- package/src/lib/agent-team/team-merge-review.js +1847 -0
- package/src/lib/automation-center-incidents.js +314 -0
- package/src/lib/automation-center-runtime.js +448 -0
- package/src/lib/automation-center.js +182 -38
- package/src/lib/automation-engine.js +535 -0
- package/src/lib/automation-execution-authority.js +241 -12
- package/src/lib/automation-execution-incident.js +737 -0
- package/src/lib/background-agent-supervisor.js +793 -215
- package/src/lib/background-command-argv.js +24 -0
- package/src/lib/background-interaction-journal.js +361 -52
- package/src/lib/background-interaction-resolver.js +32 -0
- package/src/lib/background-launch-profile.js +2 -0
- package/src/lib/causal-observability.js +2661 -0
- package/src/lib/chat-session-admission.js +76 -0
- package/src/lib/cost-budget.js +5 -7
- package/src/lib/cowork/ab-comparator-cli.js +11 -0
- package/src/lib/cowork/debate-review-cli.js +12 -0
- package/src/lib/cowork-adapter.js +159 -38
- package/src/lib/cowork-cron.js +442 -15
- package/src/lib/cowork-task-runner.js +112 -17
- package/src/lib/delivery-coordinator.js +142 -0
- package/src/lib/direct-model-usage.js +121 -0
- package/src/lib/file-lines.js +183 -47
- package/src/lib/llm-pricing.js +12 -17
- package/src/lib/loop.js +26 -2
- package/src/lib/routine-store.js +380 -21
- package/src/lib/runtime-usage-ledger.js +182 -0
- package/src/lib/scheduler-kernel/agenda-adapter.js +325 -10
- package/src/lib/scheduler-kernel/automation-adapter.js +514 -18
- package/src/lib/scheduler-kernel/automation-event-adapter.js +123 -11
- package/src/lib/scheduler-kernel/contract.js +91 -0
- package/src/lib/scheduler-kernel/cowork-cron-adapter.js +297 -4
- package/src/lib/scheduler-kernel/loop-adapter.js +547 -6
- package/src/lib/scheduler-kernel/migration-admin.js +346 -0
- package/src/lib/scheduler-kernel/migration-rollback-admin.js +932 -0
- package/src/lib/scheduler-kernel/routine-adapter.js +400 -20
- package/src/lib/scheduler-kernel/runtime-control-capabilities.js +31 -0
- package/src/lib/scheduler-kernel/runtime.js +237 -2
- package/src/lib/scheduler-kernel/source-locator-path.js +142 -0
- package/src/lib/scheduler-kernel/store.js +3957 -248
- package/src/lib/session-host-lease.js +171 -2
- package/src/lib/session-host-snapshot.js +14 -3
- package/src/lib/session-persistence-failure.js +1 -0
- package/src/lib/session-tail.js +179 -34
- package/src/lib/session-transcript-structure.js +96 -0
- package/src/lib/sub-agent-context.js +328 -22
- package/src/lib/ws-agent-handler.js +4 -1
- package/src/repl/agent-repl.js +732 -91
- package/src/repl/chat-repl.js +5 -1
- package/src/runtime/agent-core.js +1282 -207
- package/src/runtime/fallback-model.js +5 -0
- package/src/runtime/headless-runner.js +324 -82
- package/src/runtime/headless-stream.js +232 -26
- package/src/runtime/mcp-config.js +81 -10
- package/src/runtime/policies/agent-policy.js +4 -0
- package/src/workers/background-agent-worker.js +403 -96
- package/src/assets/web-panel/assets/MarkdownRenderer-aMtgN-R6.js +0 -1
- package/src/assets/web-panel/assets/PdhVaultBrowser-DKTf5xIP.js +0 -7
- package/src/assets/web-panel/assets/Projects-Dd-xuPIc.js +0 -1
- package/src/assets/web-panel/assets/SpeechSettings-C7tT9iad.js +0 -1
- package/src/assets/web-panel/assets/devWarning-Jzu62CLC.js +0 -1
- package/src/assets/web-panel/assets/index-Zb2MeDSy.js +0 -1
- package/src/assets/web-panel/assets/index-fikqml4k.js +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { existsSync } from "node:fs";
|
|
3
3
|
import { createRequire } from "node:module";
|
|
4
4
|
import { dirname, join, resolve } from "node:path";
|
|
@@ -6,6 +6,8 @@ import { getHomeDir } from "../paths.js";
|
|
|
6
6
|
import { ensurePrivateDirectory, ensurePrivateFile } from "../secure-fs.js";
|
|
7
7
|
import {
|
|
8
8
|
OCCURRENCE_STATUS,
|
|
9
|
+
RUNTIME_CONTROL_SAFE_POINTS,
|
|
10
|
+
RUNTIME_PAUSE_RESUME,
|
|
9
11
|
SchedulerKernelError,
|
|
10
12
|
canonicalJson,
|
|
11
13
|
deriveOccurrenceIdentity,
|
|
@@ -16,30 +18,86 @@ import {
|
|
|
16
18
|
normalizeIdentifier,
|
|
17
19
|
normalizeJson,
|
|
18
20
|
normalizeMaxAttempts,
|
|
21
|
+
normalizeRuntimeControlCapability,
|
|
19
22
|
} from "./contract.js";
|
|
23
|
+
import { isCanonicalSchedulerSourcePath } from "./source-locator-path.js";
|
|
20
24
|
|
|
21
25
|
const requireCjs = createRequire(import.meta.url);
|
|
22
26
|
|
|
27
|
+
const STORAGE_FAILURE_CODES = Object.freeze([
|
|
28
|
+
"ENOSPC",
|
|
29
|
+
"EROFS",
|
|
30
|
+
"EIO",
|
|
31
|
+
"SQLITE_FULL",
|
|
32
|
+
"SQLITE_IOERR",
|
|
33
|
+
"SQLITE_READONLY",
|
|
34
|
+
]);
|
|
35
|
+
const STORAGE_FAILURE_CODE_SET = new Set(STORAGE_FAILURE_CODES);
|
|
36
|
+
|
|
23
37
|
export const SCHEDULER_APPLICATION_ID = 0x4343534b; // "CCSK"
|
|
24
|
-
export const SCHEDULER_STORE_SCHEMA_VERSION =
|
|
38
|
+
export const SCHEDULER_STORE_SCHEMA_VERSION = 6;
|
|
25
39
|
export const DEFAULT_BUSY_TIMEOUT_MS = 5_000;
|
|
26
40
|
export const MAX_LEASE_MS = 24 * 60 * 60 * 1_000;
|
|
27
41
|
export const MIN_AUTHORITY_WINDOW_MS = 60_000;
|
|
28
42
|
export const MAX_AUTHORITY_WINDOW_MS = 31 * 24 * 60 * 60 * 1_000;
|
|
29
43
|
export const MAX_AUTHORITY_BUDGET = 1_000_000;
|
|
44
|
+
export const MAX_RUNTIME_CHECKPOINT_BYTES = 64 * 1024;
|
|
30
45
|
export const DEFAULT_AUTHORITY_WINDOW_MS = 24 * 60 * 60 * 1_000;
|
|
31
46
|
export const DEFAULT_AUTHORITY_MAX_RUNS = 100_000;
|
|
32
47
|
export const DEFAULT_AUTHORITY_MAX_UNITS = 100_000;
|
|
48
|
+
export const MAX_RUNTIME_CONTROL_OCCURRENCES = 200;
|
|
49
|
+
export const RUNTIME_CONTROL_OCCURRENCE_STATUSES = Object.freeze([
|
|
50
|
+
"running",
|
|
51
|
+
"pause_requested",
|
|
52
|
+
"paused",
|
|
53
|
+
]);
|
|
54
|
+
export const RUNTIME_CONTROL_JOB_KINDS = Object.freeze([
|
|
55
|
+
"agenda",
|
|
56
|
+
"automation",
|
|
57
|
+
"automation-event",
|
|
58
|
+
"cowork-cron",
|
|
59
|
+
"loop-iteration",
|
|
60
|
+
"routine",
|
|
61
|
+
]);
|
|
62
|
+
const RUNTIME_CONTROL_OCCURRENCE_STATUS_SET = new Set(
|
|
63
|
+
RUNTIME_CONTROL_OCCURRENCE_STATUSES,
|
|
64
|
+
);
|
|
65
|
+
const RUNTIME_CONTROL_JOB_KIND_SET = new Set(RUNTIME_CONTROL_JOB_KINDS);
|
|
66
|
+
export const SCHEDULER_ADJUDICATION_AUTHORITY = "local-operator";
|
|
67
|
+
export const SCHEDULER_ADJUDICATION_DECISIONS = Object.freeze({
|
|
68
|
+
CONFIRMED_APPLIED: "confirmed_applied",
|
|
69
|
+
CONFIRMED_NOT_APPLIED: "confirmed_not_applied",
|
|
70
|
+
});
|
|
71
|
+
export const SCHEDULER_MIGRATION_DOMAINS = Object.freeze([
|
|
72
|
+
"agenda",
|
|
73
|
+
"cowork-cron",
|
|
74
|
+
"routine",
|
|
75
|
+
"automation",
|
|
76
|
+
"loop-iteration",
|
|
77
|
+
]);
|
|
78
|
+
export const SCHEDULER_MIGRATION_STATES = Object.freeze({
|
|
79
|
+
PREPARED: "prepared",
|
|
80
|
+
APPLIED: "applied",
|
|
81
|
+
VERIFIED: "verified",
|
|
82
|
+
RETIRING: "retiring",
|
|
83
|
+
RETIRED: "retired",
|
|
84
|
+
ROLLING_BACK: "rolling_back",
|
|
85
|
+
ROLLED_BACK: "rolled_back",
|
|
86
|
+
});
|
|
33
87
|
|
|
34
88
|
const MIGRATION_V1_NAME = "scheduler-kernel-v1";
|
|
35
89
|
const MIGRATION_V2_NAME = "scheduler-kernel-authority-v2";
|
|
90
|
+
const MIGRATION_V3_NAME = "scheduler-kernel-adjudication-v3";
|
|
91
|
+
const MIGRATION_V4_NAME = "scheduler-kernel-domain-migration-v4";
|
|
92
|
+
const MIGRATION_V5_NAME = "scheduler-kernel-source-locator-v5";
|
|
93
|
+
const MIGRATION_V6_NAME = "scheduler-kernel-runtime-control-v6";
|
|
36
94
|
const V1_USER_TABLES = Object.freeze([
|
|
37
95
|
"events",
|
|
38
96
|
"jobs",
|
|
39
97
|
"migrations",
|
|
40
98
|
"occurrences",
|
|
41
99
|
]);
|
|
42
|
-
const
|
|
100
|
+
const V2_USER_TABLES = Object.freeze([
|
|
43
101
|
"events",
|
|
44
102
|
"jobs",
|
|
45
103
|
"migrations",
|
|
@@ -48,6 +106,21 @@ const USER_TABLES = Object.freeze([
|
|
|
48
106
|
"scheduler_authority_reservations",
|
|
49
107
|
"scheduler_authority_usage",
|
|
50
108
|
]);
|
|
109
|
+
const V3_USER_TABLES = Object.freeze([
|
|
110
|
+
...V2_USER_TABLES,
|
|
111
|
+
"scheduler_occurrence_adjudications",
|
|
112
|
+
]);
|
|
113
|
+
const V4_V5_USER_TABLES = Object.freeze([
|
|
114
|
+
...V2_USER_TABLES,
|
|
115
|
+
"scheduler_domain_migration_entries",
|
|
116
|
+
"scheduler_domain_migrations",
|
|
117
|
+
"scheduler_occurrence_adjudications",
|
|
118
|
+
]);
|
|
119
|
+
const USER_TABLES = Object.freeze([
|
|
120
|
+
...V4_V5_USER_TABLES,
|
|
121
|
+
"scheduler_occurrence_controls",
|
|
122
|
+
"scheduler_occurrence_retries",
|
|
123
|
+
]);
|
|
51
124
|
|
|
52
125
|
export const MIGRATION_V1_SQL = `
|
|
53
126
|
CREATE TABLE migrations (
|
|
@@ -209,6 +282,239 @@ export const MIGRATION_V2_CHECKSUM = createHash("sha256")
|
|
|
209
282
|
.update("\0scheduler-authority-v2-backfill-v1", "utf8")
|
|
210
283
|
.digest("hex");
|
|
211
284
|
|
|
285
|
+
export const MIGRATION_V3_SQL = `
|
|
286
|
+
CREATE TABLE scheduler_occurrence_adjudications (
|
|
287
|
+
occurrence_id TEXT PRIMARY KEY NOT NULL
|
|
288
|
+
REFERENCES occurrences(occurrence_id) ON DELETE RESTRICT,
|
|
289
|
+
job_id TEXT NOT NULL REFERENCES jobs(job_id) ON DELETE RESTRICT,
|
|
290
|
+
request_id TEXT NOT NULL UNIQUE,
|
|
291
|
+
decision TEXT NOT NULL CHECK (
|
|
292
|
+
decision IN ('confirmed_applied', 'confirmed_not_applied')
|
|
293
|
+
),
|
|
294
|
+
authority TEXT NOT NULL CHECK (authority = 'local-operator'),
|
|
295
|
+
evidence_digest TEXT NOT NULL,
|
|
296
|
+
expected_attempt INTEGER NOT NULL CHECK (expected_attempt >= 1),
|
|
297
|
+
expected_fence INTEGER NOT NULL CHECK (expected_fence >= 1),
|
|
298
|
+
reason_digest TEXT NOT NULL,
|
|
299
|
+
operator_digest TEXT NOT NULL,
|
|
300
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'applied')),
|
|
301
|
+
created_at INTEGER NOT NULL CHECK (created_at >= 0),
|
|
302
|
+
applied_at INTEGER,
|
|
303
|
+
retry_settled_at INTEGER,
|
|
304
|
+
retry_outcome_json TEXT CHECK (
|
|
305
|
+
retry_outcome_json IS NULL OR json_valid(retry_outcome_json)
|
|
306
|
+
),
|
|
307
|
+
CHECK (
|
|
308
|
+
(status = 'pending' AND applied_at IS NULL)
|
|
309
|
+
OR
|
|
310
|
+
(status = 'applied' AND applied_at IS NOT NULL)
|
|
311
|
+
),
|
|
312
|
+
CHECK (
|
|
313
|
+
(retry_settled_at IS NULL AND retry_outcome_json IS NULL)
|
|
314
|
+
OR
|
|
315
|
+
(status = 'applied' AND retry_settled_at IS NOT NULL
|
|
316
|
+
AND retry_outcome_json IS NOT NULL)
|
|
317
|
+
)
|
|
318
|
+
);
|
|
319
|
+
|
|
320
|
+
CREATE INDEX scheduler_adjudications_status
|
|
321
|
+
ON scheduler_occurrence_adjudications(status, created_at, occurrence_id);
|
|
322
|
+
`;
|
|
323
|
+
|
|
324
|
+
export const MIGRATION_V3_CHECKSUM = createHash("sha256")
|
|
325
|
+
.update(MIGRATION_V3_SQL.trim().replace(/\r\n/g, "\n"), "utf8")
|
|
326
|
+
.digest("hex");
|
|
327
|
+
|
|
328
|
+
export const MIGRATION_V4_SQL = `
|
|
329
|
+
CREATE TABLE scheduler_domain_migrations (
|
|
330
|
+
migration_id TEXT PRIMARY KEY NOT NULL,
|
|
331
|
+
manifest_digest TEXT NOT NULL UNIQUE,
|
|
332
|
+
manifest_json TEXT NOT NULL CHECK (json_valid(manifest_json)),
|
|
333
|
+
state TEXT NOT NULL CHECK (
|
|
334
|
+
state IN (
|
|
335
|
+
'prepared', 'applied', 'verified', 'retiring', 'retired',
|
|
336
|
+
'rolling_back', 'rolled_back'
|
|
337
|
+
)
|
|
338
|
+
),
|
|
339
|
+
entry_count INTEGER NOT NULL CHECK (entry_count BETWEEN 1 AND 1000),
|
|
340
|
+
created_at INTEGER NOT NULL CHECK (created_at >= 0),
|
|
341
|
+
updated_at INTEGER NOT NULL CHECK (updated_at >= 0),
|
|
342
|
+
completed_at INTEGER,
|
|
343
|
+
last_error_json TEXT CHECK (
|
|
344
|
+
last_error_json IS NULL OR json_valid(last_error_json)
|
|
345
|
+
),
|
|
346
|
+
CHECK (
|
|
347
|
+
(state IN ('retired', 'rolled_back') AND completed_at IS NOT NULL)
|
|
348
|
+
OR
|
|
349
|
+
(state NOT IN ('retired', 'rolled_back') AND completed_at IS NULL)
|
|
350
|
+
)
|
|
351
|
+
);
|
|
352
|
+
|
|
353
|
+
CREATE TABLE scheduler_domain_migration_entries (
|
|
354
|
+
migration_id TEXT NOT NULL
|
|
355
|
+
REFERENCES scheduler_domain_migrations(migration_id)
|
|
356
|
+
ON DELETE RESTRICT,
|
|
357
|
+
entry_id TEXT NOT NULL,
|
|
358
|
+
domain TEXT NOT NULL CHECK (
|
|
359
|
+
domain IN ('agenda', 'cowork-cron', 'routine', 'automation', 'loop-iteration')
|
|
360
|
+
),
|
|
361
|
+
source_id TEXT NOT NULL,
|
|
362
|
+
source_scope_digest TEXT NOT NULL,
|
|
363
|
+
source_digest TEXT NOT NULL,
|
|
364
|
+
target_job_id TEXT NOT NULL,
|
|
365
|
+
target_job_digest TEXT NOT NULL,
|
|
366
|
+
rollback_strategy TEXT NOT NULL CHECK (
|
|
367
|
+
rollback_strategy IN ('restore', 'disable')
|
|
368
|
+
),
|
|
369
|
+
state TEXT NOT NULL CHECK (
|
|
370
|
+
state IN (
|
|
371
|
+
'prepared', 'applied', 'verified', 'retiring', 'retired',
|
|
372
|
+
'rollback_target_disabled', 'source_restored', 'rolled_back'
|
|
373
|
+
)
|
|
374
|
+
),
|
|
375
|
+
target_action TEXT CHECK (
|
|
376
|
+
target_action IS NULL OR target_action IN ('created', 'updated', 'reused')
|
|
377
|
+
),
|
|
378
|
+
target_before_json TEXT CHECK (
|
|
379
|
+
target_before_json IS NULL OR json_valid(target_before_json)
|
|
380
|
+
),
|
|
381
|
+
target_applied_revision INTEGER CHECK (target_applied_revision >= 1),
|
|
382
|
+
target_applied_at INTEGER CHECK (target_applied_at >= 0),
|
|
383
|
+
target_occurrence_count_before INTEGER CHECK (
|
|
384
|
+
target_occurrence_count_before >= 0
|
|
385
|
+
),
|
|
386
|
+
target_execution_event_count_before INTEGER CHECK (
|
|
387
|
+
target_execution_event_count_before >= 0
|
|
388
|
+
),
|
|
389
|
+
target_rollback_revision INTEGER CHECK (target_rollback_revision >= 1),
|
|
390
|
+
retirement_token TEXT,
|
|
391
|
+
source_retirement_digest TEXT,
|
|
392
|
+
source_restored_digest TEXT,
|
|
393
|
+
created_at INTEGER NOT NULL CHECK (created_at >= 0),
|
|
394
|
+
updated_at INTEGER NOT NULL CHECK (updated_at >= 0),
|
|
395
|
+
PRIMARY KEY (migration_id, entry_id),
|
|
396
|
+
UNIQUE (migration_id, domain, source_scope_digest, source_id),
|
|
397
|
+
UNIQUE (migration_id, target_job_id),
|
|
398
|
+
CHECK (
|
|
399
|
+
(state = 'prepared' AND target_action IS NULL
|
|
400
|
+
AND target_applied_revision IS NULL AND target_applied_at IS NULL
|
|
401
|
+
AND target_occurrence_count_before IS NULL
|
|
402
|
+
AND target_execution_event_count_before IS NULL)
|
|
403
|
+
OR
|
|
404
|
+
(state <> 'prepared' AND state <> 'rolled_back'
|
|
405
|
+
AND target_action IS NOT NULL AND target_applied_revision IS NOT NULL
|
|
406
|
+
AND target_applied_at IS NOT NULL
|
|
407
|
+
AND target_occurrence_count_before IS NOT NULL
|
|
408
|
+
AND target_execution_event_count_before IS NOT NULL)
|
|
409
|
+
OR
|
|
410
|
+
(state = 'rolled_back')
|
|
411
|
+
),
|
|
412
|
+
CHECK (
|
|
413
|
+
(target_action IN ('updated', 'reused') AND target_before_json IS NOT NULL)
|
|
414
|
+
OR
|
|
415
|
+
(target_action = 'created' AND target_before_json IS NULL)
|
|
416
|
+
OR
|
|
417
|
+
(target_action IS NULL AND target_before_json IS NULL)
|
|
418
|
+
),
|
|
419
|
+
CHECK (
|
|
420
|
+
state NOT IN ('retiring', 'retired')
|
|
421
|
+
OR retirement_token IS NOT NULL
|
|
422
|
+
),
|
|
423
|
+
CHECK (
|
|
424
|
+
state <> 'retired'
|
|
425
|
+
OR source_retirement_digest IS NOT NULL
|
|
426
|
+
)
|
|
427
|
+
);
|
|
428
|
+
|
|
429
|
+
CREATE INDEX scheduler_domain_migrations_state
|
|
430
|
+
ON scheduler_domain_migrations(state, updated_at, migration_id);
|
|
431
|
+
CREATE INDEX scheduler_domain_migration_entries_state
|
|
432
|
+
ON scheduler_domain_migration_entries(state, updated_at, migration_id, entry_id);
|
|
433
|
+
CREATE UNIQUE INDEX scheduler_domain_migration_active_source
|
|
434
|
+
ON scheduler_domain_migration_entries(domain, source_scope_digest, source_id)
|
|
435
|
+
WHERE state <> 'rolled_back';
|
|
436
|
+
CREATE UNIQUE INDEX scheduler_domain_migration_active_target
|
|
437
|
+
ON scheduler_domain_migration_entries(target_job_id)
|
|
438
|
+
WHERE state <> 'rolled_back';
|
|
439
|
+
`;
|
|
440
|
+
|
|
441
|
+
export const MIGRATION_V4_CHECKSUM = createHash("sha256")
|
|
442
|
+
.update(MIGRATION_V4_SQL.trim().replace(/\r\n/g, "\n"), "utf8")
|
|
443
|
+
.digest("hex");
|
|
444
|
+
|
|
445
|
+
export const MIGRATION_V5_SQL = `
|
|
446
|
+
ALTER TABLE scheduler_domain_migration_entries
|
|
447
|
+
ADD COLUMN source_locator_json TEXT CHECK (
|
|
448
|
+
source_locator_json IS NULL OR json_valid(source_locator_json)
|
|
449
|
+
);
|
|
450
|
+
`;
|
|
451
|
+
|
|
452
|
+
export const MIGRATION_V5_CHECKSUM = createHash("sha256")
|
|
453
|
+
.update(MIGRATION_V5_SQL.trim().replace(/\r\n/g, "\n"), "utf8")
|
|
454
|
+
.digest("hex");
|
|
455
|
+
|
|
456
|
+
export const MIGRATION_V6_SQL = `
|
|
457
|
+
CREATE TABLE scheduler_occurrence_controls (
|
|
458
|
+
occurrence_id TEXT PRIMARY KEY NOT NULL
|
|
459
|
+
REFERENCES occurrences(occurrence_id) ON DELETE RESTRICT,
|
|
460
|
+
capability_json TEXT NOT NULL CHECK (json_valid(capability_json)),
|
|
461
|
+
capability_digest TEXT NOT NULL CHECK (
|
|
462
|
+
capability_digest GLOB 'sha256:[0-9a-f]*'
|
|
463
|
+
AND length(capability_digest) = 71
|
|
464
|
+
),
|
|
465
|
+
state TEXT NOT NULL CHECK (
|
|
466
|
+
state IN ('pause_requested', 'paused', 'resumed', 'terminal')
|
|
467
|
+
),
|
|
468
|
+
revision INTEGER NOT NULL CHECK (revision >= 1),
|
|
469
|
+
pause_request_id TEXT NOT NULL,
|
|
470
|
+
resume_request_id TEXT UNIQUE,
|
|
471
|
+
expected_fence INTEGER NOT NULL CHECK (expected_fence >= 1),
|
|
472
|
+
checkpoint_json TEXT CHECK (
|
|
473
|
+
checkpoint_json IS NULL OR json_valid(checkpoint_json)
|
|
474
|
+
),
|
|
475
|
+
requested_at INTEGER NOT NULL CHECK (requested_at >= 0),
|
|
476
|
+
paused_at INTEGER CHECK (paused_at >= 0),
|
|
477
|
+
resumed_at INTEGER CHECK (resumed_at >= 0),
|
|
478
|
+
terminal_at INTEGER CHECK (terminal_at >= 0),
|
|
479
|
+
updated_at INTEGER NOT NULL CHECK (updated_at >= 0),
|
|
480
|
+
CHECK (
|
|
481
|
+
(state = 'pause_requested' AND paused_at IS NULL
|
|
482
|
+
AND resumed_at IS NULL AND terminal_at IS NULL)
|
|
483
|
+
OR
|
|
484
|
+
(state = 'paused' AND paused_at IS NOT NULL
|
|
485
|
+
AND resumed_at IS NULL AND terminal_at IS NULL)
|
|
486
|
+
OR
|
|
487
|
+
(state = 'resumed' AND paused_at IS NOT NULL
|
|
488
|
+
AND resumed_at IS NOT NULL AND terminal_at IS NULL)
|
|
489
|
+
OR
|
|
490
|
+
(state = 'terminal' AND terminal_at IS NOT NULL)
|
|
491
|
+
)
|
|
492
|
+
);
|
|
493
|
+
|
|
494
|
+
CREATE UNIQUE INDEX scheduler_occurrence_controls_pause_request
|
|
495
|
+
ON scheduler_occurrence_controls(pause_request_id);
|
|
496
|
+
CREATE INDEX scheduler_occurrence_controls_state
|
|
497
|
+
ON scheduler_occurrence_controls(state, updated_at, occurrence_id);
|
|
498
|
+
|
|
499
|
+
CREATE TABLE scheduler_occurrence_retries (
|
|
500
|
+
occurrence_id TEXT NOT NULL
|
|
501
|
+
REFERENCES occurrences(occurrence_id) ON DELETE RESTRICT,
|
|
502
|
+
request_id TEXT NOT NULL UNIQUE,
|
|
503
|
+
expected_fence INTEGER NOT NULL CHECK (expected_fence >= 1),
|
|
504
|
+
expected_error_code TEXT NOT NULL,
|
|
505
|
+
created_at INTEGER NOT NULL CHECK (created_at >= 0),
|
|
506
|
+
claimed_at INTEGER CHECK (claimed_at >= 0),
|
|
507
|
+
PRIMARY KEY (occurrence_id, request_id)
|
|
508
|
+
);
|
|
509
|
+
|
|
510
|
+
CREATE INDEX scheduler_occurrence_retries_pending
|
|
511
|
+
ON scheduler_occurrence_retries(occurrence_id, claimed_at, expected_fence);
|
|
512
|
+
`;
|
|
513
|
+
|
|
514
|
+
export const MIGRATION_V6_CHECKSUM = createHash("sha256")
|
|
515
|
+
.update(MIGRATION_V6_SQL.trim().replace(/\r\n/g, "\n"), "utf8")
|
|
516
|
+
.digest("hex");
|
|
517
|
+
|
|
212
518
|
// Fingerprint of the normalized sqlite_master catalog produced by the v1 DDL.
|
|
213
519
|
// Unlike the migration-source checksum, this also detects added triggers/views
|
|
214
520
|
// and constraint/foreign-key changes that preserve the visible column list.
|
|
@@ -216,6 +522,14 @@ export const SCHEMA_V1_FINGERPRINT =
|
|
|
216
522
|
"cf244f675ac7430683f67046b57524eb6c14baa45d6463ed3f704be898fec887";
|
|
217
523
|
export const SCHEMA_V2_FINGERPRINT =
|
|
218
524
|
"402c58d4b3b217699591528f9be6c1c7fe341650b843eb2d8d9d907779ff27b0";
|
|
525
|
+
export const SCHEMA_V3_FINGERPRINT =
|
|
526
|
+
"aac3733641bebb5a86aea3f9c421818201f5dde051da6b95b880d503a420047b";
|
|
527
|
+
export const SCHEMA_V4_FINGERPRINT =
|
|
528
|
+
"59762b9d5da862b857edb76021ed51d05a256133dce543a3fd7d7b0403c9c081";
|
|
529
|
+
export const SCHEMA_V5_FINGERPRINT =
|
|
530
|
+
"9619647628488fc4fdc79ad36b4b6632d8257f40c23245822a18941870c187ac";
|
|
531
|
+
export const SCHEMA_V6_FINGERPRINT =
|
|
532
|
+
"6abe5595ba3f772c9643440b27d18d5d40057c85523ecb3b4986b195d67787c2";
|
|
219
533
|
|
|
220
534
|
const EXPECTED_COLUMNS = Object.freeze({
|
|
221
535
|
migrations: [
|
|
@@ -302,13 +616,98 @@ const EXPECTED_COLUMNS = Object.freeze({
|
|
|
302
616
|
["created_at", "INTEGER", 1, 0],
|
|
303
617
|
["settled_at", "INTEGER", 0, 0],
|
|
304
618
|
],
|
|
619
|
+
scheduler_occurrence_adjudications: [
|
|
620
|
+
["occurrence_id", "TEXT", 1, 1],
|
|
621
|
+
["job_id", "TEXT", 1, 0],
|
|
622
|
+
["request_id", "TEXT", 1, 0],
|
|
623
|
+
["decision", "TEXT", 1, 0],
|
|
624
|
+
["authority", "TEXT", 1, 0],
|
|
625
|
+
["evidence_digest", "TEXT", 1, 0],
|
|
626
|
+
["expected_attempt", "INTEGER", 1, 0],
|
|
627
|
+
["expected_fence", "INTEGER", 1, 0],
|
|
628
|
+
["reason_digest", "TEXT", 1, 0],
|
|
629
|
+
["operator_digest", "TEXT", 1, 0],
|
|
630
|
+
["status", "TEXT", 1, 0],
|
|
631
|
+
["created_at", "INTEGER", 1, 0],
|
|
632
|
+
["applied_at", "INTEGER", 0, 0],
|
|
633
|
+
["retry_settled_at", "INTEGER", 0, 0],
|
|
634
|
+
["retry_outcome_json", "TEXT", 0, 0],
|
|
635
|
+
],
|
|
636
|
+
scheduler_occurrence_controls: [
|
|
637
|
+
["occurrence_id", "TEXT", 1, 1],
|
|
638
|
+
["capability_json", "TEXT", 1, 0],
|
|
639
|
+
["capability_digest", "TEXT", 1, 0],
|
|
640
|
+
["state", "TEXT", 1, 0],
|
|
641
|
+
["revision", "INTEGER", 1, 0],
|
|
642
|
+
["pause_request_id", "TEXT", 1, 0],
|
|
643
|
+
["resume_request_id", "TEXT", 0, 0],
|
|
644
|
+
["expected_fence", "INTEGER", 1, 0],
|
|
645
|
+
["checkpoint_json", "TEXT", 0, 0],
|
|
646
|
+
["requested_at", "INTEGER", 1, 0],
|
|
647
|
+
["paused_at", "INTEGER", 0, 0],
|
|
648
|
+
["resumed_at", "INTEGER", 0, 0],
|
|
649
|
+
["terminal_at", "INTEGER", 0, 0],
|
|
650
|
+
["updated_at", "INTEGER", 1, 0],
|
|
651
|
+
],
|
|
652
|
+
scheduler_occurrence_retries: [
|
|
653
|
+
["occurrence_id", "TEXT", 1, 1],
|
|
654
|
+
["request_id", "TEXT", 1, 2],
|
|
655
|
+
["expected_fence", "INTEGER", 1, 0],
|
|
656
|
+
["expected_error_code", "TEXT", 1, 0],
|
|
657
|
+
["created_at", "INTEGER", 1, 0],
|
|
658
|
+
["claimed_at", "INTEGER", 0, 0],
|
|
659
|
+
],
|
|
660
|
+
scheduler_domain_migrations: [
|
|
661
|
+
["migration_id", "TEXT", 1, 1],
|
|
662
|
+
["manifest_digest", "TEXT", 1, 0],
|
|
663
|
+
["manifest_json", "TEXT", 1, 0],
|
|
664
|
+
["state", "TEXT", 1, 0],
|
|
665
|
+
["entry_count", "INTEGER", 1, 0],
|
|
666
|
+
["created_at", "INTEGER", 1, 0],
|
|
667
|
+
["updated_at", "INTEGER", 1, 0],
|
|
668
|
+
["completed_at", "INTEGER", 0, 0],
|
|
669
|
+
["last_error_json", "TEXT", 0, 0],
|
|
670
|
+
],
|
|
671
|
+
scheduler_domain_migration_entries: [
|
|
672
|
+
["migration_id", "TEXT", 1, 1],
|
|
673
|
+
["entry_id", "TEXT", 1, 2],
|
|
674
|
+
["domain", "TEXT", 1, 0],
|
|
675
|
+
["source_id", "TEXT", 1, 0],
|
|
676
|
+
["source_scope_digest", "TEXT", 1, 0],
|
|
677
|
+
["source_digest", "TEXT", 1, 0],
|
|
678
|
+
["target_job_id", "TEXT", 1, 0],
|
|
679
|
+
["target_job_digest", "TEXT", 1, 0],
|
|
680
|
+
["rollback_strategy", "TEXT", 1, 0],
|
|
681
|
+
["state", "TEXT", 1, 0],
|
|
682
|
+
["target_action", "TEXT", 0, 0],
|
|
683
|
+
["target_before_json", "TEXT", 0, 0],
|
|
684
|
+
["target_applied_revision", "INTEGER", 0, 0],
|
|
685
|
+
["target_applied_at", "INTEGER", 0, 0],
|
|
686
|
+
["target_occurrence_count_before", "INTEGER", 0, 0],
|
|
687
|
+
["target_execution_event_count_before", "INTEGER", 0, 0],
|
|
688
|
+
["target_rollback_revision", "INTEGER", 0, 0],
|
|
689
|
+
["retirement_token", "TEXT", 0, 0],
|
|
690
|
+
["source_retirement_digest", "TEXT", 0, 0],
|
|
691
|
+
["source_restored_digest", "TEXT", 0, 0],
|
|
692
|
+
["created_at", "INTEGER", 1, 0],
|
|
693
|
+
["updated_at", "INTEGER", 1, 0],
|
|
694
|
+
["source_locator_json", "TEXT", 0, 0],
|
|
695
|
+
],
|
|
305
696
|
});
|
|
306
697
|
|
|
307
698
|
const EXPECTED_INDEXES = Object.freeze([
|
|
699
|
+
"scheduler_adjudications_status",
|
|
308
700
|
"scheduler_authority_reservations_principal",
|
|
309
701
|
"scheduler_authority_reservations_status",
|
|
702
|
+
"scheduler_domain_migration_active_source",
|
|
703
|
+
"scheduler_domain_migration_active_target",
|
|
704
|
+
"scheduler_domain_migration_entries_state",
|
|
705
|
+
"scheduler_domain_migrations_state",
|
|
310
706
|
"scheduler_events_job",
|
|
311
707
|
"scheduler_events_occurrence",
|
|
708
|
+
"scheduler_occurrence_controls_pause_request",
|
|
709
|
+
"scheduler_occurrence_controls_state",
|
|
710
|
+
"scheduler_occurrence_retries_pending",
|
|
312
711
|
"scheduler_occurrences_claim",
|
|
313
712
|
"scheduler_occurrences_idempotency",
|
|
314
713
|
"scheduler_occurrences_job",
|
|
@@ -323,6 +722,45 @@ function schemaError(code, message, cause = undefined, details = undefined) {
|
|
|
323
722
|
);
|
|
324
723
|
}
|
|
325
724
|
|
|
725
|
+
function normalizedStorageFailureCode(value) {
|
|
726
|
+
if (typeof value !== "string") return null;
|
|
727
|
+
const candidate = value.trim().toUpperCase();
|
|
728
|
+
if (STORAGE_FAILURE_CODE_SET.has(candidate)) return candidate;
|
|
729
|
+
return (
|
|
730
|
+
STORAGE_FAILURE_CODES.find((code) => candidate.startsWith(`${code}_`)) ||
|
|
731
|
+
null
|
|
732
|
+
);
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
function findStorageFailureCode(error) {
|
|
736
|
+
const visited = new Set();
|
|
737
|
+
let current = error;
|
|
738
|
+
while (
|
|
739
|
+
current !== null &&
|
|
740
|
+
(typeof current === "object" || typeof current === "function") &&
|
|
741
|
+
!visited.has(current)
|
|
742
|
+
) {
|
|
743
|
+
visited.add(current);
|
|
744
|
+
const code = normalizedStorageFailureCode(current.code);
|
|
745
|
+
if (code) return code;
|
|
746
|
+
current = current.cause;
|
|
747
|
+
}
|
|
748
|
+
return null;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
function storageUnavailableError({ phase, storageCode, commitState }) {
|
|
752
|
+
return new SchedulerKernelError(
|
|
753
|
+
"SCHEDULER_STORAGE_UNAVAILABLE",
|
|
754
|
+
"Scheduler storage is unavailable",
|
|
755
|
+
{
|
|
756
|
+
phase,
|
|
757
|
+
storageCode,
|
|
758
|
+
commitState,
|
|
759
|
+
retryable: false,
|
|
760
|
+
},
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
|
|
326
764
|
function assertPositiveSafeInteger(value, field) {
|
|
327
765
|
if (!Number.isSafeInteger(value) || value < 1) {
|
|
328
766
|
throw invalidArgument(`${field} must be a positive integer`);
|
|
@@ -573,6 +1011,109 @@ function mapOccurrence(row) {
|
|
|
573
1011
|
};
|
|
574
1012
|
}
|
|
575
1013
|
|
|
1014
|
+
function mapRuntimeControlOccurrence(row) {
|
|
1015
|
+
if (!row) return null;
|
|
1016
|
+
return {
|
|
1017
|
+
id: row.occurrence_id,
|
|
1018
|
+
jobId: row.job_id,
|
|
1019
|
+
jobKind: row.job_kind,
|
|
1020
|
+
runtimeStatus: row.runtime_status,
|
|
1021
|
+
occurrenceStatus: row.occurrence_status,
|
|
1022
|
+
scheduledFor: row.scheduled_for,
|
|
1023
|
+
attempt: row.attempt,
|
|
1024
|
+
maxAttempts: row.max_attempts,
|
|
1025
|
+
fence: row.fence,
|
|
1026
|
+
createdAt: row.created_at,
|
|
1027
|
+
updatedAt: row.updated_at,
|
|
1028
|
+
control:
|
|
1029
|
+
row.control_state === null
|
|
1030
|
+
? null
|
|
1031
|
+
: {
|
|
1032
|
+
state: row.control_state,
|
|
1033
|
+
revision: row.control_revision,
|
|
1034
|
+
expectedFence: row.control_expected_fence,
|
|
1035
|
+
capabilityDigest: row.control_capability_digest,
|
|
1036
|
+
requestedAt: row.control_requested_at,
|
|
1037
|
+
pausedAt: row.control_paused_at,
|
|
1038
|
+
updatedAt: row.control_updated_at,
|
|
1039
|
+
},
|
|
1040
|
+
};
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
function normalizeRuntimeControlEnumeration(values, allowed, field) {
|
|
1044
|
+
if (!Array.isArray(values) || values.length === 0) {
|
|
1045
|
+
throw invalidArgument(`${field} must be a non-empty array`);
|
|
1046
|
+
}
|
|
1047
|
+
const normalized = [
|
|
1048
|
+
...new Set(
|
|
1049
|
+
values.map((value, index) =>
|
|
1050
|
+
normalizeIdentifier(value, `${field}[${index}]`, { maxLength: 128 }),
|
|
1051
|
+
),
|
|
1052
|
+
),
|
|
1053
|
+
];
|
|
1054
|
+
const unsupported = normalized.filter((value) => !allowed.has(value));
|
|
1055
|
+
if (unsupported.length > 0) {
|
|
1056
|
+
throw invalidArgument(`${field} contains unsupported values`, {
|
|
1057
|
+
values: unsupported.sort(),
|
|
1058
|
+
});
|
|
1059
|
+
}
|
|
1060
|
+
return normalized.sort();
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
export function schedulerRuntimeControlCapabilityDigest(capability) {
|
|
1064
|
+
return sha256PayloadDigest(
|
|
1065
|
+
normalizeRuntimeControlCapability(capability),
|
|
1066
|
+
"runtimeControl.capability",
|
|
1067
|
+
);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
function mapOccurrenceControl(row) {
|
|
1071
|
+
if (!row) return null;
|
|
1072
|
+
const capability = normalizeRuntimeControlCapability(
|
|
1073
|
+
readStoredJson(row.capability_json, "scheduler runtime control capability"),
|
|
1074
|
+
);
|
|
1075
|
+
const capabilityDigest = schedulerRuntimeControlCapabilityDigest(capability);
|
|
1076
|
+
if (capabilityDigest !== row.capability_digest) {
|
|
1077
|
+
throw schemaError(
|
|
1078
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
1079
|
+
`Scheduler runtime-control capability digest is invalid: ${row.occurrence_id}`,
|
|
1080
|
+
);
|
|
1081
|
+
}
|
|
1082
|
+
return {
|
|
1083
|
+
occurrenceId: row.occurrence_id,
|
|
1084
|
+
capability,
|
|
1085
|
+
capabilityDigest,
|
|
1086
|
+
state: row.state,
|
|
1087
|
+
revision: row.revision,
|
|
1088
|
+
pauseRequestId: row.pause_request_id,
|
|
1089
|
+
resumeRequestId: row.resume_request_id,
|
|
1090
|
+
expectedFence: row.expected_fence,
|
|
1091
|
+
checkpoint:
|
|
1092
|
+
row.checkpoint_json === null
|
|
1093
|
+
? null
|
|
1094
|
+
: readStoredJson(
|
|
1095
|
+
row.checkpoint_json,
|
|
1096
|
+
"scheduler runtime control checkpoint",
|
|
1097
|
+
),
|
|
1098
|
+
requestedAt: row.requested_at,
|
|
1099
|
+
pausedAt: row.paused_at,
|
|
1100
|
+
resumedAt: row.resumed_at,
|
|
1101
|
+
terminalAt: row.terminal_at,
|
|
1102
|
+
updatedAt: row.updated_at,
|
|
1103
|
+
};
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
function normalizeRuntimeCheckpoint(value) {
|
|
1107
|
+
const checkpoint = normalizeJson(value ?? null, "runtimeControl.checkpoint");
|
|
1108
|
+
const encoded = canonicalJson(checkpoint, "runtimeControl.checkpoint");
|
|
1109
|
+
if (Buffer.byteLength(encoded, "utf8") > MAX_RUNTIME_CHECKPOINT_BYTES) {
|
|
1110
|
+
throw invalidArgument(
|
|
1111
|
+
`runtimeControl.checkpoint exceeds ${MAX_RUNTIME_CHECKPOINT_BYTES} encoded bytes`,
|
|
1112
|
+
);
|
|
1113
|
+
}
|
|
1114
|
+
return { checkpoint, encoded };
|
|
1115
|
+
}
|
|
1116
|
+
|
|
576
1117
|
function mapEvent(row) {
|
|
577
1118
|
return {
|
|
578
1119
|
eventId: row.event_id,
|
|
@@ -586,94 +1127,647 @@ function mapEvent(row) {
|
|
|
586
1127
|
};
|
|
587
1128
|
}
|
|
588
1129
|
|
|
589
|
-
function
|
|
590
|
-
return
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
.
|
|
595
|
-
|
|
1130
|
+
function mapAdjudication(row) {
|
|
1131
|
+
if (!row) return null;
|
|
1132
|
+
return {
|
|
1133
|
+
occurrenceId: row.occurrence_id,
|
|
1134
|
+
jobId: row.job_id,
|
|
1135
|
+
requestId: row.request_id,
|
|
1136
|
+
decision: row.decision,
|
|
1137
|
+
authority: row.authority,
|
|
1138
|
+
evidenceDigest: row.evidence_digest,
|
|
1139
|
+
expectedAttempt: row.expected_attempt,
|
|
1140
|
+
expectedFence: row.expected_fence,
|
|
1141
|
+
reasonDigest: row.reason_digest,
|
|
1142
|
+
operatorDigest: row.operator_digest,
|
|
1143
|
+
status: row.status,
|
|
1144
|
+
createdAt: row.created_at,
|
|
1145
|
+
appliedAt: row.applied_at,
|
|
1146
|
+
retrySettledAt: row.retry_settled_at,
|
|
1147
|
+
retryOutcome:
|
|
1148
|
+
row.retry_outcome_json === null
|
|
1149
|
+
? null
|
|
1150
|
+
: readStoredJson(row.retry_outcome_json, "adjudication retry outcome"),
|
|
1151
|
+
};
|
|
596
1152
|
}
|
|
597
1153
|
|
|
598
|
-
function
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
}
|
|
609
|
-
if (
|
|
610
|
-
!Array.isArray(rows) ||
|
|
611
|
-
rows.length !== 1 ||
|
|
612
|
-
rows[0]?.quick_check !== "ok"
|
|
613
|
-
) {
|
|
614
|
-
throw schemaError(
|
|
615
|
-
"SCHEDULER_SCHEMA_CORRUPT",
|
|
616
|
-
"Scheduler database failed SQLite quick_check",
|
|
617
|
-
undefined,
|
|
618
|
-
{ result: rows },
|
|
619
|
-
);
|
|
620
|
-
}
|
|
1154
|
+
function schedulerJobDefinition(job) {
|
|
1155
|
+
return {
|
|
1156
|
+
id: job.id,
|
|
1157
|
+
kind: job.kind,
|
|
1158
|
+
trigger: job.trigger,
|
|
1159
|
+
payload: job.payload,
|
|
1160
|
+
authority: job.authority,
|
|
1161
|
+
enabled: job.enabled,
|
|
1162
|
+
maxAttempts: job.maxAttempts,
|
|
1163
|
+
};
|
|
621
1164
|
}
|
|
622
1165
|
|
|
623
|
-
function
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
.
|
|
627
|
-
|
|
628
|
-
String(column.type || "").toUpperCase(),
|
|
629
|
-
column.notnull,
|
|
630
|
-
column.pk,
|
|
631
|
-
]);
|
|
632
|
-
const expected = EXPECTED_COLUMNS[table];
|
|
633
|
-
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
|
634
|
-
throw schemaError(
|
|
635
|
-
"SCHEDULER_SCHEMA_CORRUPT",
|
|
636
|
-
`Scheduler table shape is invalid: ${table}`,
|
|
637
|
-
undefined,
|
|
638
|
-
{ expected, actual },
|
|
639
|
-
);
|
|
640
|
-
}
|
|
1166
|
+
export function schedulerJobDefinitionDigest(job) {
|
|
1167
|
+
return sha256PayloadDigest(
|
|
1168
|
+
schedulerJobDefinition(normalizeJobInput(schedulerJobDefinition(job))),
|
|
1169
|
+
"schedulerMigration.targetJob",
|
|
1170
|
+
);
|
|
641
1171
|
}
|
|
642
1172
|
|
|
643
|
-
function
|
|
644
|
-
|
|
645
|
-
.
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
.all()
|
|
649
|
-
.map((row) => ({
|
|
650
|
-
type: row.type,
|
|
651
|
-
name: row.name,
|
|
652
|
-
tbl_name: row.tbl_name,
|
|
653
|
-
sql: String(row.sql || "")
|
|
654
|
-
.replace(/\s+/g, " ")
|
|
655
|
-
.trim(),
|
|
656
|
-
}));
|
|
657
|
-
return createHash("sha256")
|
|
658
|
-
.update(JSON.stringify(catalog), "utf8")
|
|
659
|
-
.digest("hex");
|
|
1173
|
+
export function schedulerMigrationSourceDigest(source) {
|
|
1174
|
+
return sha256PayloadDigest(
|
|
1175
|
+
normalizeJson(source, "schedulerMigration.source"),
|
|
1176
|
+
"schedulerMigration.source",
|
|
1177
|
+
);
|
|
660
1178
|
}
|
|
661
1179
|
|
|
662
|
-
function
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
1180
|
+
export function schedulerMigrationScopeDigest(sourceScope) {
|
|
1181
|
+
return sha256PayloadDigest(
|
|
1182
|
+
normalizeJson(sourceScope, "schedulerMigration.sourceScope"),
|
|
1183
|
+
"schedulerMigration.sourceScope",
|
|
1184
|
+
);
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
const SENSITIVE_SOURCE_LOCATOR_FIELD =
|
|
1188
|
+
/(?:pass(?:word)?|secret|token|credential|api[_-]?key|private[_-]?key)/iu;
|
|
1189
|
+
|
|
1190
|
+
function assertNonSensitiveSourceLocator(value, field) {
|
|
1191
|
+
if (Array.isArray(value)) {
|
|
1192
|
+
value.forEach((item, index) =>
|
|
1193
|
+
assertNonSensitiveSourceLocator(item, `${field}[${index}]`),
|
|
672
1194
|
);
|
|
1195
|
+
return;
|
|
1196
|
+
}
|
|
1197
|
+
if (!value || typeof value !== "object") return;
|
|
1198
|
+
for (const [key, nested] of Object.entries(value)) {
|
|
1199
|
+
if (SENSITIVE_SOURCE_LOCATOR_FIELD.test(key)) {
|
|
1200
|
+
throw invalidArgument(`${field} must not contain sensitive fields`, {
|
|
1201
|
+
field: key,
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
assertNonSensitiveSourceLocator(nested, `${field}.${key}`);
|
|
673
1205
|
}
|
|
1206
|
+
}
|
|
674
1207
|
|
|
675
|
-
|
|
676
|
-
|
|
1208
|
+
function normalizeSchedulerMigrationSourceLocator(
|
|
1209
|
+
value,
|
|
1210
|
+
domain,
|
|
1211
|
+
sourceScopeDigest,
|
|
1212
|
+
field,
|
|
1213
|
+
{ allowLegacyAutomationCurrent = false } = {},
|
|
1214
|
+
) {
|
|
1215
|
+
if (value === undefined || value === null) return null;
|
|
1216
|
+
const locator = normalizeJson(value, field);
|
|
1217
|
+
if (!locator || typeof locator !== "object" || Array.isArray(locator)) {
|
|
1218
|
+
throw invalidArgument(`${field} must be an object`);
|
|
1219
|
+
}
|
|
1220
|
+
if (locator.schemaVersion !== 1) {
|
|
1221
|
+
throw invalidArgument(`${field}.schemaVersion must be 1`);
|
|
1222
|
+
}
|
|
1223
|
+
const type = normalizeIdentifier(locator.type, `${field}.type`, {
|
|
1224
|
+
maxLength: 128,
|
|
1225
|
+
});
|
|
1226
|
+
const locatorTypes = {
|
|
1227
|
+
agenda: {
|
|
1228
|
+
type: "agenda-store",
|
|
1229
|
+
key: "directory",
|
|
1230
|
+
store: "agent-schedule",
|
|
1231
|
+
},
|
|
1232
|
+
"cowork-cron": {
|
|
1233
|
+
type: "cowork-workspace",
|
|
1234
|
+
key: "workspace",
|
|
1235
|
+
store: "cowork-schedules",
|
|
1236
|
+
},
|
|
1237
|
+
routine: {
|
|
1238
|
+
type: "routine-store",
|
|
1239
|
+
key: "directory",
|
|
1240
|
+
store: "routines",
|
|
1241
|
+
},
|
|
1242
|
+
automation: {
|
|
1243
|
+
type: "automation-database",
|
|
1244
|
+
key: "database",
|
|
1245
|
+
store: "automation-engine",
|
|
1246
|
+
},
|
|
1247
|
+
"loop-iteration": {
|
|
1248
|
+
type: "jsonl-session",
|
|
1249
|
+
key: "sessionId",
|
|
1250
|
+
additionalKey: "directory",
|
|
1251
|
+
store: "jsonl-session",
|
|
1252
|
+
},
|
|
1253
|
+
};
|
|
1254
|
+
const locatorType = locatorTypes[domain];
|
|
1255
|
+
if (!locatorType || locatorType.type !== type) {
|
|
1256
|
+
throw invalidArgument(`${field}.type does not match migration domain`);
|
|
1257
|
+
}
|
|
1258
|
+
const allowed = new Set([
|
|
1259
|
+
"schemaVersion",
|
|
1260
|
+
"type",
|
|
1261
|
+
locatorType.key,
|
|
1262
|
+
...(locatorType.additionalKey ? [locatorType.additionalKey] : []),
|
|
1263
|
+
]);
|
|
1264
|
+
const unknown = Object.keys(locator).filter((key) => !allowed.has(key));
|
|
1265
|
+
if (unknown.length > 0) {
|
|
1266
|
+
throw invalidArgument(`${field} contains unknown fields`, {
|
|
1267
|
+
fields: unknown.sort(),
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
const location = normalizeIdentifier(
|
|
1271
|
+
locator[locatorType.key],
|
|
1272
|
+
`${field}.${locatorType.key}`,
|
|
1273
|
+
{ maxLength: 4096 },
|
|
1274
|
+
);
|
|
1275
|
+
const additionalLocation = locatorType.additionalKey
|
|
1276
|
+
? normalizeIdentifier(
|
|
1277
|
+
locator[locatorType.additionalKey],
|
|
1278
|
+
`${field}.${locatorType.additionalKey}`,
|
|
1279
|
+
{ maxLength: 4096 },
|
|
1280
|
+
)
|
|
1281
|
+
: null;
|
|
1282
|
+
const pathKeys = new Set(["directory", "workspace", "database"]);
|
|
1283
|
+
for (const [key, candidate] of [
|
|
1284
|
+
[locatorType.key, location],
|
|
1285
|
+
[locatorType.additionalKey, additionalLocation],
|
|
1286
|
+
]) {
|
|
1287
|
+
if (
|
|
1288
|
+
key &&
|
|
1289
|
+
pathKeys.has(key) &&
|
|
1290
|
+
!(domain === "automation" && String(candidate).startsWith("memory:")) &&
|
|
1291
|
+
!isCanonicalSchedulerSourcePath(candidate)
|
|
1292
|
+
) {
|
|
1293
|
+
throw invalidArgument(
|
|
1294
|
+
`${field}.${key} must be an absolute path in canonical form; Windows paths require a fully-qualified drive or UNC share`,
|
|
1295
|
+
);
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
const derivedScope = {
|
|
1299
|
+
store: locatorType.store,
|
|
1300
|
+
[locatorType.key]: location,
|
|
1301
|
+
};
|
|
1302
|
+
const legacyAutomationCurrentScope =
|
|
1303
|
+
allowLegacyAutomationCurrent &&
|
|
1304
|
+
domain === "automation" &&
|
|
1305
|
+
schedulerMigrationScopeDigest({
|
|
1306
|
+
store: "automation-engine",
|
|
1307
|
+
database: "current",
|
|
1308
|
+
}) === sourceScopeDigest;
|
|
1309
|
+
if (
|
|
1310
|
+
schedulerMigrationScopeDigest(derivedScope) !== sourceScopeDigest &&
|
|
1311
|
+
!legacyAutomationCurrentScope
|
|
1312
|
+
) {
|
|
1313
|
+
throw invalidArgument(`${field} must resolve to sourceScope`);
|
|
1314
|
+
}
|
|
1315
|
+
const normalized = {
|
|
1316
|
+
schemaVersion: 1,
|
|
1317
|
+
type,
|
|
1318
|
+
[locatorType.key]: location,
|
|
1319
|
+
...(locatorType.additionalKey
|
|
1320
|
+
? { [locatorType.additionalKey]: additionalLocation }
|
|
1321
|
+
: {}),
|
|
1322
|
+
};
|
|
1323
|
+
assertNonSensitiveSourceLocator(normalized, field);
|
|
1324
|
+
return normalized;
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
function normalizeDomainMigrationEntry(input, index) {
|
|
1328
|
+
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
|
1329
|
+
throw invalidArgument(`entries[${index}] must be an object`);
|
|
1330
|
+
}
|
|
1331
|
+
const allowed = new Set([
|
|
1332
|
+
"domain",
|
|
1333
|
+
"sourceId",
|
|
1334
|
+
"sourceScope",
|
|
1335
|
+
"sourceLocator",
|
|
1336
|
+
"source",
|
|
1337
|
+
"targetJob",
|
|
1338
|
+
"rollbackStrategy",
|
|
1339
|
+
]);
|
|
1340
|
+
const unknown = Object.keys(input).filter((key) => !allowed.has(key));
|
|
1341
|
+
if (unknown.length > 0) {
|
|
1342
|
+
throw invalidArgument(`entries[${index}] contains unknown fields`, {
|
|
1343
|
+
fields: unknown.sort(),
|
|
1344
|
+
});
|
|
1345
|
+
}
|
|
1346
|
+
if (!SCHEDULER_MIGRATION_DOMAINS.includes(input.domain)) {
|
|
1347
|
+
throw invalidArgument(
|
|
1348
|
+
`entries[${index}].domain must be a supported scheduler migration domain`,
|
|
1349
|
+
);
|
|
1350
|
+
}
|
|
1351
|
+
const sourceId = normalizeIdentifier(
|
|
1352
|
+
input.sourceId,
|
|
1353
|
+
`entries[${index}].sourceId`,
|
|
1354
|
+
);
|
|
1355
|
+
const sourceScopeDigest = schedulerMigrationScopeDigest(input.sourceScope);
|
|
1356
|
+
const sourceLocator = normalizeSchedulerMigrationSourceLocator(
|
|
1357
|
+
input.sourceLocator,
|
|
1358
|
+
input.domain,
|
|
1359
|
+
sourceScopeDigest,
|
|
1360
|
+
`entries[${index}].sourceLocator`,
|
|
1361
|
+
);
|
|
1362
|
+
const sourceDigest = schedulerMigrationSourceDigest(input.source);
|
|
1363
|
+
const targetJob = normalizeJobInput(input.targetJob);
|
|
1364
|
+
const targetJobDigest = schedulerJobDefinitionDigest(targetJob);
|
|
1365
|
+
const rollbackStrategy = input.rollbackStrategy ?? "restore";
|
|
1366
|
+
if (!["restore", "disable"].includes(rollbackStrategy)) {
|
|
1367
|
+
throw invalidArgument(
|
|
1368
|
+
`entries[${index}].rollbackStrategy must be restore or disable`,
|
|
1369
|
+
);
|
|
1370
|
+
}
|
|
1371
|
+
const identity = {
|
|
1372
|
+
schemaVersion: 1,
|
|
1373
|
+
domain: input.domain,
|
|
1374
|
+
sourceId,
|
|
1375
|
+
sourceScopeDigest,
|
|
1376
|
+
sourceDigest,
|
|
1377
|
+
targetJobId: targetJob.id,
|
|
1378
|
+
targetJobDigest,
|
|
1379
|
+
rollbackStrategy,
|
|
1380
|
+
};
|
|
1381
|
+
const entryDigest = sha256PayloadDigest(
|
|
1382
|
+
identity,
|
|
1383
|
+
`schedulerMigration.entries[${index}]`,
|
|
1384
|
+
);
|
|
1385
|
+
return {
|
|
1386
|
+
...identity,
|
|
1387
|
+
sourceLocator,
|
|
1388
|
+
entryId: `scheduler-migration-entry-${entryDigest.slice("sha256:".length)}`,
|
|
1389
|
+
targetJob,
|
|
1390
|
+
};
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
function normalizeDomainMigrationPlan(input) {
|
|
1394
|
+
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
|
1395
|
+
throw invalidArgument("scheduler migration plan must be an object");
|
|
1396
|
+
}
|
|
1397
|
+
const unknown = Object.keys(input).filter((key) => key !== "entries");
|
|
1398
|
+
if (unknown.length > 0) {
|
|
1399
|
+
throw invalidArgument("scheduler migration plan contains unknown fields", {
|
|
1400
|
+
fields: unknown.sort(),
|
|
1401
|
+
});
|
|
1402
|
+
}
|
|
1403
|
+
if (
|
|
1404
|
+
!Array.isArray(input.entries) ||
|
|
1405
|
+
input.entries.length < 1 ||
|
|
1406
|
+
input.entries.length > 1_000
|
|
1407
|
+
) {
|
|
1408
|
+
throw invalidArgument(
|
|
1409
|
+
"scheduler migration plan must contain between 1 and 1000 entries",
|
|
1410
|
+
);
|
|
1411
|
+
}
|
|
1412
|
+
const entries = input.entries
|
|
1413
|
+
.map(normalizeDomainMigrationEntry)
|
|
1414
|
+
.sort((left, right) => left.entryId.localeCompare(right.entryId));
|
|
1415
|
+
const entryIds = new Set();
|
|
1416
|
+
const sources = new Set();
|
|
1417
|
+
const targets = new Set();
|
|
1418
|
+
for (const entry of entries) {
|
|
1419
|
+
const sourceKey = canonicalJson([
|
|
1420
|
+
entry.domain,
|
|
1421
|
+
entry.sourceScopeDigest,
|
|
1422
|
+
entry.sourceId,
|
|
1423
|
+
]);
|
|
1424
|
+
if (entryIds.has(entry.entryId) || sources.has(sourceKey)) {
|
|
1425
|
+
throw invalidArgument(
|
|
1426
|
+
"scheduler migration plan contains duplicate sources",
|
|
1427
|
+
);
|
|
1428
|
+
}
|
|
1429
|
+
if (targets.has(entry.targetJobId)) {
|
|
1430
|
+
throw invalidArgument(
|
|
1431
|
+
"scheduler migration plan contains duplicate target jobs",
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1434
|
+
entryIds.add(entry.entryId);
|
|
1435
|
+
sources.add(sourceKey);
|
|
1436
|
+
targets.add(entry.targetJobId);
|
|
1437
|
+
}
|
|
1438
|
+
const manifest = {
|
|
1439
|
+
schemaVersion: 1,
|
|
1440
|
+
entries: entries.map((entry) => ({
|
|
1441
|
+
schemaVersion: entry.schemaVersion,
|
|
1442
|
+
entryId: entry.entryId,
|
|
1443
|
+
domain: entry.domain,
|
|
1444
|
+
sourceId: entry.sourceId,
|
|
1445
|
+
sourceScopeDigest: entry.sourceScopeDigest,
|
|
1446
|
+
sourceDigest: entry.sourceDigest,
|
|
1447
|
+
targetJobId: entry.targetJobId,
|
|
1448
|
+
targetJobDigest: entry.targetJobDigest,
|
|
1449
|
+
rollbackStrategy: entry.rollbackStrategy,
|
|
1450
|
+
targetJob: entry.targetJob,
|
|
1451
|
+
})),
|
|
1452
|
+
};
|
|
1453
|
+
const manifestDigest = sha256PayloadDigest(
|
|
1454
|
+
manifest,
|
|
1455
|
+
"schedulerMigration.manifest",
|
|
1456
|
+
);
|
|
1457
|
+
return {
|
|
1458
|
+
migrationId: `scheduler-domain-migration-${manifestDigest.slice("sha256:".length)}`,
|
|
1459
|
+
manifestDigest,
|
|
1460
|
+
manifest,
|
|
1461
|
+
entries,
|
|
1462
|
+
};
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
function mapDomainMigrationEntry(row) {
|
|
1466
|
+
if (!row) return null;
|
|
1467
|
+
let sourceLocator = null;
|
|
1468
|
+
if (row.source_locator_json !== null) {
|
|
1469
|
+
try {
|
|
1470
|
+
sourceLocator = normalizeSchedulerMigrationSourceLocator(
|
|
1471
|
+
readStoredJson(
|
|
1472
|
+
row.source_locator_json,
|
|
1473
|
+
"scheduler migration source locator",
|
|
1474
|
+
),
|
|
1475
|
+
row.domain,
|
|
1476
|
+
row.source_scope_digest,
|
|
1477
|
+
"scheduler migration source locator",
|
|
1478
|
+
{ allowLegacyAutomationCurrent: true },
|
|
1479
|
+
);
|
|
1480
|
+
} catch (cause) {
|
|
1481
|
+
if (cause?.code === "SCHEDULER_DATA_CORRUPT") throw cause;
|
|
1482
|
+
throw schemaError(
|
|
1483
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
1484
|
+
"Stored scheduler migration source locator is invalid or unbound",
|
|
1485
|
+
cause,
|
|
1486
|
+
);
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
return {
|
|
1490
|
+
migrationId: row.migration_id,
|
|
1491
|
+
entryId: row.entry_id,
|
|
1492
|
+
domain: row.domain,
|
|
1493
|
+
sourceId: row.source_id,
|
|
1494
|
+
sourceScopeDigest: row.source_scope_digest,
|
|
1495
|
+
sourceLocator,
|
|
1496
|
+
sourceLocatorDigest:
|
|
1497
|
+
sourceLocator === null
|
|
1498
|
+
? null
|
|
1499
|
+
: sha256PayloadDigest(
|
|
1500
|
+
sourceLocator,
|
|
1501
|
+
"schedulerMigration.sourceLocator",
|
|
1502
|
+
),
|
|
1503
|
+
sourceDigest: row.source_digest,
|
|
1504
|
+
targetJobId: row.target_job_id,
|
|
1505
|
+
targetJobDigest: row.target_job_digest,
|
|
1506
|
+
rollbackStrategy: row.rollback_strategy,
|
|
1507
|
+
state: row.state,
|
|
1508
|
+
targetAction: row.target_action,
|
|
1509
|
+
targetBefore:
|
|
1510
|
+
row.target_before_json === null
|
|
1511
|
+
? null
|
|
1512
|
+
: readStoredJson(
|
|
1513
|
+
row.target_before_json,
|
|
1514
|
+
"scheduler migration target before",
|
|
1515
|
+
),
|
|
1516
|
+
targetAppliedRevision: row.target_applied_revision,
|
|
1517
|
+
targetAppliedAt: row.target_applied_at,
|
|
1518
|
+
targetOccurrenceCountBefore: row.target_occurrence_count_before,
|
|
1519
|
+
targetExecutionEventCountBefore: row.target_execution_event_count_before,
|
|
1520
|
+
targetRollbackRevision: row.target_rollback_revision,
|
|
1521
|
+
retirementToken: row.retirement_token,
|
|
1522
|
+
sourceRetirementDigest: row.source_retirement_digest,
|
|
1523
|
+
sourceRestoredDigest: row.source_restored_digest,
|
|
1524
|
+
createdAt: row.created_at,
|
|
1525
|
+
updatedAt: row.updated_at,
|
|
1526
|
+
};
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
function mapDomainMigration(row, entries = undefined) {
|
|
1530
|
+
if (!row) return null;
|
|
1531
|
+
return {
|
|
1532
|
+
id: row.migration_id,
|
|
1533
|
+
manifestDigest: row.manifest_digest,
|
|
1534
|
+
manifest: readStoredJson(row.manifest_json, "scheduler migration manifest"),
|
|
1535
|
+
state: row.state,
|
|
1536
|
+
entryCount: row.entry_count,
|
|
1537
|
+
createdAt: row.created_at,
|
|
1538
|
+
updatedAt: row.updated_at,
|
|
1539
|
+
completedAt: row.completed_at,
|
|
1540
|
+
lastError:
|
|
1541
|
+
row.last_error_json === null
|
|
1542
|
+
? null
|
|
1543
|
+
: readStoredJson(row.last_error_json, "scheduler migration error"),
|
|
1544
|
+
...(entries === undefined ? {} : { entries }),
|
|
1545
|
+
};
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
function schedulerMigrationError(code, message, details = undefined) {
|
|
1549
|
+
return new SchedulerKernelError(code, message, details);
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
function schedulerAdjudicationError(code, message, details = undefined) {
|
|
1553
|
+
return new SchedulerKernelError(code, message, details);
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
function normalizeAdjudicationDecision(value) {
|
|
1557
|
+
if (!Object.values(SCHEDULER_ADJUDICATION_DECISIONS).includes(value)) {
|
|
1558
|
+
throw invalidArgument(
|
|
1559
|
+
"decision must be confirmed_applied or confirmed_not_applied",
|
|
1560
|
+
);
|
|
1561
|
+
}
|
|
1562
|
+
return value;
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
function normalizeSha256Digest(value, field) {
|
|
1566
|
+
if (typeof value !== "string" || !/^sha256:[a-f0-9]{64}$/.test(value)) {
|
|
1567
|
+
throw invalidArgument(`${field} must be a sha256 payload digest`);
|
|
1568
|
+
}
|
|
1569
|
+
return value;
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
function isOutcomeUnknownError(error) {
|
|
1573
|
+
return (
|
|
1574
|
+
typeof error?.code === "string" &&
|
|
1575
|
+
/^[A-Z0-9_]+_OUTCOME_UNKNOWN$/.test(error.code)
|
|
1576
|
+
);
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
function sha256PayloadDigest(value, field) {
|
|
1580
|
+
return `sha256:${createHash("sha256")
|
|
1581
|
+
.update(canonicalJson(value, field), "utf8")
|
|
1582
|
+
.digest("hex")}`;
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
export function schedulerAdjudicationReasonDigest(reason) {
|
|
1586
|
+
if (typeof reason !== "string" || reason.trim().length === 0) {
|
|
1587
|
+
throw invalidArgument("adjudication reason must be a non-empty string");
|
|
1588
|
+
}
|
|
1589
|
+
if (Buffer.byteLength(reason, "utf8") > 4_000) {
|
|
1590
|
+
throw invalidArgument("adjudication reason must not exceed 4000 bytes");
|
|
1591
|
+
}
|
|
1592
|
+
return sha256PayloadDigest(reason, "adjudication.reason");
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
export function schedulerAdjudicationOperatorDigest(identity) {
|
|
1596
|
+
return sha256PayloadDigest(
|
|
1597
|
+
normalizeJson(identity, "adjudication.operatorIdentity"),
|
|
1598
|
+
"adjudication.operatorIdentity",
|
|
1599
|
+
);
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
function adjudicationEvidence(occurrence, reservation) {
|
|
1603
|
+
return {
|
|
1604
|
+
schemaVersion: 1,
|
|
1605
|
+
occurrence: {
|
|
1606
|
+
id: occurrence.id,
|
|
1607
|
+
jobId: occurrence.jobId,
|
|
1608
|
+
jobRevision: occurrence.jobRevision,
|
|
1609
|
+
idempotencyKey: occurrence.idempotencyKey,
|
|
1610
|
+
triggerKey: occurrence.triggerKey,
|
|
1611
|
+
scheduledFor: occurrence.scheduledFor,
|
|
1612
|
+
status: occurrence.status,
|
|
1613
|
+
attempt: occurrence.attempt,
|
|
1614
|
+
maxAttempts: occurrence.maxAttempts,
|
|
1615
|
+
fence: occurrence.fence,
|
|
1616
|
+
authorityDigest: sha256PayloadDigest(
|
|
1617
|
+
occurrence.authority,
|
|
1618
|
+
"adjudication.authority",
|
|
1619
|
+
),
|
|
1620
|
+
payloadDigest: sha256PayloadDigest(
|
|
1621
|
+
occurrence.payload,
|
|
1622
|
+
"adjudication.payload",
|
|
1623
|
+
),
|
|
1624
|
+
lastError: occurrence.lastError,
|
|
1625
|
+
settledAt: occurrence.settledAt,
|
|
1626
|
+
},
|
|
1627
|
+
reservation:
|
|
1628
|
+
reservation === null
|
|
1629
|
+
? null
|
|
1630
|
+
: {
|
|
1631
|
+
jobId: reservation.jobId,
|
|
1632
|
+
principal: reservation.principal,
|
|
1633
|
+
policyRevision: reservation.policyRevision,
|
|
1634
|
+
windowStartedAt: reservation.windowStartedAt,
|
|
1635
|
+
units: reservation.units,
|
|
1636
|
+
status: reservation.status,
|
|
1637
|
+
outcomeDigest:
|
|
1638
|
+
reservation.outcome === null
|
|
1639
|
+
? null
|
|
1640
|
+
: sha256PayloadDigest(
|
|
1641
|
+
reservation.outcome,
|
|
1642
|
+
"adjudication.reservationOutcome",
|
|
1643
|
+
),
|
|
1644
|
+
createdAt: reservation.createdAt,
|
|
1645
|
+
settledAt: reservation.settledAt,
|
|
1646
|
+
},
|
|
1647
|
+
};
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
function deterministicAdjudicationRequestId({
|
|
1651
|
+
occurrenceId,
|
|
1652
|
+
decision,
|
|
1653
|
+
evidenceDigest,
|
|
1654
|
+
reasonDigest,
|
|
1655
|
+
}) {
|
|
1656
|
+
const digest = sha256PayloadDigest(
|
|
1657
|
+
{
|
|
1658
|
+
schemaVersion: 1,
|
|
1659
|
+
occurrenceId,
|
|
1660
|
+
decision,
|
|
1661
|
+
evidenceDigest,
|
|
1662
|
+
reasonDigest,
|
|
1663
|
+
},
|
|
1664
|
+
"adjudication.request",
|
|
1665
|
+
);
|
|
1666
|
+
return `scheduler-adjudication-${digest.slice("sha256:".length)}`;
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
function listUserTables(db) {
|
|
1670
|
+
return db
|
|
1671
|
+
.prepare(
|
|
1672
|
+
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
|
|
1673
|
+
)
|
|
1674
|
+
.all()
|
|
1675
|
+
.map((row) => row.name);
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
function assertDatabaseIntegrity(db) {
|
|
1679
|
+
let rows;
|
|
1680
|
+
try {
|
|
1681
|
+
rows = db.pragma("quick_check(1)");
|
|
1682
|
+
} catch (cause) {
|
|
1683
|
+
throw schemaError(
|
|
1684
|
+
"SCHEDULER_SCHEMA_CORRUPT",
|
|
1685
|
+
"Scheduler database integrity check could not run",
|
|
1686
|
+
cause,
|
|
1687
|
+
);
|
|
1688
|
+
}
|
|
1689
|
+
if (
|
|
1690
|
+
!Array.isArray(rows) ||
|
|
1691
|
+
rows.length !== 1 ||
|
|
1692
|
+
rows[0]?.quick_check !== "ok"
|
|
1693
|
+
) {
|
|
1694
|
+
throw schemaError(
|
|
1695
|
+
"SCHEDULER_SCHEMA_CORRUPT",
|
|
1696
|
+
"Scheduler database failed SQLite quick_check",
|
|
1697
|
+
undefined,
|
|
1698
|
+
{ result: rows },
|
|
1699
|
+
);
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
function verifyTableShape(db, table, version) {
|
|
1704
|
+
const actual = db
|
|
1705
|
+
.pragma(`table_info(${JSON.stringify(table)})`)
|
|
1706
|
+
.map((column) => [
|
|
1707
|
+
column.name,
|
|
1708
|
+
String(column.type || "").toUpperCase(),
|
|
1709
|
+
column.notnull,
|
|
1710
|
+
column.pk,
|
|
1711
|
+
]);
|
|
1712
|
+
const expected =
|
|
1713
|
+
version < 5 && table === "scheduler_domain_migration_entries"
|
|
1714
|
+
? EXPECTED_COLUMNS[table].filter(
|
|
1715
|
+
([name]) => name !== "source_locator_json",
|
|
1716
|
+
)
|
|
1717
|
+
: EXPECTED_COLUMNS[table];
|
|
1718
|
+
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
|
1719
|
+
throw schemaError(
|
|
1720
|
+
"SCHEDULER_SCHEMA_CORRUPT",
|
|
1721
|
+
`Scheduler table shape is invalid: ${table}`,
|
|
1722
|
+
undefined,
|
|
1723
|
+
{ expected, actual },
|
|
1724
|
+
);
|
|
1725
|
+
}
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
function schemaFingerprint(db) {
|
|
1729
|
+
const catalog = db
|
|
1730
|
+
.prepare(
|
|
1731
|
+
"SELECT type, name, tbl_name, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name",
|
|
1732
|
+
)
|
|
1733
|
+
.all()
|
|
1734
|
+
.map((row) => ({
|
|
1735
|
+
type: row.type,
|
|
1736
|
+
name: row.name,
|
|
1737
|
+
tbl_name: row.tbl_name,
|
|
1738
|
+
sql: String(row.sql || "")
|
|
1739
|
+
.replace(/\s+/g, " ")
|
|
1740
|
+
.trim(),
|
|
1741
|
+
}));
|
|
1742
|
+
return createHash("sha256")
|
|
1743
|
+
.update(JSON.stringify(catalog), "utf8")
|
|
1744
|
+
.digest("hex");
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
function verifySchema(db, version = SCHEDULER_STORE_SCHEMA_VERSION) {
|
|
1748
|
+
assertDatabaseIntegrity(db);
|
|
1749
|
+
const tables = listUserTables(db);
|
|
1750
|
+
const expectedTables =
|
|
1751
|
+
version === 1
|
|
1752
|
+
? V1_USER_TABLES
|
|
1753
|
+
: version === 2
|
|
1754
|
+
? V2_USER_TABLES
|
|
1755
|
+
: version === 3
|
|
1756
|
+
? V3_USER_TABLES
|
|
1757
|
+
: version <= 5
|
|
1758
|
+
? V4_V5_USER_TABLES
|
|
1759
|
+
: USER_TABLES;
|
|
1760
|
+
if (JSON.stringify(tables) !== JSON.stringify(expectedTables)) {
|
|
1761
|
+
throw schemaError(
|
|
1762
|
+
"SCHEDULER_SCHEMA_UNKNOWN",
|
|
1763
|
+
"Scheduler database has unknown or missing tables",
|
|
1764
|
+
undefined,
|
|
1765
|
+
{ expected: expectedTables, actual: tables },
|
|
1766
|
+
);
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
let applicationId;
|
|
1770
|
+
let userVersion;
|
|
677
1771
|
try {
|
|
678
1772
|
applicationId = db.pragma("application_id", { simple: true });
|
|
679
1773
|
userVersion = db.pragma("user_version", { simple: true });
|
|
@@ -693,11 +1787,21 @@ function verifySchema(db, version = SCHEDULER_STORE_SCHEMA_VERSION) {
|
|
|
693
1787
|
);
|
|
694
1788
|
}
|
|
695
1789
|
|
|
696
|
-
for (const table of expectedTables) verifyTableShape(db, table);
|
|
1790
|
+
for (const table of expectedTables) verifyTableShape(db, table, version);
|
|
697
1791
|
|
|
698
1792
|
const actualFingerprint = schemaFingerprint(db);
|
|
699
1793
|
const expectedFingerprint =
|
|
700
|
-
version === 1
|
|
1794
|
+
version === 1
|
|
1795
|
+
? SCHEMA_V1_FINGERPRINT
|
|
1796
|
+
: version === 2
|
|
1797
|
+
? SCHEMA_V2_FINGERPRINT
|
|
1798
|
+
: version === 3
|
|
1799
|
+
? SCHEMA_V3_FINGERPRINT
|
|
1800
|
+
: version === 4
|
|
1801
|
+
? SCHEMA_V4_FINGERPRINT
|
|
1802
|
+
: version === 5
|
|
1803
|
+
? SCHEMA_V5_FINGERPRINT
|
|
1804
|
+
: SCHEMA_V6_FINGERPRINT;
|
|
701
1805
|
if (actualFingerprint !== expectedFingerprint) {
|
|
702
1806
|
throw schemaError(
|
|
703
1807
|
"SCHEDULER_SCHEMA_CORRUPT",
|
|
@@ -729,10 +1833,26 @@ function verifySchema(db, version = SCHEDULER_STORE_SCHEMA_VERSION) {
|
|
|
729
1833
|
migrations[0].version !== 1 ||
|
|
730
1834
|
migrations[0].name !== MIGRATION_V1_NAME ||
|
|
731
1835
|
migrations[0].checksum !== MIGRATION_V1_CHECKSUM ||
|
|
732
|
-
(version
|
|
1836
|
+
(version >= 2 &&
|
|
733
1837
|
(migrations[1]?.version !== 2 ||
|
|
734
1838
|
migrations[1]?.name !== MIGRATION_V2_NAME ||
|
|
735
|
-
migrations[1]?.checksum !== MIGRATION_V2_CHECKSUM))
|
|
1839
|
+
migrations[1]?.checksum !== MIGRATION_V2_CHECKSUM)) ||
|
|
1840
|
+
(version >= 3 &&
|
|
1841
|
+
(migrations[2]?.version !== 3 ||
|
|
1842
|
+
migrations[2]?.name !== MIGRATION_V3_NAME ||
|
|
1843
|
+
migrations[2]?.checksum !== MIGRATION_V3_CHECKSUM)) ||
|
|
1844
|
+
(version >= 4 &&
|
|
1845
|
+
(migrations[3]?.version !== 4 ||
|
|
1846
|
+
migrations[3]?.name !== MIGRATION_V4_NAME ||
|
|
1847
|
+
migrations[3]?.checksum !== MIGRATION_V4_CHECKSUM)) ||
|
|
1848
|
+
(version >= 5 &&
|
|
1849
|
+
(migrations[4]?.version !== 5 ||
|
|
1850
|
+
migrations[4]?.name !== MIGRATION_V5_NAME ||
|
|
1851
|
+
migrations[4]?.checksum !== MIGRATION_V5_CHECKSUM)) ||
|
|
1852
|
+
(version >= 6 &&
|
|
1853
|
+
(migrations[5]?.version !== 6 ||
|
|
1854
|
+
migrations[5]?.name !== MIGRATION_V6_NAME ||
|
|
1855
|
+
migrations[5]?.checksum !== MIGRATION_V6_CHECKSUM))
|
|
736
1856
|
) {
|
|
737
1857
|
throw schemaError(
|
|
738
1858
|
"SCHEDULER_SCHEMA_UNKNOWN",
|
|
@@ -748,12 +1868,14 @@ function verifySchema(db, version = SCHEDULER_STORE_SCHEMA_VERSION) {
|
|
|
748
1868
|
)
|
|
749
1869
|
.all()
|
|
750
1870
|
.map((row) => row.name);
|
|
751
|
-
const expectedIndexes =
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
1871
|
+
const expectedIndexes = EXPECTED_INDEXES.filter(
|
|
1872
|
+
(name) =>
|
|
1873
|
+
(version >= 2 || !name.startsWith("scheduler_authority_")) &&
|
|
1874
|
+
(version >= 3 || !name.startsWith("scheduler_adjudications_")) &&
|
|
1875
|
+
(version >= 4 || !name.startsWith("scheduler_domain_migration")) &&
|
|
1876
|
+
(version >= 6 || !name.startsWith("scheduler_occurrence_controls_")) &&
|
|
1877
|
+
(version >= 6 || !name.startsWith("scheduler_occurrence_retries_")),
|
|
1878
|
+
);
|
|
757
1879
|
if (JSON.stringify(indexes) !== JSON.stringify(expectedIndexes)) {
|
|
758
1880
|
throw schemaError(
|
|
759
1881
|
"SCHEDULER_SCHEMA_CORRUPT",
|
|
@@ -860,6 +1982,22 @@ function migrateAuthorityV2(db, now) {
|
|
|
860
1982
|
}
|
|
861
1983
|
}
|
|
862
1984
|
|
|
1985
|
+
function migrateAdjudicationV3(db) {
|
|
1986
|
+
db.exec(MIGRATION_V3_SQL);
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
function migrateDomainMigrationV4(db) {
|
|
1990
|
+
db.exec(MIGRATION_V4_SQL);
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1993
|
+
function migrateSourceLocatorV5(db) {
|
|
1994
|
+
db.exec(MIGRATION_V5_SQL);
|
|
1995
|
+
}
|
|
1996
|
+
|
|
1997
|
+
function migrateRuntimeControlV6(db) {
|
|
1998
|
+
db.exec(MIGRATION_V6_SQL);
|
|
1999
|
+
}
|
|
2000
|
+
|
|
863
2001
|
function initializeOrVerifySchema(db, now) {
|
|
864
2002
|
assertDatabaseIntegrity(db);
|
|
865
2003
|
const tables = listUserTables(db);
|
|
@@ -867,12 +2005,28 @@ function initializeOrVerifySchema(db, now) {
|
|
|
867
2005
|
const migrate = db.transaction(() => {
|
|
868
2006
|
db.exec(MIGRATION_V1_SQL);
|
|
869
2007
|
migrateAuthorityV2(db, now);
|
|
2008
|
+
migrateAdjudicationV3(db);
|
|
2009
|
+
migrateDomainMigrationV4(db);
|
|
2010
|
+
migrateSourceLocatorV5(db);
|
|
2011
|
+
migrateRuntimeControlV6(db);
|
|
870
2012
|
db.prepare(
|
|
871
2013
|
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
872
2014
|
).run(1, MIGRATION_V1_NAME, MIGRATION_V1_CHECKSUM, now);
|
|
873
2015
|
db.prepare(
|
|
874
2016
|
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
875
2017
|
).run(2, MIGRATION_V2_NAME, MIGRATION_V2_CHECKSUM, now);
|
|
2018
|
+
db.prepare(
|
|
2019
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2020
|
+
).run(3, MIGRATION_V3_NAME, MIGRATION_V3_CHECKSUM, now);
|
|
2021
|
+
db.prepare(
|
|
2022
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2023
|
+
).run(4, MIGRATION_V4_NAME, MIGRATION_V4_CHECKSUM, now);
|
|
2024
|
+
db.prepare(
|
|
2025
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2026
|
+
).run(5, MIGRATION_V5_NAME, MIGRATION_V5_CHECKSUM, now);
|
|
2027
|
+
db.prepare(
|
|
2028
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2029
|
+
).run(6, MIGRATION_V6_NAME, MIGRATION_V6_CHECKSUM, now);
|
|
876
2030
|
db.pragma(`application_id = ${SCHEDULER_APPLICATION_ID}`);
|
|
877
2031
|
db.pragma(`user_version = ${SCHEDULER_STORE_SCHEMA_VERSION}`);
|
|
878
2032
|
});
|
|
@@ -888,13 +2042,93 @@ function initializeOrVerifySchema(db, now) {
|
|
|
888
2042
|
verifySchema(db, 1);
|
|
889
2043
|
const migrate = db.transaction(() => {
|
|
890
2044
|
migrateAuthorityV2(db, now);
|
|
2045
|
+
migrateAdjudicationV3(db);
|
|
2046
|
+
migrateDomainMigrationV4(db);
|
|
2047
|
+
migrateSourceLocatorV5(db);
|
|
2048
|
+
migrateRuntimeControlV6(db);
|
|
891
2049
|
db.prepare(
|
|
892
2050
|
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
893
2051
|
).run(2, MIGRATION_V2_NAME, MIGRATION_V2_CHECKSUM, now);
|
|
2052
|
+
db.prepare(
|
|
2053
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2054
|
+
).run(3, MIGRATION_V3_NAME, MIGRATION_V3_CHECKSUM, now);
|
|
2055
|
+
db.prepare(
|
|
2056
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2057
|
+
).run(4, MIGRATION_V4_NAME, MIGRATION_V4_CHECKSUM, now);
|
|
2058
|
+
db.prepare(
|
|
2059
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2060
|
+
).run(5, MIGRATION_V5_NAME, MIGRATION_V5_CHECKSUM, now);
|
|
2061
|
+
db.prepare(
|
|
2062
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2063
|
+
).run(6, MIGRATION_V6_NAME, MIGRATION_V6_CHECKSUM, now);
|
|
894
2064
|
db.pragma(`user_version = ${SCHEDULER_STORE_SCHEMA_VERSION}`);
|
|
895
2065
|
});
|
|
896
2066
|
migrate.immediate();
|
|
897
|
-
}
|
|
2067
|
+
} else if (db.pragma("user_version", { simple: true }) === 2) {
|
|
2068
|
+
verifySchema(db, 2);
|
|
2069
|
+
const migrate = db.transaction(() => {
|
|
2070
|
+
migrateAdjudicationV3(db);
|
|
2071
|
+
migrateDomainMigrationV4(db);
|
|
2072
|
+
migrateSourceLocatorV5(db);
|
|
2073
|
+
migrateRuntimeControlV6(db);
|
|
2074
|
+
db.prepare(
|
|
2075
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2076
|
+
).run(3, MIGRATION_V3_NAME, MIGRATION_V3_CHECKSUM, now);
|
|
2077
|
+
db.prepare(
|
|
2078
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2079
|
+
).run(4, MIGRATION_V4_NAME, MIGRATION_V4_CHECKSUM, now);
|
|
2080
|
+
db.prepare(
|
|
2081
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2082
|
+
).run(5, MIGRATION_V5_NAME, MIGRATION_V5_CHECKSUM, now);
|
|
2083
|
+
db.prepare(
|
|
2084
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2085
|
+
).run(6, MIGRATION_V6_NAME, MIGRATION_V6_CHECKSUM, now);
|
|
2086
|
+
db.pragma(`user_version = ${SCHEDULER_STORE_SCHEMA_VERSION}`);
|
|
2087
|
+
});
|
|
2088
|
+
migrate.immediate();
|
|
2089
|
+
} else if (db.pragma("user_version", { simple: true }) === 3) {
|
|
2090
|
+
verifySchema(db, 3);
|
|
2091
|
+
const migrate = db.transaction(() => {
|
|
2092
|
+
migrateDomainMigrationV4(db);
|
|
2093
|
+
migrateSourceLocatorV5(db);
|
|
2094
|
+
migrateRuntimeControlV6(db);
|
|
2095
|
+
db.prepare(
|
|
2096
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2097
|
+
).run(4, MIGRATION_V4_NAME, MIGRATION_V4_CHECKSUM, now);
|
|
2098
|
+
db.prepare(
|
|
2099
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2100
|
+
).run(5, MIGRATION_V5_NAME, MIGRATION_V5_CHECKSUM, now);
|
|
2101
|
+
db.prepare(
|
|
2102
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2103
|
+
).run(6, MIGRATION_V6_NAME, MIGRATION_V6_CHECKSUM, now);
|
|
2104
|
+
db.pragma(`user_version = ${SCHEDULER_STORE_SCHEMA_VERSION}`);
|
|
2105
|
+
});
|
|
2106
|
+
migrate.immediate();
|
|
2107
|
+
} else if (db.pragma("user_version", { simple: true }) === 4) {
|
|
2108
|
+
verifySchema(db, 4);
|
|
2109
|
+
const migrate = db.transaction(() => {
|
|
2110
|
+
migrateSourceLocatorV5(db);
|
|
2111
|
+
migrateRuntimeControlV6(db);
|
|
2112
|
+
db.prepare(
|
|
2113
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2114
|
+
).run(5, MIGRATION_V5_NAME, MIGRATION_V5_CHECKSUM, now);
|
|
2115
|
+
db.prepare(
|
|
2116
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2117
|
+
).run(6, MIGRATION_V6_NAME, MIGRATION_V6_CHECKSUM, now);
|
|
2118
|
+
db.pragma(`user_version = ${SCHEDULER_STORE_SCHEMA_VERSION}`);
|
|
2119
|
+
});
|
|
2120
|
+
migrate.immediate();
|
|
2121
|
+
} else if (db.pragma("user_version", { simple: true }) === 5) {
|
|
2122
|
+
verifySchema(db, 5);
|
|
2123
|
+
const migrate = db.transaction(() => {
|
|
2124
|
+
migrateRuntimeControlV6(db);
|
|
2125
|
+
db.prepare(
|
|
2126
|
+
"INSERT INTO migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)",
|
|
2127
|
+
).run(6, MIGRATION_V6_NAME, MIGRATION_V6_CHECKSUM, now);
|
|
2128
|
+
db.pragma(`user_version = ${SCHEDULER_STORE_SCHEMA_VERSION}`);
|
|
2129
|
+
});
|
|
2130
|
+
migrate.immediate();
|
|
2131
|
+
}
|
|
898
2132
|
verifySchema(db);
|
|
899
2133
|
}
|
|
900
2134
|
|
|
@@ -929,11 +2163,21 @@ function openDatabase({ file, Database, busyTimeoutMs, clock }) {
|
|
|
929
2163
|
}
|
|
930
2164
|
const now = normalizeEpochMs(clock(), "clock result");
|
|
931
2165
|
if (target !== ":memory:") {
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
2166
|
+
try {
|
|
2167
|
+
ensurePrivateDirectory(dirname(target));
|
|
2168
|
+
// Existing WAL/SHM files participate in the next SQLite open, so validate
|
|
2169
|
+
// them before the native driver consumes either sidecar.
|
|
2170
|
+
for (const candidate of sqliteStorageFiles(target)) {
|
|
2171
|
+
if (existsSync(candidate)) ensurePrivateFile(candidate);
|
|
2172
|
+
}
|
|
2173
|
+
} catch (cause) {
|
|
2174
|
+
const storageCode = findStorageFailureCode(cause);
|
|
2175
|
+
if (!storageCode) throw cause;
|
|
2176
|
+
throw storageUnavailableError({
|
|
2177
|
+
phase: "open",
|
|
2178
|
+
storageCode,
|
|
2179
|
+
commitState: "unknown",
|
|
2180
|
+
});
|
|
937
2181
|
}
|
|
938
2182
|
}
|
|
939
2183
|
|
|
@@ -954,6 +2198,14 @@ function openDatabase({ file, Database, busyTimeoutMs, clock }) {
|
|
|
954
2198
|
// Preserve the authoritative open/schema failure.
|
|
955
2199
|
}
|
|
956
2200
|
if (cause instanceof SchedulerKernelError) throw cause;
|
|
2201
|
+
const storageCode = findStorageFailureCode(cause);
|
|
2202
|
+
if (storageCode) {
|
|
2203
|
+
throw storageUnavailableError({
|
|
2204
|
+
phase: "open",
|
|
2205
|
+
storageCode,
|
|
2206
|
+
commitState: "unknown",
|
|
2207
|
+
});
|
|
2208
|
+
}
|
|
957
2209
|
throw schemaError(
|
|
958
2210
|
"SCHEDULER_SCHEMA_CORRUPT",
|
|
959
2211
|
"Scheduler database could not be opened or verified",
|
|
@@ -962,12 +2214,27 @@ function openDatabase({ file, Database, busyTimeoutMs, clock }) {
|
|
|
962
2214
|
}
|
|
963
2215
|
|
|
964
2216
|
if (target !== ":memory:") {
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
2217
|
+
try {
|
|
2218
|
+
// A quiet WAL database may not have materialized sidecars yet. On Windows,
|
|
2219
|
+
// asking secure-fs to classify a missing path deliberately falls back to a
|
|
2220
|
+
// native ACL preflight. Only inspect files that SQLite actually created;
|
|
2221
|
+
// later opens repeat this check for any surviving sidecars.
|
|
2222
|
+
for (const candidate of sqliteStorageFiles(target)) {
|
|
2223
|
+
if (existsSync(candidate)) ensurePrivateFile(candidate);
|
|
2224
|
+
}
|
|
2225
|
+
} catch (cause) {
|
|
2226
|
+
try {
|
|
2227
|
+
db.close();
|
|
2228
|
+
} catch {
|
|
2229
|
+
// Preserve the authoritative storage or security failure.
|
|
2230
|
+
}
|
|
2231
|
+
const storageCode = findStorageFailureCode(cause);
|
|
2232
|
+
if (!storageCode) throw cause;
|
|
2233
|
+
throw storageUnavailableError({
|
|
2234
|
+
phase: "open",
|
|
2235
|
+
storageCode,
|
|
2236
|
+
commitState: "unknown",
|
|
2237
|
+
});
|
|
971
2238
|
}
|
|
972
2239
|
}
|
|
973
2240
|
return { db, target };
|
|
@@ -994,12 +2261,35 @@ export class SchedulerStore {
|
|
|
994
2261
|
getOccurrenceByKey: db.prepare(
|
|
995
2262
|
"SELECT * FROM occurrences WHERE idempotency_key = ?",
|
|
996
2263
|
),
|
|
2264
|
+
getOccurrenceControl: db.prepare(
|
|
2265
|
+
"SELECT * FROM scheduler_occurrence_controls WHERE occurrence_id = ?",
|
|
2266
|
+
),
|
|
2267
|
+
getOccurrenceControlByPauseRequest: db.prepare(
|
|
2268
|
+
"SELECT * FROM scheduler_occurrence_controls WHERE pause_request_id = ?",
|
|
2269
|
+
),
|
|
2270
|
+
getOccurrenceControlByResumeRequest: db.prepare(
|
|
2271
|
+
"SELECT * FROM scheduler_occurrence_controls WHERE resume_request_id = ?",
|
|
2272
|
+
),
|
|
2273
|
+
getOccurrenceRetryByRequest: db.prepare(
|
|
2274
|
+
"SELECT * FROM scheduler_occurrence_retries WHERE request_id = ?",
|
|
2275
|
+
),
|
|
997
2276
|
listOccurrencesByTrigger: db.prepare(
|
|
998
2277
|
`SELECT * FROM occurrences
|
|
999
2278
|
WHERE job_id = ? AND trigger_key = ?
|
|
1000
2279
|
ORDER BY created_at ASC, occurrence_id ASC
|
|
1001
2280
|
LIMIT ?`,
|
|
1002
2281
|
),
|
|
2282
|
+
getDomainMigration: db.prepare(
|
|
2283
|
+
"SELECT * FROM scheduler_domain_migrations WHERE migration_id = ?",
|
|
2284
|
+
),
|
|
2285
|
+
getDomainMigrationEntry: db.prepare(
|
|
2286
|
+
`SELECT * FROM scheduler_domain_migration_entries
|
|
2287
|
+
WHERE migration_id = ? AND entry_id = ?`,
|
|
2288
|
+
),
|
|
2289
|
+
listDomainMigrationEntries: db.prepare(
|
|
2290
|
+
`SELECT * FROM scheduler_domain_migration_entries
|
|
2291
|
+
WHERE migration_id = ? ORDER BY entry_id`,
|
|
2292
|
+
),
|
|
1003
2293
|
insertEvent: db.prepare(`
|
|
1004
2294
|
INSERT INTO events
|
|
1005
2295
|
(job_id, occurrence_id, event_type, occurred_at, owner_id, fence, data_json)
|
|
@@ -1024,8 +2314,46 @@ export class SchedulerStore {
|
|
|
1024
2314
|
|
|
1025
2315
|
_write(callback) {
|
|
1026
2316
|
this._assertOpen();
|
|
1027
|
-
|
|
1028
|
-
|
|
2317
|
+
let bodyState = "not_started";
|
|
2318
|
+
let bodyObservedTransaction = false;
|
|
2319
|
+
try {
|
|
2320
|
+
const transaction = this.db.transaction((...args) => {
|
|
2321
|
+
bodyState = "running";
|
|
2322
|
+
try {
|
|
2323
|
+
bodyObservedTransaction = this.db.inTransaction === true;
|
|
2324
|
+
} catch {
|
|
2325
|
+
bodyObservedTransaction = false;
|
|
2326
|
+
}
|
|
2327
|
+
const result = callback(...args);
|
|
2328
|
+
bodyState = "returned";
|
|
2329
|
+
return result;
|
|
2330
|
+
});
|
|
2331
|
+
return transaction.immediate();
|
|
2332
|
+
} catch (cause) {
|
|
2333
|
+
if (cause instanceof SchedulerKernelError) throw cause;
|
|
2334
|
+
const storageCode = findStorageFailureCode(cause);
|
|
2335
|
+
if (!storageCode) throw cause;
|
|
2336
|
+
let transactionClosed = false;
|
|
2337
|
+
try {
|
|
2338
|
+
transactionClosed = this.db.inTransaction === false;
|
|
2339
|
+
} catch {
|
|
2340
|
+
transactionClosed = false;
|
|
2341
|
+
}
|
|
2342
|
+
// Once the body returns, the wrapper may already be in COMMIT and an I/O
|
|
2343
|
+
// failure cannot distinguish an applied commit from an aborted one. A
|
|
2344
|
+
// body failure is only safe to call not-committed when we observed the
|
|
2345
|
+
// native transaction and can also observe that its rollback closed it.
|
|
2346
|
+
throw storageUnavailableError({
|
|
2347
|
+
phase: "write",
|
|
2348
|
+
storageCode,
|
|
2349
|
+
commitState:
|
|
2350
|
+
bodyState === "running" &&
|
|
2351
|
+
bodyObservedTransaction &&
|
|
2352
|
+
transactionClosed
|
|
2353
|
+
? "not_committed"
|
|
2354
|
+
: "unknown",
|
|
2355
|
+
});
|
|
2356
|
+
}
|
|
1029
2357
|
}
|
|
1030
2358
|
|
|
1031
2359
|
_appendEvent({
|
|
@@ -1172,148 +2500,1942 @@ export class SchedulerStore {
|
|
|
1172
2500
|
});
|
|
1173
2501
|
}
|
|
1174
2502
|
|
|
1175
|
-
|
|
1176
|
-
const
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
2503
|
+
_domainMigration(migrationId) {
|
|
2504
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
2505
|
+
const row = this.statements.getDomainMigration.get(id);
|
|
2506
|
+
if (!row) {
|
|
2507
|
+
throw schedulerMigrationError(
|
|
2508
|
+
"SCHEDULER_MIGRATION_NOT_FOUND",
|
|
2509
|
+
`Scheduler domain migration does not exist: ${id}`,
|
|
1181
2510
|
);
|
|
1182
2511
|
}
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
2512
|
+
return mapDomainMigration(
|
|
2513
|
+
row,
|
|
2514
|
+
this.statements.listDomainMigrationEntries
|
|
2515
|
+
.all(id)
|
|
2516
|
+
.map(mapDomainMigrationEntry),
|
|
2517
|
+
);
|
|
2518
|
+
}
|
|
2519
|
+
|
|
2520
|
+
getDomainMigration(migrationId) {
|
|
2521
|
+
this._assertOpen();
|
|
2522
|
+
return this._domainMigration(migrationId);
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
getActiveDomainMigrationBySource({ domain, sourceScope, sourceId } = {}) {
|
|
2526
|
+
this._assertOpen();
|
|
2527
|
+
if (!SCHEDULER_MIGRATION_DOMAINS.includes(domain)) {
|
|
2528
|
+
throw invalidArgument(
|
|
2529
|
+
"domain must be a supported scheduler migration domain",
|
|
1187
2530
|
);
|
|
1188
2531
|
}
|
|
1189
|
-
const
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
scheduledFor: scheduled,
|
|
1193
|
-
triggerKey: key,
|
|
1194
|
-
});
|
|
1195
|
-
const payloadJson = canonicalJson(
|
|
1196
|
-
payload === undefined ? job.payload : payload,
|
|
1197
|
-
"occurrence.payload",
|
|
1198
|
-
);
|
|
1199
|
-
const result = this.db
|
|
2532
|
+
const scopeDigest = schedulerMigrationScopeDigest(sourceScope);
|
|
2533
|
+
const id = normalizeIdentifier(sourceId, "sourceId");
|
|
2534
|
+
const row = this.db
|
|
1200
2535
|
.prepare(
|
|
1201
|
-
`
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
scheduled_for, available_at, status, attempt, max_attempts, fence,
|
|
1205
|
-
lease_owner, lease_expires_at, authority_json, payload_json,
|
|
1206
|
-
last_error_json, result_json, created_at, updated_at, settled_at)
|
|
1207
|
-
VALUES
|
|
1208
|
-
(@occurrenceId, @jobId, @jobRevision, @idempotencyKey, @triggerKey,
|
|
1209
|
-
@scheduledFor, @availableAt, 'queued', 0, @maxAttempts, 0,
|
|
1210
|
-
NULL, NULL, @authorityJson, @payloadJson,
|
|
1211
|
-
NULL, NULL, @now, @now, NULL)
|
|
1212
|
-
ON CONFLICT(idempotency_key) DO NOTHING
|
|
1213
|
-
`,
|
|
2536
|
+
`SELECT migration_id FROM scheduler_domain_migration_entries
|
|
2537
|
+
WHERE domain = ? AND source_scope_digest = ? AND source_id = ?
|
|
2538
|
+
AND state <> 'rolled_back'`,
|
|
1214
2539
|
)
|
|
1215
|
-
.
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
const
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
2540
|
+
.get(domain, scopeDigest, id);
|
|
2541
|
+
return row ? this._domainMigration(row.migration_id) : null;
|
|
2542
|
+
}
|
|
2543
|
+
|
|
2544
|
+
bindDomainMigrationSourceLocator({
|
|
2545
|
+
migrationId,
|
|
2546
|
+
entryId,
|
|
2547
|
+
sourceLocator,
|
|
2548
|
+
expectedSourceDigest,
|
|
2549
|
+
expectedTargetJobId,
|
|
2550
|
+
} = {}) {
|
|
2551
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
2552
|
+
const normalizedEntryId = normalizeIdentifier(entryId, "entryId");
|
|
2553
|
+
const sourceDigest = normalizeSha256Digest(
|
|
2554
|
+
expectedSourceDigest,
|
|
2555
|
+
"expectedSourceDigest",
|
|
2556
|
+
);
|
|
2557
|
+
const targetJobId = normalizeIdentifier(
|
|
2558
|
+
expectedTargetJobId,
|
|
2559
|
+
"expectedTargetJobId",
|
|
2560
|
+
);
|
|
2561
|
+
const now = this._now();
|
|
2562
|
+
return this._write(() => {
|
|
2563
|
+
const row = this.statements.getDomainMigrationEntry.get(
|
|
2564
|
+
id,
|
|
2565
|
+
normalizedEntryId,
|
|
1232
2566
|
);
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
2567
|
+
if (!row) {
|
|
2568
|
+
throw schedulerMigrationError(
|
|
2569
|
+
"SCHEDULER_MIGRATION_NOT_FOUND",
|
|
2570
|
+
`Scheduler migration entry does not exist: ${normalizedEntryId}`,
|
|
2571
|
+
);
|
|
2572
|
+
}
|
|
2573
|
+
if (
|
|
2574
|
+
row.source_digest !== sourceDigest ||
|
|
2575
|
+
row.target_job_id !== targetJobId
|
|
2576
|
+
) {
|
|
2577
|
+
throw schedulerMigrationError(
|
|
2578
|
+
"SCHEDULER_MIGRATION_SOURCE_MISMATCH",
|
|
2579
|
+
"Scheduler migration locator evidence does not match the journal",
|
|
2580
|
+
);
|
|
2581
|
+
}
|
|
2582
|
+
const locator = normalizeSchedulerMigrationSourceLocator(
|
|
2583
|
+
sourceLocator,
|
|
2584
|
+
row.domain,
|
|
2585
|
+
row.source_scope_digest,
|
|
2586
|
+
"sourceLocator",
|
|
2587
|
+
{ allowLegacyAutomationCurrent: true },
|
|
1239
2588
|
);
|
|
2589
|
+
if (locator === null) {
|
|
2590
|
+
throw invalidArgument("sourceLocator is required");
|
|
2591
|
+
}
|
|
2592
|
+
const encoded = canonicalJson(
|
|
2593
|
+
locator,
|
|
2594
|
+
"schedulerMigration.sourceLocator",
|
|
2595
|
+
);
|
|
2596
|
+
if (row.source_locator_json !== null) {
|
|
2597
|
+
if (row.source_locator_json !== encoded) {
|
|
2598
|
+
throw schedulerMigrationError(
|
|
2599
|
+
"SCHEDULER_MIGRATION_LOCATOR_CONFLICT",
|
|
2600
|
+
"Scheduler migration source locator is already bound differently",
|
|
2601
|
+
);
|
|
2602
|
+
}
|
|
2603
|
+
return {
|
|
2604
|
+
...mapDomainMigrationEntry(row),
|
|
2605
|
+
deduplicated: true,
|
|
2606
|
+
};
|
|
2607
|
+
}
|
|
2608
|
+
const result = this.db
|
|
2609
|
+
.prepare(
|
|
2610
|
+
`UPDATE scheduler_domain_migration_entries
|
|
2611
|
+
SET source_locator_json = ?, updated_at = ?
|
|
2612
|
+
WHERE migration_id = ? AND entry_id = ?
|
|
2613
|
+
AND source_locator_json IS NULL
|
|
2614
|
+
AND source_digest = ? AND target_job_id = ?`,
|
|
2615
|
+
)
|
|
2616
|
+
.run(encoded, now, id, normalizedEntryId, sourceDigest, targetJobId);
|
|
2617
|
+
if (result.changes !== 1) {
|
|
2618
|
+
throw schedulerMigrationError(
|
|
2619
|
+
"SCHEDULER_MIGRATION_LOCATOR_CONFLICT",
|
|
2620
|
+
"Scheduler migration source locator changed during binding",
|
|
2621
|
+
);
|
|
2622
|
+
}
|
|
2623
|
+
this.db
|
|
2624
|
+
.prepare(
|
|
2625
|
+
`UPDATE scheduler_domain_migrations
|
|
2626
|
+
SET updated_at = ? WHERE migration_id = ?`,
|
|
2627
|
+
)
|
|
2628
|
+
.run(now, id);
|
|
2629
|
+
return {
|
|
2630
|
+
...mapDomainMigrationEntry(
|
|
2631
|
+
this.statements.getDomainMigrationEntry.get(id, normalizedEntryId),
|
|
2632
|
+
),
|
|
2633
|
+
deduplicated: false,
|
|
2634
|
+
};
|
|
2635
|
+
});
|
|
2636
|
+
}
|
|
2637
|
+
|
|
2638
|
+
listDomainMigrations({ state, limit = 50 } = {}) {
|
|
2639
|
+
this._assertOpen();
|
|
2640
|
+
const boundedLimit = Math.min(
|
|
2641
|
+
200,
|
|
2642
|
+
Math.max(1, Number.isSafeInteger(Number(limit)) ? Number(limit) : 50),
|
|
2643
|
+
);
|
|
2644
|
+
let normalizedState = null;
|
|
2645
|
+
if (state !== undefined) {
|
|
2646
|
+
normalizedState = normalizeIdentifier(state, "state", { maxLength: 32 });
|
|
2647
|
+
if (
|
|
2648
|
+
!Object.values(SCHEDULER_MIGRATION_STATES).includes(normalizedState)
|
|
2649
|
+
) {
|
|
2650
|
+
throw invalidArgument("state is not a scheduler migration state");
|
|
2651
|
+
}
|
|
1240
2652
|
}
|
|
1241
|
-
|
|
1242
|
-
this.
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
return
|
|
2653
|
+
const rows = normalizedState
|
|
2654
|
+
? this.db
|
|
2655
|
+
.prepare(
|
|
2656
|
+
`SELECT * FROM scheduler_domain_migrations
|
|
2657
|
+
WHERE state = ? ORDER BY updated_at DESC, migration_id LIMIT ?`,
|
|
2658
|
+
)
|
|
2659
|
+
.all(normalizedState, boundedLimit)
|
|
2660
|
+
: this.db
|
|
2661
|
+
.prepare(
|
|
2662
|
+
`SELECT * FROM scheduler_domain_migrations
|
|
2663
|
+
ORDER BY updated_at DESC, migration_id LIMIT ?`,
|
|
2664
|
+
)
|
|
2665
|
+
.all(boundedLimit);
|
|
2666
|
+
return rows.map((row) => mapDomainMigration(row));
|
|
1255
2667
|
}
|
|
1256
2668
|
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
scheduledFor,
|
|
1260
|
-
triggerKey,
|
|
1261
|
-
availableAt,
|
|
1262
|
-
payload,
|
|
1263
|
-
} = {}) {
|
|
1264
|
-
const id = normalizeIdentifier(jobId, "jobId");
|
|
1265
|
-
const scheduled = normalizeEpochMs(scheduledFor, "scheduledFor");
|
|
1266
|
-
const key = normalizeIdentifier(triggerKey, "triggerKey");
|
|
1267
|
-
const available = normalizeEpochMs(availableAt ?? scheduled, "availableAt");
|
|
2669
|
+
prepareDomainMigration(input) {
|
|
2670
|
+
const plan = normalizeDomainMigrationPlan(input);
|
|
1268
2671
|
const now = this._now();
|
|
1269
|
-
return this._write(() =>
|
|
1270
|
-
|
|
1271
|
-
|
|
2672
|
+
return this._write(() => {
|
|
2673
|
+
const activeIds = new Set();
|
|
2674
|
+
const findActiveSource = this.db.prepare(
|
|
2675
|
+
`SELECT migration_id FROM scheduler_domain_migration_entries
|
|
2676
|
+
WHERE domain = ? AND source_scope_digest = ? AND source_id = ?
|
|
2677
|
+
AND state <> 'rolled_back'`,
|
|
2678
|
+
);
|
|
2679
|
+
const findActiveTarget = this.db.prepare(
|
|
2680
|
+
`SELECT migration_id FROM scheduler_domain_migration_entries
|
|
2681
|
+
WHERE target_job_id = ? AND state <> 'rolled_back'`,
|
|
2682
|
+
);
|
|
2683
|
+
for (const entry of plan.entries) {
|
|
2684
|
+
const source = findActiveSource.get(
|
|
2685
|
+
entry.domain,
|
|
2686
|
+
entry.sourceScopeDigest,
|
|
2687
|
+
entry.sourceId,
|
|
2688
|
+
);
|
|
2689
|
+
const target = findActiveTarget.get(entry.targetJobId);
|
|
2690
|
+
if (source) activeIds.add(source.migration_id);
|
|
2691
|
+
if (target) activeIds.add(target.migration_id);
|
|
2692
|
+
}
|
|
2693
|
+
if (activeIds.size > 0) {
|
|
2694
|
+
if (activeIds.size !== 1) {
|
|
2695
|
+
throw schedulerMigrationError(
|
|
2696
|
+
"SCHEDULER_MIGRATION_CONFLICT",
|
|
2697
|
+
"Scheduler migration sources or targets belong to different active migrations",
|
|
2698
|
+
);
|
|
2699
|
+
}
|
|
2700
|
+
const [activeId] = activeIds;
|
|
2701
|
+
const active = this.statements.getDomainMigration.get(activeId);
|
|
2702
|
+
if (!active) {
|
|
2703
|
+
throw schemaError(
|
|
2704
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
2705
|
+
"Active scheduler migration entry has no parent journal",
|
|
2706
|
+
);
|
|
2707
|
+
}
|
|
2708
|
+
const activeMigration = this._domainMigration(activeId);
|
|
2709
|
+
const matchedEntries = [];
|
|
2710
|
+
const samePlan =
|
|
2711
|
+
activeMigration.entries.length === plan.entries.length &&
|
|
2712
|
+
activeMigration.entries.every((entry) => {
|
|
2713
|
+
const candidate = plan.entries.find(
|
|
2714
|
+
(item) =>
|
|
2715
|
+
item.domain === entry.domain &&
|
|
2716
|
+
item.sourceId === entry.sourceId &&
|
|
2717
|
+
item.sourceScopeDigest === entry.sourceScopeDigest &&
|
|
2718
|
+
item.targetJobId === entry.targetJobId,
|
|
2719
|
+
);
|
|
2720
|
+
const locatorMatches =
|
|
2721
|
+
candidate &&
|
|
2722
|
+
(entry.sourceLocator === null ||
|
|
2723
|
+
candidate.sourceLocator === null ||
|
|
2724
|
+
canonicalJson(
|
|
2725
|
+
entry.sourceLocator,
|
|
2726
|
+
"schedulerMigration.sourceLocator",
|
|
2727
|
+
) ===
|
|
2728
|
+
canonicalJson(
|
|
2729
|
+
candidate.sourceLocator,
|
|
2730
|
+
"schedulerMigration.sourceLocator",
|
|
2731
|
+
));
|
|
2732
|
+
const matches =
|
|
2733
|
+
candidate &&
|
|
2734
|
+
locatorMatches &&
|
|
2735
|
+
candidate.sourceDigest === entry.sourceDigest &&
|
|
2736
|
+
candidate.targetJobDigest === entry.targetJobDigest &&
|
|
2737
|
+
candidate.rollbackStrategy === entry.rollbackStrategy;
|
|
2738
|
+
if (matches) matchedEntries.push({ entry, candidate });
|
|
2739
|
+
return matches;
|
|
2740
|
+
});
|
|
2741
|
+
if (!samePlan) {
|
|
2742
|
+
const comparison = {
|
|
2743
|
+
active: activeMigration.entries.map((entry) => ({
|
|
2744
|
+
domain: entry.domain,
|
|
2745
|
+
sourceId: entry.sourceId,
|
|
2746
|
+
sourceScopeDigest: entry.sourceScopeDigest,
|
|
2747
|
+
sourceDigest: entry.sourceDigest,
|
|
2748
|
+
targetJobId: entry.targetJobId,
|
|
2749
|
+
targetJobDigest: entry.targetJobDigest,
|
|
2750
|
+
rollbackStrategy: entry.rollbackStrategy,
|
|
2751
|
+
})),
|
|
2752
|
+
requested: plan.entries.map((entry) => ({
|
|
2753
|
+
domain: entry.domain,
|
|
2754
|
+
sourceId: entry.sourceId,
|
|
2755
|
+
sourceScopeDigest: entry.sourceScopeDigest,
|
|
2756
|
+
sourceDigest: entry.sourceDigest,
|
|
2757
|
+
targetJobId: entry.targetJobId,
|
|
2758
|
+
targetJobDigest: entry.targetJobDigest,
|
|
2759
|
+
rollbackStrategy: entry.rollbackStrategy,
|
|
2760
|
+
})),
|
|
2761
|
+
};
|
|
2762
|
+
throw schedulerMigrationError(
|
|
2763
|
+
"SCHEDULER_MIGRATION_CONFLICT",
|
|
2764
|
+
"A scheduler source or target already has a different active migration",
|
|
2765
|
+
comparison,
|
|
2766
|
+
);
|
|
2767
|
+
}
|
|
2768
|
+
let locatorBackfilled = false;
|
|
2769
|
+
const backfillLocator = this.db.prepare(
|
|
2770
|
+
`UPDATE scheduler_domain_migration_entries
|
|
2771
|
+
SET source_locator_json = ?, updated_at = ?
|
|
2772
|
+
WHERE migration_id = ? AND entry_id = ?
|
|
2773
|
+
AND source_locator_json IS NULL`,
|
|
2774
|
+
);
|
|
2775
|
+
for (const { entry, candidate } of matchedEntries) {
|
|
2776
|
+
if (
|
|
2777
|
+
entry.sourceLocator !== null ||
|
|
2778
|
+
candidate.sourceLocator === null
|
|
2779
|
+
) {
|
|
2780
|
+
continue;
|
|
2781
|
+
}
|
|
2782
|
+
const result = backfillLocator.run(
|
|
2783
|
+
canonicalJson(
|
|
2784
|
+
candidate.sourceLocator,
|
|
2785
|
+
"schedulerMigration.sourceLocator",
|
|
2786
|
+
),
|
|
2787
|
+
now,
|
|
2788
|
+
activeId,
|
|
2789
|
+
entry.entryId,
|
|
2790
|
+
);
|
|
2791
|
+
if (result.changes !== 1) {
|
|
2792
|
+
throw schemaError(
|
|
2793
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
2794
|
+
"Scheduler migration source locator changed during binding",
|
|
2795
|
+
);
|
|
2796
|
+
}
|
|
2797
|
+
locatorBackfilled = true;
|
|
2798
|
+
}
|
|
2799
|
+
if (locatorBackfilled) {
|
|
2800
|
+
this.db
|
|
2801
|
+
.prepare(
|
|
2802
|
+
`UPDATE scheduler_domain_migrations
|
|
2803
|
+
SET updated_at = ? WHERE migration_id = ?`,
|
|
2804
|
+
)
|
|
2805
|
+
.run(now, activeId);
|
|
2806
|
+
}
|
|
2807
|
+
return {
|
|
2808
|
+
...(locatorBackfilled
|
|
2809
|
+
? this._domainMigration(activeId)
|
|
2810
|
+
: activeMigration),
|
|
2811
|
+
deduplicated: true,
|
|
2812
|
+
};
|
|
2813
|
+
}
|
|
2814
|
+
|
|
2815
|
+
const attemptManifest = {
|
|
2816
|
+
...plan.manifest,
|
|
2817
|
+
schemaVersion: 2,
|
|
2818
|
+
planDigest: plan.manifestDigest,
|
|
2819
|
+
attemptId: randomUUID(),
|
|
2820
|
+
};
|
|
2821
|
+
const manifestDigest = sha256PayloadDigest(
|
|
2822
|
+
attemptManifest,
|
|
2823
|
+
"schedulerMigration.attemptManifest",
|
|
2824
|
+
);
|
|
2825
|
+
const migrationId = `scheduler-domain-migration-${manifestDigest.slice(
|
|
2826
|
+
"sha256:".length,
|
|
2827
|
+
)}`;
|
|
2828
|
+
this.db
|
|
2829
|
+
.prepare(
|
|
2830
|
+
`INSERT INTO scheduler_domain_migrations
|
|
2831
|
+
(migration_id, manifest_digest, manifest_json, state, entry_count,
|
|
2832
|
+
created_at, updated_at, completed_at, last_error_json)
|
|
2833
|
+
VALUES (?, ?, ?, 'prepared', ?, ?, ?, NULL, NULL)`,
|
|
2834
|
+
)
|
|
2835
|
+
.run(
|
|
2836
|
+
migrationId,
|
|
2837
|
+
manifestDigest,
|
|
2838
|
+
canonicalJson(attemptManifest, "schedulerMigration.manifest"),
|
|
2839
|
+
plan.entries.length,
|
|
2840
|
+
now,
|
|
2841
|
+
now,
|
|
2842
|
+
);
|
|
2843
|
+
const insert = this.db.prepare(
|
|
2844
|
+
`INSERT INTO scheduler_domain_migration_entries
|
|
2845
|
+
(migration_id, entry_id, domain, source_id, source_scope_digest,
|
|
2846
|
+
source_locator_json, source_digest, target_job_id,
|
|
2847
|
+
target_job_digest, rollback_strategy, state,
|
|
2848
|
+
target_action, target_before_json, target_applied_revision,
|
|
2849
|
+
target_applied_at, target_occurrence_count_before,
|
|
2850
|
+
target_execution_event_count_before, target_rollback_revision,
|
|
2851
|
+
retirement_token, source_retirement_digest, source_restored_digest,
|
|
2852
|
+
created_at, updated_at)
|
|
2853
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'prepared',
|
|
2854
|
+
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
|
|
2855
|
+
?, ?)`,
|
|
2856
|
+
);
|
|
2857
|
+
for (const entry of plan.entries) {
|
|
2858
|
+
try {
|
|
2859
|
+
insert.run(
|
|
2860
|
+
migrationId,
|
|
2861
|
+
entry.entryId,
|
|
2862
|
+
entry.domain,
|
|
2863
|
+
entry.sourceId,
|
|
2864
|
+
entry.sourceScopeDigest,
|
|
2865
|
+
entry.sourceLocator === null
|
|
2866
|
+
? null
|
|
2867
|
+
: canonicalJson(
|
|
2868
|
+
entry.sourceLocator,
|
|
2869
|
+
"schedulerMigration.sourceLocator",
|
|
2870
|
+
),
|
|
2871
|
+
entry.sourceDigest,
|
|
2872
|
+
entry.targetJobId,
|
|
2873
|
+
entry.targetJobDigest,
|
|
2874
|
+
entry.rollbackStrategy,
|
|
2875
|
+
now,
|
|
2876
|
+
now,
|
|
2877
|
+
);
|
|
2878
|
+
} catch (cause) {
|
|
2879
|
+
if (String(cause?.code || "").startsWith("SQLITE_CONSTRAINT")) {
|
|
2880
|
+
throw schedulerMigrationError(
|
|
2881
|
+
"SCHEDULER_MIGRATION_CONFLICT",
|
|
2882
|
+
"A scheduler source or target already has an active migration",
|
|
2883
|
+
{
|
|
2884
|
+
domain: entry.domain,
|
|
2885
|
+
sourceId: entry.sourceId,
|
|
2886
|
+
targetJobId: entry.targetJobId,
|
|
2887
|
+
},
|
|
2888
|
+
);
|
|
2889
|
+
}
|
|
2890
|
+
throw cause;
|
|
2891
|
+
}
|
|
2892
|
+
}
|
|
2893
|
+
return {
|
|
2894
|
+
...this._domainMigration(migrationId),
|
|
2895
|
+
deduplicated: false,
|
|
2896
|
+
};
|
|
2897
|
+
});
|
|
1272
2898
|
}
|
|
1273
2899
|
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
2900
|
+
applyDomainMigration(migrationId) {
|
|
2901
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
2902
|
+
const now = this._now();
|
|
2903
|
+
return this._write(() => {
|
|
2904
|
+
const migration = this._domainMigration(id);
|
|
2905
|
+
if (migration.state !== SCHEDULER_MIGRATION_STATES.PREPARED) {
|
|
2906
|
+
if (
|
|
2907
|
+
[
|
|
2908
|
+
SCHEDULER_MIGRATION_STATES.APPLIED,
|
|
2909
|
+
SCHEDULER_MIGRATION_STATES.VERIFIED,
|
|
2910
|
+
SCHEDULER_MIGRATION_STATES.RETIRING,
|
|
2911
|
+
SCHEDULER_MIGRATION_STATES.RETIRED,
|
|
2912
|
+
].includes(migration.state)
|
|
2913
|
+
) {
|
|
2914
|
+
return { ...migration, deduplicated: true };
|
|
2915
|
+
}
|
|
2916
|
+
throw schedulerMigrationError(
|
|
2917
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
2918
|
+
`Scheduler migration cannot be applied from state ${migration.state}`,
|
|
2919
|
+
);
|
|
2920
|
+
}
|
|
2921
|
+
const manifestById = new Map(
|
|
2922
|
+
migration.manifest.entries.map((entry) => [entry.entryId, entry]),
|
|
2923
|
+
);
|
|
2924
|
+
const updateEntry = this.db.prepare(
|
|
2925
|
+
`UPDATE scheduler_domain_migration_entries
|
|
2926
|
+
SET state = 'applied', target_action = ?, target_before_json = ?,
|
|
2927
|
+
target_applied_revision = ?, target_applied_at = ?,
|
|
2928
|
+
target_occurrence_count_before = ?,
|
|
2929
|
+
target_execution_event_count_before = ?, updated_at = ?
|
|
2930
|
+
WHERE migration_id = ? AND entry_id = ? AND state = 'prepared'`,
|
|
2931
|
+
);
|
|
2932
|
+
for (const entry of migration.entries) {
|
|
2933
|
+
const manifestEntry = manifestById.get(entry.entryId);
|
|
2934
|
+
if (!manifestEntry) {
|
|
2935
|
+
throw schemaError(
|
|
2936
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
2937
|
+
"Scheduler migration entry is absent from its manifest",
|
|
2938
|
+
);
|
|
2939
|
+
}
|
|
2940
|
+
const targetJob = normalizeJobInput(manifestEntry.targetJob);
|
|
2941
|
+
if (
|
|
2942
|
+
targetJob.id !== entry.targetJobId ||
|
|
2943
|
+
schedulerJobDefinitionDigest(targetJob) !== entry.targetJobDigest
|
|
2944
|
+
) {
|
|
2945
|
+
throw schemaError(
|
|
2946
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
2947
|
+
"Scheduler migration target does not match its stored digest",
|
|
2948
|
+
);
|
|
2949
|
+
}
|
|
2950
|
+
const current = mapJob(this.statements.getJob.get(entry.targetJobId));
|
|
2951
|
+
const stagedTargetJob = { ...targetJob, enabled: false };
|
|
2952
|
+
const occurrenceCount = this.db
|
|
2953
|
+
.prepare("SELECT COUNT(*) AS count FROM occurrences WHERE job_id = ?")
|
|
2954
|
+
.get(entry.targetJobId).count;
|
|
2955
|
+
const executionEventCount = this.db
|
|
2956
|
+
.prepare(
|
|
2957
|
+
`SELECT COUNT(*) AS count FROM events
|
|
2958
|
+
WHERE job_id = ? AND occurrence_id IS NOT NULL`,
|
|
2959
|
+
)
|
|
2960
|
+
.get(entry.targetJobId).count;
|
|
2961
|
+
let action;
|
|
2962
|
+
let applied;
|
|
2963
|
+
if (!current) {
|
|
2964
|
+
applied = this.createJob(stagedTargetJob);
|
|
2965
|
+
action = "created";
|
|
2966
|
+
} else if (
|
|
2967
|
+
schedulerJobDefinitionDigest(current) ===
|
|
2968
|
+
schedulerJobDefinitionDigest(stagedTargetJob)
|
|
2969
|
+
) {
|
|
2970
|
+
applied = current;
|
|
2971
|
+
action = "reused";
|
|
2972
|
+
} else {
|
|
2973
|
+
applied = this.updateJob(current.id, current.revision, {
|
|
2974
|
+
kind: stagedTargetJob.kind,
|
|
2975
|
+
trigger: stagedTargetJob.trigger,
|
|
2976
|
+
payload: stagedTargetJob.payload,
|
|
2977
|
+
authority: stagedTargetJob.authority,
|
|
2978
|
+
enabled: false,
|
|
2979
|
+
maxAttempts: stagedTargetJob.maxAttempts,
|
|
2980
|
+
});
|
|
2981
|
+
action = "updated";
|
|
2982
|
+
}
|
|
2983
|
+
const result = updateEntry.run(
|
|
2984
|
+
action,
|
|
2985
|
+
current === null
|
|
2986
|
+
? null
|
|
2987
|
+
: canonicalJson(
|
|
2988
|
+
schedulerJobDefinition(current),
|
|
2989
|
+
"schedulerMigration.targetBefore",
|
|
2990
|
+
),
|
|
2991
|
+
applied.revision,
|
|
2992
|
+
now,
|
|
2993
|
+
occurrenceCount,
|
|
2994
|
+
executionEventCount,
|
|
2995
|
+
now,
|
|
2996
|
+
id,
|
|
2997
|
+
entry.entryId,
|
|
2998
|
+
);
|
|
2999
|
+
if (result.changes !== 1) {
|
|
3000
|
+
throw schedulerMigrationError(
|
|
3001
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3002
|
+
"Scheduler migration entry changed while it was being applied",
|
|
3003
|
+
);
|
|
3004
|
+
}
|
|
3005
|
+
}
|
|
3006
|
+
this.db
|
|
3007
|
+
.prepare(
|
|
3008
|
+
`UPDATE scheduler_domain_migrations
|
|
3009
|
+
SET state = 'applied', updated_at = ?
|
|
3010
|
+
WHERE migration_id = ? AND state = 'prepared'`,
|
|
3011
|
+
)
|
|
3012
|
+
.run(now, id);
|
|
3013
|
+
return { ...this._domainMigration(id), deduplicated: false };
|
|
3014
|
+
});
|
|
3015
|
+
}
|
|
3016
|
+
|
|
3017
|
+
verifyDomainMigration(migrationId, { sources } = {}) {
|
|
3018
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
3019
|
+
if (!Array.isArray(sources)) {
|
|
3020
|
+
throw invalidArgument("sources must be an array");
|
|
3021
|
+
}
|
|
3022
|
+
const supplied = new Map();
|
|
3023
|
+
for (const [index, source] of sources.entries()) {
|
|
3024
|
+
if (!source || typeof source !== "object" || Array.isArray(source)) {
|
|
3025
|
+
throw invalidArgument(`sources[${index}] must be an object`);
|
|
3026
|
+
}
|
|
3027
|
+
const entryId = normalizeIdentifier(
|
|
3028
|
+
source.entryId,
|
|
3029
|
+
`sources[${index}].entryId`,
|
|
3030
|
+
);
|
|
3031
|
+
if (supplied.has(entryId)) {
|
|
3032
|
+
throw invalidArgument("sources contains duplicate migration entries");
|
|
3033
|
+
}
|
|
3034
|
+
supplied.set(entryId, schedulerMigrationSourceDigest(source.source));
|
|
3035
|
+
}
|
|
3036
|
+
const now = this._now();
|
|
3037
|
+
return this._write(() => {
|
|
3038
|
+
const migration = this._domainMigration(id);
|
|
3039
|
+
if (migration.state === SCHEDULER_MIGRATION_STATES.VERIFIED) {
|
|
3040
|
+
return { ...migration, deduplicated: true };
|
|
3041
|
+
}
|
|
3042
|
+
if (migration.state !== SCHEDULER_MIGRATION_STATES.APPLIED) {
|
|
3043
|
+
throw schedulerMigrationError(
|
|
3044
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3045
|
+
`Scheduler migration cannot be verified from state ${migration.state}`,
|
|
3046
|
+
);
|
|
3047
|
+
}
|
|
3048
|
+
if (supplied.size !== migration.entries.length) {
|
|
3049
|
+
throw schedulerMigrationError(
|
|
3050
|
+
"SCHEDULER_MIGRATION_SOURCE_MISMATCH",
|
|
3051
|
+
"Verification must provide every scheduler migration source exactly once",
|
|
3052
|
+
);
|
|
3053
|
+
}
|
|
3054
|
+
for (const entry of migration.entries) {
|
|
3055
|
+
if (supplied.get(entry.entryId) !== entry.sourceDigest) {
|
|
3056
|
+
throw schedulerMigrationError(
|
|
3057
|
+
"SCHEDULER_MIGRATION_SOURCE_CHANGED",
|
|
3058
|
+
`Scheduler migration source changed before verification: ${entry.sourceId}`,
|
|
3059
|
+
{ entryId: entry.entryId, sourceId: entry.sourceId },
|
|
3060
|
+
);
|
|
3061
|
+
}
|
|
3062
|
+
const job = mapJob(this.statements.getJob.get(entry.targetJobId));
|
|
3063
|
+
const manifestEntry = migration.manifest.entries.find(
|
|
3064
|
+
(candidate) => candidate.entryId === entry.entryId,
|
|
3065
|
+
);
|
|
3066
|
+
if (!manifestEntry) {
|
|
3067
|
+
throw schemaError(
|
|
3068
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
3069
|
+
"Scheduler migration entry is absent from its manifest",
|
|
3070
|
+
);
|
|
3071
|
+
}
|
|
3072
|
+
const stagedDigest = schedulerJobDefinitionDigest({
|
|
3073
|
+
...normalizeJobInput(manifestEntry.targetJob),
|
|
3074
|
+
enabled: false,
|
|
3075
|
+
});
|
|
3076
|
+
if (
|
|
3077
|
+
!job ||
|
|
3078
|
+
job.revision !== entry.targetAppliedRevision ||
|
|
3079
|
+
schedulerJobDefinitionDigest(job) !== stagedDigest
|
|
3080
|
+
) {
|
|
3081
|
+
throw schedulerMigrationError(
|
|
3082
|
+
"SCHEDULER_MIGRATION_TARGET_CHANGED",
|
|
3083
|
+
`Scheduler migration target changed before verification: ${entry.targetJobId}`,
|
|
3084
|
+
{ entryId: entry.entryId, targetJobId: entry.targetJobId },
|
|
3085
|
+
);
|
|
3086
|
+
}
|
|
3087
|
+
}
|
|
3088
|
+
const entries = this.db
|
|
3089
|
+
.prepare(
|
|
3090
|
+
`UPDATE scheduler_domain_migration_entries
|
|
3091
|
+
SET state = 'verified', updated_at = ?
|
|
3092
|
+
WHERE migration_id = ? AND state = 'applied'`,
|
|
3093
|
+
)
|
|
3094
|
+
.run(now, id);
|
|
3095
|
+
if (entries.changes !== migration.entries.length) {
|
|
3096
|
+
throw schedulerMigrationError(
|
|
3097
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3098
|
+
"Scheduler migration entries changed during verification",
|
|
3099
|
+
);
|
|
3100
|
+
}
|
|
3101
|
+
this.db
|
|
3102
|
+
.prepare(
|
|
3103
|
+
`UPDATE scheduler_domain_migrations
|
|
3104
|
+
SET state = 'verified', updated_at = ?
|
|
3105
|
+
WHERE migration_id = ? AND state = 'applied'`,
|
|
3106
|
+
)
|
|
3107
|
+
.run(now, id);
|
|
3108
|
+
return { ...this._domainMigration(id), deduplicated: false };
|
|
3109
|
+
});
|
|
3110
|
+
}
|
|
3111
|
+
|
|
3112
|
+
beginDomainMigrationRetirement(migrationId) {
|
|
3113
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
3114
|
+
const now = this._now();
|
|
3115
|
+
return this._write(() => {
|
|
3116
|
+
const migration = this._domainMigration(id);
|
|
3117
|
+
if (migration.state === SCHEDULER_MIGRATION_STATES.RETIRING) {
|
|
3118
|
+
return { ...migration, deduplicated: true };
|
|
3119
|
+
}
|
|
3120
|
+
if (migration.state !== SCHEDULER_MIGRATION_STATES.VERIFIED) {
|
|
3121
|
+
throw schedulerMigrationError(
|
|
3122
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3123
|
+
`Scheduler migration cannot retire sources from state ${migration.state}`,
|
|
3124
|
+
);
|
|
3125
|
+
}
|
|
3126
|
+
const update = this.db.prepare(
|
|
3127
|
+
`UPDATE scheduler_domain_migration_entries
|
|
3128
|
+
SET state = 'retiring', retirement_token = ?, updated_at = ?
|
|
3129
|
+
WHERE migration_id = ? AND entry_id = ? AND state = 'verified'`,
|
|
3130
|
+
);
|
|
3131
|
+
for (const entry of migration.entries) {
|
|
3132
|
+
const token = `scheduler-retirement-${randomUUID()}`;
|
|
3133
|
+
if (update.run(token, now, id, entry.entryId).changes !== 1) {
|
|
3134
|
+
throw schedulerMigrationError(
|
|
3135
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3136
|
+
"Scheduler migration entry changed during retirement preparation",
|
|
3137
|
+
);
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
this.db
|
|
3141
|
+
.prepare(
|
|
3142
|
+
`UPDATE scheduler_domain_migrations
|
|
3143
|
+
SET state = 'retiring', updated_at = ?
|
|
3144
|
+
WHERE migration_id = ? AND state = 'verified'`,
|
|
3145
|
+
)
|
|
3146
|
+
.run(now, id);
|
|
3147
|
+
return { ...this._domainMigration(id), deduplicated: false };
|
|
3148
|
+
});
|
|
3149
|
+
}
|
|
3150
|
+
|
|
3151
|
+
confirmDomainMigrationEntryRetired({
|
|
3152
|
+
migrationId,
|
|
3153
|
+
entryId,
|
|
3154
|
+
retirementToken,
|
|
3155
|
+
source,
|
|
3156
|
+
} = {}) {
|
|
3157
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
3158
|
+
const normalizedEntryId = normalizeIdentifier(entryId, "entryId");
|
|
3159
|
+
const token = normalizeIdentifier(retirementToken, "retirementToken");
|
|
3160
|
+
const retirementDigest = schedulerMigrationSourceDigest(source);
|
|
3161
|
+
const now = this._now();
|
|
3162
|
+
return this._write(() => {
|
|
3163
|
+
const entry = mapDomainMigrationEntry(
|
|
3164
|
+
this.statements.getDomainMigrationEntry.get(id, normalizedEntryId),
|
|
3165
|
+
);
|
|
3166
|
+
if (!entry) {
|
|
3167
|
+
throw schedulerMigrationError(
|
|
3168
|
+
"SCHEDULER_MIGRATION_NOT_FOUND",
|
|
3169
|
+
`Scheduler migration entry does not exist: ${normalizedEntryId}`,
|
|
3170
|
+
);
|
|
3171
|
+
}
|
|
3172
|
+
if (entry.state === "retired") {
|
|
3173
|
+
if (
|
|
3174
|
+
entry.retirementToken !== token ||
|
|
3175
|
+
entry.sourceRetirementDigest !== retirementDigest
|
|
3176
|
+
) {
|
|
3177
|
+
throw schedulerMigrationError(
|
|
3178
|
+
"SCHEDULER_MIGRATION_SOURCE_MISMATCH",
|
|
3179
|
+
"Retired scheduler source evidence does not match the journal",
|
|
3180
|
+
);
|
|
3181
|
+
}
|
|
3182
|
+
return { ...entry, deduplicated: true };
|
|
3183
|
+
}
|
|
3184
|
+
if (entry.state !== "retiring" || entry.retirementToken !== token) {
|
|
3185
|
+
throw schedulerMigrationError(
|
|
3186
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3187
|
+
"Scheduler migration entry is not waiting for this retirement token",
|
|
3188
|
+
);
|
|
3189
|
+
}
|
|
3190
|
+
const updated = this.db
|
|
3191
|
+
.prepare(
|
|
3192
|
+
`UPDATE scheduler_domain_migration_entries
|
|
3193
|
+
SET state = 'retired', source_retirement_digest = ?, updated_at = ?
|
|
3194
|
+
WHERE migration_id = ? AND entry_id = ? AND state = 'retiring'
|
|
3195
|
+
AND retirement_token = ?`,
|
|
3196
|
+
)
|
|
3197
|
+
.run(retirementDigest, now, id, normalizedEntryId, token);
|
|
3198
|
+
if (updated.changes !== 1) {
|
|
3199
|
+
throw schedulerMigrationError(
|
|
3200
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3201
|
+
"Scheduler migration entry changed while retirement was confirmed",
|
|
3202
|
+
);
|
|
3203
|
+
}
|
|
3204
|
+
const remaining = this.db
|
|
3205
|
+
.prepare(
|
|
3206
|
+
`SELECT COUNT(*) AS count FROM scheduler_domain_migration_entries
|
|
3207
|
+
WHERE migration_id = ? AND state <> 'retired'`,
|
|
3208
|
+
)
|
|
3209
|
+
.get(id).count;
|
|
3210
|
+
if (remaining === 0) {
|
|
3211
|
+
const refreshed = this._domainMigration(id);
|
|
3212
|
+
const manifestById = new Map(
|
|
3213
|
+
refreshed.manifest.entries.map((candidate) => [
|
|
3214
|
+
candidate.entryId,
|
|
3215
|
+
candidate,
|
|
3216
|
+
]),
|
|
3217
|
+
);
|
|
3218
|
+
const activations = [];
|
|
3219
|
+
for (const candidate of refreshed.entries) {
|
|
3220
|
+
const manifestEntry = manifestById.get(candidate.entryId);
|
|
3221
|
+
if (!manifestEntry) {
|
|
3222
|
+
throw schemaError(
|
|
3223
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
3224
|
+
"Scheduler migration entry is absent from its manifest",
|
|
3225
|
+
);
|
|
3226
|
+
}
|
|
3227
|
+
const intended = normalizeJobInput(manifestEntry.targetJob);
|
|
3228
|
+
const current = mapJob(
|
|
3229
|
+
this.statements.getJob.get(candidate.targetJobId),
|
|
3230
|
+
);
|
|
3231
|
+
const stagedDigest = schedulerJobDefinitionDigest({
|
|
3232
|
+
...intended,
|
|
3233
|
+
enabled: false,
|
|
3234
|
+
});
|
|
3235
|
+
if (
|
|
3236
|
+
current &&
|
|
3237
|
+
schedulerJobDefinitionDigest(current) === candidate.targetJobDigest
|
|
3238
|
+
) {
|
|
3239
|
+
this.db
|
|
3240
|
+
.prepare(
|
|
3241
|
+
`UPDATE scheduler_domain_migration_entries
|
|
3242
|
+
SET target_applied_revision = ?, updated_at = ?
|
|
3243
|
+
WHERE migration_id = ? AND entry_id = ? AND state = 'retired'`,
|
|
3244
|
+
)
|
|
3245
|
+
.run(current.revision, now, id, candidate.entryId);
|
|
3246
|
+
continue;
|
|
3247
|
+
}
|
|
3248
|
+
if (
|
|
3249
|
+
!current ||
|
|
3250
|
+
current.revision !== candidate.targetAppliedRevision ||
|
|
3251
|
+
schedulerJobDefinitionDigest(current) !== stagedDigest
|
|
3252
|
+
) {
|
|
3253
|
+
throw schedulerMigrationError(
|
|
3254
|
+
"SCHEDULER_MIGRATION_TARGET_CHANGED",
|
|
3255
|
+
`Scheduler migration target changed before activation: ${candidate.targetJobId}`,
|
|
3256
|
+
{
|
|
3257
|
+
entryId: candidate.entryId,
|
|
3258
|
+
targetJobId: candidate.targetJobId,
|
|
3259
|
+
},
|
|
3260
|
+
);
|
|
3261
|
+
}
|
|
3262
|
+
activations.push({ candidate, current, intended });
|
|
3263
|
+
}
|
|
3264
|
+
for (const { candidate, current, intended } of activations) {
|
|
3265
|
+
const activated =
|
|
3266
|
+
schedulerJobDefinitionDigest(current) === candidate.targetJobDigest
|
|
3267
|
+
? current
|
|
3268
|
+
: this.updateJob(current.id, current.revision, {
|
|
3269
|
+
kind: intended.kind,
|
|
3270
|
+
trigger: intended.trigger,
|
|
3271
|
+
payload: intended.payload,
|
|
3272
|
+
authority: intended.authority,
|
|
3273
|
+
enabled: intended.enabled,
|
|
3274
|
+
maxAttempts: intended.maxAttempts,
|
|
3275
|
+
});
|
|
3276
|
+
this.db
|
|
3277
|
+
.prepare(
|
|
3278
|
+
`UPDATE scheduler_domain_migration_entries
|
|
3279
|
+
SET target_applied_revision = ?, updated_at = ?
|
|
3280
|
+
WHERE migration_id = ? AND entry_id = ? AND state = 'retired'`,
|
|
3281
|
+
)
|
|
3282
|
+
.run(activated.revision, now, id, candidate.entryId);
|
|
3283
|
+
}
|
|
3284
|
+
this.db
|
|
3285
|
+
.prepare(
|
|
3286
|
+
`UPDATE scheduler_domain_migrations
|
|
3287
|
+
SET state = 'retired', updated_at = ?, completed_at = ?
|
|
3288
|
+
WHERE migration_id = ? AND state = 'retiring'`,
|
|
3289
|
+
)
|
|
3290
|
+
.run(now, now, id);
|
|
3291
|
+
}
|
|
3292
|
+
return {
|
|
3293
|
+
...mapDomainMigrationEntry(
|
|
3294
|
+
this.statements.getDomainMigrationEntry.get(id, normalizedEntryId),
|
|
3295
|
+
),
|
|
3296
|
+
deduplicated: false,
|
|
3297
|
+
};
|
|
3298
|
+
});
|
|
3299
|
+
}
|
|
3300
|
+
|
|
3301
|
+
beginDomainMigrationRollback(migrationId) {
|
|
3302
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
3303
|
+
const now = this._now();
|
|
3304
|
+
return this._write(() => {
|
|
3305
|
+
const migration = this._domainMigration(id);
|
|
3306
|
+
if (migration.state === SCHEDULER_MIGRATION_STATES.ROLLING_BACK) {
|
|
3307
|
+
return this._rollbackDomainMigrationTargetsInTransaction(id, now);
|
|
3308
|
+
}
|
|
3309
|
+
if (migration.state === SCHEDULER_MIGRATION_STATES.ROLLED_BACK) {
|
|
3310
|
+
return { ...migration, deduplicated: true };
|
|
3311
|
+
}
|
|
3312
|
+
if (
|
|
3313
|
+
![
|
|
3314
|
+
SCHEDULER_MIGRATION_STATES.PREPARED,
|
|
3315
|
+
SCHEDULER_MIGRATION_STATES.APPLIED,
|
|
3316
|
+
SCHEDULER_MIGRATION_STATES.VERIFIED,
|
|
3317
|
+
SCHEDULER_MIGRATION_STATES.RETIRING,
|
|
3318
|
+
SCHEDULER_MIGRATION_STATES.RETIRED,
|
|
3319
|
+
].includes(migration.state)
|
|
3320
|
+
) {
|
|
3321
|
+
throw schedulerMigrationError(
|
|
3322
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3323
|
+
`Scheduler migration cannot roll back from state ${migration.state}`,
|
|
3324
|
+
);
|
|
3325
|
+
}
|
|
3326
|
+
this.db
|
|
3327
|
+
.prepare(
|
|
3328
|
+
`UPDATE scheduler_domain_migrations
|
|
3329
|
+
SET state = 'rolling_back', updated_at = ?, completed_at = NULL
|
|
3330
|
+
WHERE migration_id = ?
|
|
3331
|
+
AND state IN ('prepared', 'applied', 'verified', 'retiring', 'retired')`,
|
|
3332
|
+
)
|
|
3333
|
+
.run(now, id);
|
|
3334
|
+
return this._rollbackDomainMigrationTargetsInTransaction(
|
|
3335
|
+
id,
|
|
3336
|
+
now,
|
|
3337
|
+
migration.state,
|
|
3338
|
+
);
|
|
3339
|
+
});
|
|
3340
|
+
}
|
|
3341
|
+
|
|
3342
|
+
restoreDomainMigrationEntrySource({
|
|
3343
|
+
migrationId,
|
|
3344
|
+
entryId,
|
|
3345
|
+
retirementToken,
|
|
3346
|
+
restoreSource,
|
|
3347
|
+
} = {}) {
|
|
3348
|
+
if (typeof restoreSource !== "function") {
|
|
3349
|
+
throw invalidArgument("restoreSource must be a function");
|
|
3350
|
+
}
|
|
3351
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
3352
|
+
const normalizedEntryId = normalizeIdentifier(entryId, "entryId");
|
|
3353
|
+
const token =
|
|
3354
|
+
retirementToken === null || retirementToken === undefined
|
|
3355
|
+
? null
|
|
3356
|
+
: normalizeIdentifier(retirementToken, "retirementToken");
|
|
3357
|
+
const now = this._now();
|
|
3358
|
+
return this._write(() =>
|
|
3359
|
+
this._restoreDomainMigrationEntrySourceInTransaction({
|
|
3360
|
+
migrationId: id,
|
|
3361
|
+
entryId: normalizedEntryId,
|
|
3362
|
+
retirementToken: token,
|
|
3363
|
+
restoreSource,
|
|
3364
|
+
now,
|
|
3365
|
+
}),
|
|
3366
|
+
);
|
|
3367
|
+
}
|
|
3368
|
+
|
|
3369
|
+
confirmDomainMigrationEntrySourceRestored({
|
|
3370
|
+
migrationId,
|
|
3371
|
+
entryId,
|
|
3372
|
+
retirementToken,
|
|
3373
|
+
source,
|
|
3374
|
+
} = {}) {
|
|
3375
|
+
return this.restoreDomainMigrationEntrySource({
|
|
3376
|
+
migrationId,
|
|
3377
|
+
entryId,
|
|
3378
|
+
retirementToken,
|
|
3379
|
+
restoreSource: () => source,
|
|
3380
|
+
});
|
|
3381
|
+
}
|
|
3382
|
+
|
|
3383
|
+
_restoreDomainMigrationEntrySourceInTransaction({
|
|
3384
|
+
migrationId,
|
|
3385
|
+
entryId,
|
|
3386
|
+
retirementToken,
|
|
3387
|
+
restoreSource,
|
|
3388
|
+
now,
|
|
3389
|
+
}) {
|
|
3390
|
+
const migration = this._domainMigration(migrationId);
|
|
3391
|
+
if (
|
|
3392
|
+
![
|
|
3393
|
+
SCHEDULER_MIGRATION_STATES.ROLLING_BACK,
|
|
3394
|
+
SCHEDULER_MIGRATION_STATES.ROLLED_BACK,
|
|
3395
|
+
].includes(migration.state)
|
|
3396
|
+
) {
|
|
3397
|
+
throw schedulerMigrationError(
|
|
3398
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3399
|
+
"Scheduler migration is not rolling back",
|
|
3400
|
+
);
|
|
3401
|
+
}
|
|
3402
|
+
const entry = mapDomainMigrationEntry(
|
|
3403
|
+
this.statements.getDomainMigrationEntry.get(migrationId, entryId),
|
|
3404
|
+
);
|
|
3405
|
+
if (!entry || entry.retirementToken !== retirementToken) {
|
|
3406
|
+
throw schedulerMigrationError(
|
|
3407
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3408
|
+
"Scheduler migration source restoration token does not match",
|
|
3409
|
+
);
|
|
3410
|
+
}
|
|
3411
|
+
if (entry.state === "rolled_back") {
|
|
3412
|
+
if (entry.sourceRestoredDigest !== entry.sourceDigest) {
|
|
3413
|
+
throw schedulerMigrationError(
|
|
3414
|
+
"SCHEDULER_MIGRATION_SOURCE_MISMATCH",
|
|
3415
|
+
"Restored scheduler source evidence does not match the journal",
|
|
3416
|
+
);
|
|
3417
|
+
}
|
|
3418
|
+
return { ...entry, deduplicated: true };
|
|
3419
|
+
}
|
|
3420
|
+
if (entry.state === "rollback_target_disabled") {
|
|
3421
|
+
this._assertDomainMigrationRollbackTarget(migration, entry);
|
|
3422
|
+
} else if (entry.state !== "prepared") {
|
|
3423
|
+
throw schedulerMigrationError(
|
|
3424
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3425
|
+
`Scheduler migration entry cannot restore its source from state ${entry.state}`,
|
|
3426
|
+
);
|
|
3427
|
+
}
|
|
3428
|
+
|
|
3429
|
+
const restoredSource = restoreSource();
|
|
3430
|
+
if (restoredSource && typeof restoredSource.then === "function") {
|
|
3431
|
+
throw invalidArgument("restoreSource must complete synchronously");
|
|
3432
|
+
}
|
|
3433
|
+
const restoredDigest = schedulerMigrationSourceDigest(restoredSource);
|
|
3434
|
+
if (restoredDigest !== entry.sourceDigest) {
|
|
3435
|
+
throw schedulerMigrationError(
|
|
3436
|
+
"SCHEDULER_MIGRATION_SOURCE_CHANGED",
|
|
3437
|
+
"Restored scheduler source does not match its pre-migration digest",
|
|
3438
|
+
{ entryId, sourceId: entry.sourceId },
|
|
3439
|
+
);
|
|
3440
|
+
}
|
|
3441
|
+
const update = this.db
|
|
3442
|
+
.prepare(
|
|
3443
|
+
`UPDATE scheduler_domain_migration_entries
|
|
3444
|
+
SET state = 'rolled_back', source_restored_digest = ?, updated_at = ?
|
|
3445
|
+
WHERE migration_id = ? AND entry_id = ?
|
|
3446
|
+
AND state IN ('prepared', 'rollback_target_disabled')`,
|
|
3447
|
+
)
|
|
3448
|
+
.run(restoredDigest, now, migrationId, entryId);
|
|
3449
|
+
if (update.changes !== 1) {
|
|
3450
|
+
throw schedulerMigrationError(
|
|
3451
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3452
|
+
"Scheduler migration source restoration lost its journal fence",
|
|
3453
|
+
);
|
|
3454
|
+
}
|
|
3455
|
+
const remaining = this.db
|
|
3456
|
+
.prepare(
|
|
3457
|
+
`SELECT COUNT(*) AS count FROM scheduler_domain_migration_entries
|
|
3458
|
+
WHERE migration_id = ? AND state <> 'rolled_back'`,
|
|
3459
|
+
)
|
|
3460
|
+
.get(migrationId).count;
|
|
3461
|
+
if (remaining === 0) {
|
|
3462
|
+
this.db
|
|
3463
|
+
.prepare(
|
|
3464
|
+
`UPDATE scheduler_domain_migrations
|
|
3465
|
+
SET state = 'rolled_back', updated_at = ?, completed_at = ?
|
|
3466
|
+
WHERE migration_id = ? AND state = 'rolling_back'`,
|
|
3467
|
+
)
|
|
3468
|
+
.run(now, now, migrationId);
|
|
3469
|
+
}
|
|
3470
|
+
return {
|
|
3471
|
+
...mapDomainMigrationEntry(
|
|
3472
|
+
this.statements.getDomainMigrationEntry.get(migrationId, entryId),
|
|
3473
|
+
),
|
|
3474
|
+
deduplicated: false,
|
|
3475
|
+
};
|
|
3476
|
+
}
|
|
3477
|
+
|
|
3478
|
+
rollbackDomainMigrationTargets(migrationId) {
|
|
3479
|
+
const id = normalizeIdentifier(migrationId, "migrationId");
|
|
3480
|
+
const now = this._now();
|
|
3481
|
+
return this._write(() => {
|
|
3482
|
+
const migration = this._domainMigration(id);
|
|
3483
|
+
if (migration.state === SCHEDULER_MIGRATION_STATES.ROLLED_BACK) {
|
|
3484
|
+
return { ...migration, deduplicated: true };
|
|
3485
|
+
}
|
|
3486
|
+
if (migration.state !== SCHEDULER_MIGRATION_STATES.ROLLING_BACK) {
|
|
3487
|
+
throw schedulerMigrationError(
|
|
3488
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3489
|
+
"Scheduler migration is not rolling back",
|
|
3490
|
+
);
|
|
3491
|
+
}
|
|
3492
|
+
return this._rollbackDomainMigrationTargetsInTransaction(id, now);
|
|
3493
|
+
});
|
|
3494
|
+
}
|
|
3495
|
+
|
|
3496
|
+
_rollbackDomainMigrationTargetsInTransaction(
|
|
3497
|
+
migrationId,
|
|
3498
|
+
now,
|
|
3499
|
+
rollbackFromState = null,
|
|
3500
|
+
) {
|
|
3501
|
+
const migration = this._domainMigration(migrationId);
|
|
3502
|
+
const invalid = migration.entries.filter(
|
|
3503
|
+
(entry) =>
|
|
3504
|
+
![
|
|
3505
|
+
"prepared",
|
|
3506
|
+
"applied",
|
|
3507
|
+
"verified",
|
|
3508
|
+
"retired",
|
|
3509
|
+
"retiring",
|
|
3510
|
+
"source_restored",
|
|
3511
|
+
"rollback_target_disabled",
|
|
3512
|
+
"rolled_back",
|
|
3513
|
+
].includes(entry.state),
|
|
3514
|
+
);
|
|
3515
|
+
if (invalid.length > 0) {
|
|
3516
|
+
throw schedulerMigrationError(
|
|
3517
|
+
"SCHEDULER_MIGRATION_STATE_CONFLICT",
|
|
3518
|
+
"Scheduler migration entries are not ready for target rollback",
|
|
3519
|
+
{ entryIds: invalid.map((entry) => entry.entryId) },
|
|
3520
|
+
);
|
|
3521
|
+
}
|
|
3522
|
+
const targetRollbackEntries = migration.entries.filter(
|
|
3523
|
+
(entry) => entry.state === "rollback_target_disabled",
|
|
3524
|
+
);
|
|
3525
|
+
for (const entry of targetRollbackEntries) {
|
|
3526
|
+
this._assertDomainMigrationRollbackTarget(migration, entry);
|
|
3527
|
+
}
|
|
3528
|
+
const pending = [...migration.entries]
|
|
3529
|
+
.reverse()
|
|
3530
|
+
.filter(
|
|
3531
|
+
(entry) =>
|
|
3532
|
+
!["prepared", "rollback_target_disabled", "rolled_back"].includes(
|
|
3533
|
+
entry.state,
|
|
3534
|
+
),
|
|
3535
|
+
);
|
|
3536
|
+
const snapshots = new Map();
|
|
3537
|
+
for (const entry of pending) {
|
|
3538
|
+
const current = mapJob(this.statements.getJob.get(entry.targetJobId));
|
|
3539
|
+
if (!current || current.revision !== entry.targetAppliedRevision) {
|
|
3540
|
+
throw schedulerMigrationError(
|
|
3541
|
+
"SCHEDULER_MIGRATION_TARGET_CHANGED",
|
|
3542
|
+
`Scheduler migration target revision changed before rollback: ${entry.targetJobId}`,
|
|
3543
|
+
{
|
|
3544
|
+
expectedRevision: entry.targetAppliedRevision,
|
|
3545
|
+
actualRevision: current?.revision ?? null,
|
|
3546
|
+
},
|
|
3547
|
+
);
|
|
3548
|
+
}
|
|
3549
|
+
this._assertDomainMigrationTargetBeforeRollback(
|
|
3550
|
+
migration,
|
|
3551
|
+
entry,
|
|
3552
|
+
current,
|
|
3553
|
+
rollbackFromState,
|
|
3554
|
+
);
|
|
3555
|
+
this._assertDomainMigrationTargetExecutionEvidence(entry);
|
|
3556
|
+
snapshots.set(entry.entryId, current);
|
|
3557
|
+
}
|
|
3558
|
+
for (const entry of pending) {
|
|
3559
|
+
if (entry.state === "rollback_target_disabled") continue;
|
|
3560
|
+
const current = snapshots.get(entry.entryId);
|
|
3561
|
+
let rolledBack;
|
|
3562
|
+
if (
|
|
3563
|
+
entry.targetAction === "created" ||
|
|
3564
|
+
entry.rollbackStrategy === "disable"
|
|
3565
|
+
) {
|
|
3566
|
+
rolledBack = current.enabled
|
|
3567
|
+
? this.updateJob(current.id, current.revision, { enabled: false })
|
|
3568
|
+
: current;
|
|
3569
|
+
} else {
|
|
3570
|
+
const before = normalizeJobInput(entry.targetBefore);
|
|
3571
|
+
rolledBack =
|
|
3572
|
+
schedulerJobDefinitionDigest(current) ===
|
|
3573
|
+
schedulerJobDefinitionDigest(before)
|
|
3574
|
+
? current
|
|
3575
|
+
: this.updateJob(current.id, current.revision, {
|
|
3576
|
+
kind: before.kind,
|
|
3577
|
+
trigger: before.trigger,
|
|
3578
|
+
payload: before.payload,
|
|
3579
|
+
authority: before.authority,
|
|
3580
|
+
enabled: before.enabled,
|
|
3581
|
+
maxAttempts: before.maxAttempts,
|
|
3582
|
+
});
|
|
3583
|
+
}
|
|
3584
|
+
this.db
|
|
3585
|
+
.prepare(
|
|
3586
|
+
`UPDATE scheduler_domain_migration_entries
|
|
3587
|
+
SET state = 'rollback_target_disabled',
|
|
3588
|
+
target_rollback_revision = ?, updated_at = ?
|
|
3589
|
+
WHERE migration_id = ? AND entry_id = ?
|
|
3590
|
+
AND state IN (
|
|
3591
|
+
'applied', 'verified', 'retiring', 'retired', 'source_restored'
|
|
3592
|
+
)`,
|
|
3593
|
+
)
|
|
3594
|
+
.run(rolledBack.revision, now, migrationId, entry.entryId);
|
|
3595
|
+
}
|
|
3596
|
+
this.db
|
|
3597
|
+
.prepare(
|
|
3598
|
+
`UPDATE scheduler_domain_migration_entries
|
|
3599
|
+
SET source_restored_digest = NULL, updated_at = ?
|
|
3600
|
+
WHERE migration_id = ? AND state = 'rollback_target_disabled'
|
|
3601
|
+
AND source_restored_digest IS NOT NULL`,
|
|
3602
|
+
)
|
|
3603
|
+
.run(now, migrationId);
|
|
3604
|
+
return {
|
|
3605
|
+
...this._domainMigration(migrationId),
|
|
3606
|
+
deduplicated: pending.length === 0,
|
|
3607
|
+
};
|
|
3608
|
+
}
|
|
3609
|
+
|
|
3610
|
+
_assertDomainMigrationTargetBeforeRollback(
|
|
3611
|
+
migration,
|
|
3612
|
+
entry,
|
|
3613
|
+
current,
|
|
3614
|
+
rollbackFromState,
|
|
3615
|
+
) {
|
|
3616
|
+
const manifestEntry = migration.manifest.entries.find(
|
|
3617
|
+
(candidate) => candidate.entryId === entry.entryId,
|
|
3618
|
+
);
|
|
3619
|
+
if (!manifestEntry) {
|
|
3620
|
+
throw schemaError(
|
|
3621
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
3622
|
+
"Scheduler migration entry is absent from its manifest",
|
|
3623
|
+
);
|
|
3624
|
+
}
|
|
3625
|
+
const intended = normalizeJobInput(manifestEntry.targetJob);
|
|
3626
|
+
const resumedRetiredMigration =
|
|
3627
|
+
rollbackFromState === null &&
|
|
3628
|
+
entry.state === "retired" &&
|
|
3629
|
+
migration.entries.every((candidate) =>
|
|
3630
|
+
["retired", "rollback_target_disabled", "rolled_back"].includes(
|
|
3631
|
+
candidate.state,
|
|
3632
|
+
),
|
|
3633
|
+
);
|
|
3634
|
+
const targetWasActivated =
|
|
3635
|
+
rollbackFromState === SCHEDULER_MIGRATION_STATES.RETIRED ||
|
|
3636
|
+
resumedRetiredMigration;
|
|
3637
|
+
const expected = targetWasActivated
|
|
3638
|
+
? intended
|
|
3639
|
+
: { ...intended, enabled: false };
|
|
3640
|
+
const expectedDigest = schedulerJobDefinitionDigest(expected);
|
|
3641
|
+
const actualDigest = schedulerJobDefinitionDigest(current);
|
|
3642
|
+
if (actualDigest !== expectedDigest) {
|
|
3643
|
+
throw schedulerMigrationError(
|
|
3644
|
+
"SCHEDULER_MIGRATION_TARGET_CHANGED",
|
|
3645
|
+
`Scheduler migration target definition changed before rollback: ${entry.targetJobId}`,
|
|
3646
|
+
{
|
|
3647
|
+
expectedRevision: entry.targetAppliedRevision,
|
|
3648
|
+
actualRevision: current.revision,
|
|
3649
|
+
expectedDefinitionDigest: expectedDigest,
|
|
3650
|
+
actualDefinitionDigest: actualDigest,
|
|
3651
|
+
},
|
|
3652
|
+
);
|
|
3653
|
+
}
|
|
3654
|
+
}
|
|
3655
|
+
|
|
3656
|
+
_assertDomainMigrationRollbackTarget(migration, entry) {
|
|
3657
|
+
const current = mapJob(this.statements.getJob.get(entry.targetJobId));
|
|
3658
|
+
const manifestEntry = migration.manifest.entries.find(
|
|
3659
|
+
(candidate) => candidate.entryId === entry.entryId,
|
|
3660
|
+
);
|
|
3661
|
+
if (
|
|
3662
|
+
!current ||
|
|
3663
|
+
current.revision !== entry.targetRollbackRevision ||
|
|
3664
|
+
!manifestEntry ||
|
|
3665
|
+
!this._isExpectedDomainMigrationRollbackTarget(
|
|
3666
|
+
entry,
|
|
3667
|
+
current,
|
|
3668
|
+
manifestEntry.targetJob,
|
|
3669
|
+
)
|
|
3670
|
+
) {
|
|
3671
|
+
throw schedulerMigrationError(
|
|
3672
|
+
"SCHEDULER_MIGRATION_TARGET_CHANGED",
|
|
3673
|
+
`Scheduler migration target changed after rollback: ${entry.targetJobId}`,
|
|
3674
|
+
{
|
|
3675
|
+
expectedRevision: entry.targetRollbackRevision,
|
|
3676
|
+
actualRevision: current?.revision ?? null,
|
|
3677
|
+
},
|
|
3678
|
+
);
|
|
3679
|
+
}
|
|
3680
|
+
this._assertDomainMigrationTargetExecutionEvidence(entry);
|
|
3681
|
+
}
|
|
3682
|
+
|
|
3683
|
+
_isExpectedDomainMigrationRollbackTarget(entry, current, targetJob) {
|
|
3684
|
+
if (
|
|
3685
|
+
entry.targetAction === "created" ||
|
|
3686
|
+
entry.rollbackStrategy === "disable"
|
|
3687
|
+
) {
|
|
3688
|
+
const intended = normalizeJobInput(targetJob);
|
|
3689
|
+
return (
|
|
3690
|
+
schedulerJobDefinitionDigest(current) ===
|
|
3691
|
+
schedulerJobDefinitionDigest({ ...intended, enabled: false })
|
|
3692
|
+
);
|
|
3693
|
+
}
|
|
3694
|
+
return (
|
|
3695
|
+
entry.targetBefore !== null &&
|
|
3696
|
+
schedulerJobDefinitionDigest(current) ===
|
|
3697
|
+
schedulerJobDefinitionDigest(entry.targetBefore)
|
|
3698
|
+
);
|
|
3699
|
+
}
|
|
3700
|
+
|
|
3701
|
+
_assertDomainMigrationTargetExecutionEvidence(entry) {
|
|
3702
|
+
const counts = this.db
|
|
3703
|
+
.prepare(
|
|
3704
|
+
`SELECT
|
|
3705
|
+
(SELECT COUNT(*) FROM occurrences WHERE job_id = ?) AS occurrences,
|
|
3706
|
+
(SELECT COUNT(*) FROM events
|
|
3707
|
+
WHERE job_id = ? AND occurrence_id IS NOT NULL) AS execution_events`,
|
|
3708
|
+
)
|
|
3709
|
+
.get(entry.targetJobId, entry.targetJobId);
|
|
3710
|
+
if (
|
|
3711
|
+
counts.occurrences !== entry.targetOccurrenceCountBefore ||
|
|
3712
|
+
counts.execution_events !== entry.targetExecutionEventCountBefore
|
|
3713
|
+
) {
|
|
3714
|
+
throw schedulerMigrationError(
|
|
3715
|
+
"SCHEDULER_MIGRATION_EXECUTION_EVIDENCE",
|
|
3716
|
+
`Scheduler migration target has execution evidence and cannot be rolled back: ${entry.targetJobId}`,
|
|
3717
|
+
{
|
|
3718
|
+
occurrenceCountBefore: entry.targetOccurrenceCountBefore,
|
|
3719
|
+
occurrenceCountNow: counts.occurrences,
|
|
3720
|
+
executionEventCountBefore: entry.targetExecutionEventCountBefore,
|
|
3721
|
+
executionEventCountNow: counts.execution_events,
|
|
3722
|
+
},
|
|
3723
|
+
);
|
|
3724
|
+
}
|
|
3725
|
+
}
|
|
3726
|
+
|
|
3727
|
+
_enqueueOccurrence({ id, scheduled, key, available, payload, now }) {
|
|
3728
|
+
const job = mapJob(this.statements.getJob.get(id));
|
|
3729
|
+
if (!job) {
|
|
3730
|
+
throw new SchedulerKernelError(
|
|
3731
|
+
"SCHEDULER_NOT_FOUND",
|
|
3732
|
+
`Scheduler job does not exist: ${id}`,
|
|
3733
|
+
);
|
|
3734
|
+
}
|
|
3735
|
+
if (!job.enabled) {
|
|
3736
|
+
throw new SchedulerKernelError(
|
|
3737
|
+
"SCHEDULER_JOB_DISABLED",
|
|
3738
|
+
`Scheduler job is disabled: ${id}`,
|
|
3739
|
+
);
|
|
3740
|
+
}
|
|
3741
|
+
const identity = deriveOccurrenceIdentity({
|
|
3742
|
+
jobId: id,
|
|
3743
|
+
jobRevision: job.revision,
|
|
3744
|
+
scheduledFor: scheduled,
|
|
3745
|
+
triggerKey: key,
|
|
3746
|
+
});
|
|
3747
|
+
const payloadJson = canonicalJson(
|
|
3748
|
+
payload === undefined ? job.payload : payload,
|
|
3749
|
+
"occurrence.payload",
|
|
3750
|
+
);
|
|
3751
|
+
const result = this.db
|
|
3752
|
+
.prepare(
|
|
3753
|
+
`
|
|
3754
|
+
INSERT INTO occurrences
|
|
3755
|
+
(occurrence_id, job_id, job_revision, idempotency_key, trigger_key,
|
|
3756
|
+
scheduled_for, available_at, status, attempt, max_attempts, fence,
|
|
3757
|
+
lease_owner, lease_expires_at, authority_json, payload_json,
|
|
3758
|
+
last_error_json, result_json, created_at, updated_at, settled_at)
|
|
3759
|
+
VALUES
|
|
3760
|
+
(@occurrenceId, @jobId, @jobRevision, @idempotencyKey, @triggerKey,
|
|
3761
|
+
@scheduledFor, @availableAt, 'queued', 0, @maxAttempts, 0,
|
|
3762
|
+
NULL, NULL, @authorityJson, @payloadJson,
|
|
3763
|
+
NULL, NULL, @now, @now, NULL)
|
|
3764
|
+
ON CONFLICT(idempotency_key) DO NOTHING
|
|
3765
|
+
`,
|
|
3766
|
+
)
|
|
3767
|
+
.run({
|
|
3768
|
+
...identity,
|
|
3769
|
+
jobId: id,
|
|
3770
|
+
jobRevision: job.revision,
|
|
3771
|
+
triggerKey: key,
|
|
3772
|
+
scheduledFor: scheduled,
|
|
3773
|
+
availableAt: available,
|
|
3774
|
+
maxAttempts: job.maxAttempts,
|
|
3775
|
+
authorityJson: canonicalJson(job.authority, "occurrence.authority"),
|
|
3776
|
+
payloadJson,
|
|
3777
|
+
now,
|
|
3778
|
+
});
|
|
3779
|
+
const row = this.statements.getOccurrenceByKey.get(identity.idempotencyKey);
|
|
3780
|
+
if (!row || row.occurrence_id !== identity.occurrenceId) {
|
|
3781
|
+
throw schemaError(
|
|
3782
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
3783
|
+
"Occurrence idempotency identity does not match stored data",
|
|
3784
|
+
);
|
|
3785
|
+
}
|
|
3786
|
+
if (row.payload_json !== payloadJson) {
|
|
3787
|
+
throw new SchedulerKernelError(
|
|
3788
|
+
"SCHEDULER_IDEMPOTENCY_PAYLOAD_MISMATCH",
|
|
3789
|
+
"Occurrence idempotency identity was reused with a different payload",
|
|
3790
|
+
{ jobId: id, triggerKey: key },
|
|
3791
|
+
);
|
|
3792
|
+
}
|
|
3793
|
+
if (result.changes === 1) {
|
|
3794
|
+
this._appendEvent({
|
|
3795
|
+
jobId: id,
|
|
3796
|
+
occurrenceId: identity.occurrenceId,
|
|
3797
|
+
type: "occurrence_enqueued",
|
|
3798
|
+
occurredAt: now,
|
|
3799
|
+
data: {
|
|
3800
|
+
jobRevision: job.revision,
|
|
3801
|
+
scheduledFor: scheduled,
|
|
3802
|
+
triggerKey: key,
|
|
3803
|
+
},
|
|
3804
|
+
});
|
|
3805
|
+
}
|
|
3806
|
+
return { ...mapOccurrence(row), deduplicated: result.changes === 0 };
|
|
3807
|
+
}
|
|
3808
|
+
|
|
3809
|
+
enqueueOccurrence({
|
|
3810
|
+
jobId,
|
|
3811
|
+
scheduledFor,
|
|
3812
|
+
triggerKey,
|
|
3813
|
+
availableAt,
|
|
3814
|
+
payload,
|
|
3815
|
+
} = {}) {
|
|
3816
|
+
const id = normalizeIdentifier(jobId, "jobId");
|
|
3817
|
+
const scheduled = normalizeEpochMs(scheduledFor, "scheduledFor");
|
|
3818
|
+
const key = normalizeIdentifier(triggerKey, "triggerKey");
|
|
3819
|
+
const available = normalizeEpochMs(availableAt ?? scheduled, "availableAt");
|
|
3820
|
+
const now = this._now();
|
|
3821
|
+
return this._write(() =>
|
|
3822
|
+
this._enqueueOccurrence({ id, scheduled, key, available, payload, now }),
|
|
3823
|
+
);
|
|
3824
|
+
}
|
|
3825
|
+
|
|
3826
|
+
enqueueOccurrenceOncePerTrigger({
|
|
3827
|
+
jobId,
|
|
3828
|
+
scheduledFor,
|
|
3829
|
+
triggerKey,
|
|
3830
|
+
availableAt,
|
|
3831
|
+
payload,
|
|
3832
|
+
} = {}) {
|
|
3833
|
+
const id = normalizeIdentifier(jobId, "jobId");
|
|
3834
|
+
const scheduled = normalizeEpochMs(scheduledFor, "scheduledFor");
|
|
3835
|
+
const key = normalizeIdentifier(triggerKey, "triggerKey");
|
|
3836
|
+
const available = normalizeEpochMs(availableAt ?? scheduled, "availableAt");
|
|
3837
|
+
const now = this._now();
|
|
3838
|
+
return this._write(() => {
|
|
3839
|
+
const prior = this.statements.listOccurrencesByTrigger.all(id, key, 2);
|
|
3840
|
+
if (prior.length > 1) {
|
|
3841
|
+
throw schemaError(
|
|
3842
|
+
"SCHEDULER_DATA_CORRUPT",
|
|
3843
|
+
"Multiple occurrences exist for a unique job trigger",
|
|
3844
|
+
);
|
|
3845
|
+
}
|
|
3846
|
+
if (prior.length === 1) {
|
|
3847
|
+
return { ...mapOccurrence(prior[0]), deduplicated: true };
|
|
3848
|
+
}
|
|
3849
|
+
return this._enqueueOccurrence({
|
|
3850
|
+
id,
|
|
3851
|
+
scheduled,
|
|
3852
|
+
key,
|
|
3853
|
+
available,
|
|
3854
|
+
payload,
|
|
3855
|
+
now,
|
|
3856
|
+
});
|
|
3857
|
+
});
|
|
3858
|
+
}
|
|
3859
|
+
|
|
3860
|
+
getOccurrence(occurrenceId) {
|
|
3861
|
+
this._assertOpen();
|
|
3862
|
+
return mapOccurrence(
|
|
3863
|
+
this.statements.getOccurrence.get(
|
|
3864
|
+
normalizeIdentifier(occurrenceId, "occurrenceId"),
|
|
3865
|
+
),
|
|
3866
|
+
);
|
|
3867
|
+
}
|
|
3868
|
+
|
|
3869
|
+
getOccurrenceControl(occurrenceId) {
|
|
3870
|
+
this._assertOpen();
|
|
3871
|
+
return mapOccurrenceControl(
|
|
3872
|
+
this.statements.getOccurrenceControl.get(
|
|
3873
|
+
normalizeIdentifier(occurrenceId, "occurrenceId"),
|
|
3874
|
+
),
|
|
3875
|
+
);
|
|
3876
|
+
}
|
|
3877
|
+
|
|
3878
|
+
/**
|
|
3879
|
+
* Enumerate the small, public runtime-control surface used by Automation
|
|
3880
|
+
* Center. This intentionally does not reuse mapOccurrence(): payload,
|
|
3881
|
+
* authority, lease ownership, checkpoints, results and error bodies never
|
|
3882
|
+
* cross this read boundary.
|
|
3883
|
+
*/
|
|
3884
|
+
listRuntimeControlOccurrences({
|
|
3885
|
+
statuses = RUNTIME_CONTROL_OCCURRENCE_STATUSES,
|
|
3886
|
+
jobKinds = RUNTIME_CONTROL_JOB_KINDS,
|
|
3887
|
+
limit = 50,
|
|
3888
|
+
} = {}) {
|
|
3889
|
+
this._assertOpen();
|
|
3890
|
+
const selectedStatuses = normalizeRuntimeControlEnumeration(
|
|
3891
|
+
statuses,
|
|
3892
|
+
RUNTIME_CONTROL_OCCURRENCE_STATUS_SET,
|
|
3893
|
+
"statuses",
|
|
3894
|
+
);
|
|
3895
|
+
const selectedJobKinds = normalizeRuntimeControlEnumeration(
|
|
3896
|
+
jobKinds,
|
|
3897
|
+
RUNTIME_CONTROL_JOB_KIND_SET,
|
|
3898
|
+
"jobKinds",
|
|
3899
|
+
);
|
|
3900
|
+
if (!Number.isSafeInteger(limit) || limit < 1) {
|
|
3901
|
+
throw invalidArgument("limit must be a positive integer");
|
|
3902
|
+
}
|
|
3903
|
+
const boundedLimit = Math.min(limit, MAX_RUNTIME_CONTROL_OCCURRENCES);
|
|
3904
|
+
const statusParameters = selectedStatuses.map(() => "?").join(", ");
|
|
3905
|
+
const kindParameters = selectedJobKinds.map(() => "?").join(", ");
|
|
3906
|
+
const rows = this.db
|
|
3907
|
+
.prepare(
|
|
3908
|
+
`SELECT o.occurrence_id, o.job_id, j.kind AS job_kind,
|
|
3909
|
+
CASE
|
|
3910
|
+
WHEN c.state = 'pause_requested' THEN 'pause_requested'
|
|
3911
|
+
WHEN c.state = 'paused' THEN 'paused'
|
|
3912
|
+
ELSE 'running'
|
|
3913
|
+
END AS runtime_status,
|
|
3914
|
+
o.status AS occurrence_status, o.scheduled_for, o.attempt,
|
|
3915
|
+
o.max_attempts, o.fence, o.created_at, o.updated_at,
|
|
3916
|
+
c.state AS control_state, c.revision AS control_revision,
|
|
3917
|
+
c.expected_fence AS control_expected_fence,
|
|
3918
|
+
c.capability_digest AS control_capability_digest,
|
|
3919
|
+
c.requested_at AS control_requested_at,
|
|
3920
|
+
c.paused_at AS control_paused_at,
|
|
3921
|
+
c.updated_at AS control_updated_at
|
|
3922
|
+
FROM occurrences o
|
|
3923
|
+
JOIN jobs j ON j.job_id = o.job_id
|
|
3924
|
+
LEFT JOIN scheduler_occurrence_controls c
|
|
3925
|
+
ON c.occurrence_id = o.occurrence_id
|
|
3926
|
+
WHERE j.kind IN (${kindParameters})
|
|
3927
|
+
AND (
|
|
3928
|
+
(o.status = 'running' AND c.occurrence_id IS NULL)
|
|
3929
|
+
OR (o.status = 'running' AND c.state = 'pause_requested')
|
|
3930
|
+
OR (o.status = 'running' AND c.state = 'resumed')
|
|
3931
|
+
OR (o.status = 'retry_wait' AND c.state = 'paused')
|
|
3932
|
+
)
|
|
3933
|
+
AND CASE
|
|
3934
|
+
WHEN c.state = 'pause_requested' THEN 'pause_requested'
|
|
3935
|
+
WHEN c.state = 'paused' THEN 'paused'
|
|
3936
|
+
ELSE 'running'
|
|
3937
|
+
END IN (${statusParameters})
|
|
3938
|
+
ORDER BY o.updated_at DESC, o.occurrence_id ASC
|
|
3939
|
+
LIMIT ?`,
|
|
3940
|
+
)
|
|
3941
|
+
.all(...selectedJobKinds, ...selectedStatuses, boundedLimit);
|
|
3942
|
+
return rows.map(mapRuntimeControlOccurrence);
|
|
3943
|
+
}
|
|
3944
|
+
|
|
3945
|
+
requestOccurrencePause({
|
|
3946
|
+
occurrenceId,
|
|
3947
|
+
expectedFence,
|
|
3948
|
+
expectedRevision,
|
|
3949
|
+
requestId,
|
|
3950
|
+
capability,
|
|
3951
|
+
} = {}) {
|
|
3952
|
+
const id = normalizeIdentifier(occurrenceId, "occurrenceId");
|
|
3953
|
+
const fence = assertPositiveSafeInteger(expectedFence, "expectedFence");
|
|
3954
|
+
const request = normalizeIdentifier(requestId, "requestId", {
|
|
3955
|
+
maxLength: 128,
|
|
3956
|
+
});
|
|
3957
|
+
const revision =
|
|
3958
|
+
expectedRevision === undefined
|
|
3959
|
+
? null
|
|
3960
|
+
: (() => {
|
|
3961
|
+
if (
|
|
3962
|
+
!Number.isSafeInteger(expectedRevision) ||
|
|
3963
|
+
expectedRevision < 0
|
|
3964
|
+
) {
|
|
3965
|
+
throw invalidArgument(
|
|
3966
|
+
"expectedRevision must be a non-negative integer",
|
|
3967
|
+
);
|
|
3968
|
+
}
|
|
3969
|
+
return expectedRevision;
|
|
3970
|
+
})();
|
|
3971
|
+
const normalizedCapability = normalizeRuntimeControlCapability(capability);
|
|
3972
|
+
if (
|
|
3973
|
+
normalizedCapability.pauseResume !== RUNTIME_PAUSE_RESUME.CHECKPOINT_V1
|
|
3974
|
+
) {
|
|
3975
|
+
throw new SchedulerKernelError(
|
|
3976
|
+
"SCHEDULER_PAUSE_UNSUPPORTED",
|
|
3977
|
+
"Scheduler adapter does not declare durable checkpoint pause/resume",
|
|
3978
|
+
);
|
|
3979
|
+
}
|
|
3980
|
+
const capabilityJson = canonicalJson(
|
|
3981
|
+
normalizedCapability,
|
|
3982
|
+
"runtimeControl.capability",
|
|
3983
|
+
);
|
|
3984
|
+
const capabilityDigest =
|
|
3985
|
+
schedulerRuntimeControlCapabilityDigest(normalizedCapability);
|
|
3986
|
+
const now = this._now();
|
|
3987
|
+
return this._write(() => {
|
|
3988
|
+
const occurrence = this.statements.getOccurrence.get(id);
|
|
3989
|
+
if (!occurrence) {
|
|
3990
|
+
throw new SchedulerKernelError(
|
|
3991
|
+
"SCHEDULER_NOT_FOUND",
|
|
3992
|
+
`Scheduler occurrence does not exist: ${id}`,
|
|
3993
|
+
);
|
|
3994
|
+
}
|
|
3995
|
+
const requestOwner =
|
|
3996
|
+
this.statements.getOccurrenceControlByPauseRequest.get(request);
|
|
3997
|
+
if (requestOwner && requestOwner.occurrence_id !== id) {
|
|
3998
|
+
throw new SchedulerKernelError(
|
|
3999
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4000
|
+
"Scheduler pause request is already bound to another occurrence",
|
|
4001
|
+
);
|
|
4002
|
+
}
|
|
4003
|
+
const current = this.statements.getOccurrenceControl.get(id);
|
|
4004
|
+
if (current?.pause_request_id === request) {
|
|
4005
|
+
if (
|
|
4006
|
+
current.expected_fence !== fence ||
|
|
4007
|
+
current.capability_digest !== capabilityDigest
|
|
4008
|
+
) {
|
|
4009
|
+
throw new SchedulerKernelError(
|
|
4010
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4011
|
+
`Scheduler pause request evidence changed: ${id}`,
|
|
4012
|
+
);
|
|
4013
|
+
}
|
|
4014
|
+
return { ...mapOccurrenceControl(current), deduplicated: true };
|
|
4015
|
+
}
|
|
4016
|
+
const actualRevision = current?.revision ?? 0;
|
|
4017
|
+
if (revision !== null && actualRevision !== revision) {
|
|
4018
|
+
throw new SchedulerKernelError(
|
|
4019
|
+
"SCHEDULER_REVISION_CONFLICT",
|
|
4020
|
+
`Scheduler occurrence control revision changed: ${id}`,
|
|
4021
|
+
{ expectedRevision: revision, actualRevision },
|
|
4022
|
+
);
|
|
4023
|
+
}
|
|
4024
|
+
if (current && current.state !== "terminal") {
|
|
4025
|
+
throw new SchedulerKernelError(
|
|
4026
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4027
|
+
`Scheduler occurrence already has an active runtime control: ${id}`,
|
|
4028
|
+
{ state: current.state, revision: current.revision },
|
|
4029
|
+
);
|
|
4030
|
+
}
|
|
4031
|
+
if (occurrence.status !== OCCURRENCE_STATUS.RUNNING) {
|
|
4032
|
+
throw new SchedulerKernelError(
|
|
4033
|
+
"SCHEDULER_OCCURRENCE_NOT_RUNNING",
|
|
4034
|
+
`Scheduler occurrence is not running: ${id}`,
|
|
4035
|
+
{ status: occurrence.status },
|
|
4036
|
+
);
|
|
4037
|
+
}
|
|
4038
|
+
if (occurrence.fence !== fence) {
|
|
4039
|
+
throw new SchedulerKernelError(
|
|
4040
|
+
"SCHEDULER_FENCE_CONFLICT",
|
|
4041
|
+
`Scheduler occurrence fence changed: ${id}`,
|
|
4042
|
+
{ expectedFence: fence, actualFence: occurrence.fence },
|
|
4043
|
+
);
|
|
4044
|
+
}
|
|
4045
|
+
if (occurrence.lease_expires_at <= now) throw leaseLost(id);
|
|
4046
|
+
if (current) {
|
|
4047
|
+
this.db
|
|
4048
|
+
.prepare(
|
|
4049
|
+
`UPDATE scheduler_occurrence_controls
|
|
4050
|
+
SET capability_json = @capabilityJson,
|
|
4051
|
+
capability_digest = @capabilityDigest,
|
|
4052
|
+
state = 'pause_requested', revision = revision + 1,
|
|
4053
|
+
pause_request_id = @requestId, resume_request_id = NULL,
|
|
4054
|
+
expected_fence = @expectedFence, checkpoint_json = NULL,
|
|
4055
|
+
requested_at = @now, paused_at = NULL, resumed_at = NULL,
|
|
4056
|
+
terminal_at = NULL, updated_at = @now
|
|
4057
|
+
WHERE occurrence_id = @occurrenceId AND state = 'terminal'`,
|
|
4058
|
+
)
|
|
4059
|
+
.run({
|
|
4060
|
+
occurrenceId: id,
|
|
4061
|
+
capabilityJson,
|
|
4062
|
+
capabilityDigest,
|
|
4063
|
+
requestId: request,
|
|
4064
|
+
expectedFence: fence,
|
|
4065
|
+
now,
|
|
4066
|
+
});
|
|
4067
|
+
} else {
|
|
4068
|
+
this.db
|
|
4069
|
+
.prepare(
|
|
4070
|
+
`INSERT INTO scheduler_occurrence_controls
|
|
4071
|
+
(occurrence_id, capability_json, capability_digest, state,
|
|
4072
|
+
revision, pause_request_id, resume_request_id, expected_fence,
|
|
4073
|
+
checkpoint_json, requested_at, paused_at, resumed_at,
|
|
4074
|
+
terminal_at, updated_at)
|
|
4075
|
+
VALUES (@occurrenceId, @capabilityJson, @capabilityDigest,
|
|
4076
|
+
'pause_requested', 1, @requestId, NULL, @expectedFence,
|
|
4077
|
+
NULL, @now, NULL, NULL, NULL, @now)`,
|
|
4078
|
+
)
|
|
4079
|
+
.run({
|
|
4080
|
+
occurrenceId: id,
|
|
4081
|
+
capabilityJson,
|
|
4082
|
+
capabilityDigest,
|
|
4083
|
+
requestId: request,
|
|
4084
|
+
expectedFence: fence,
|
|
4085
|
+
now,
|
|
4086
|
+
});
|
|
4087
|
+
}
|
|
4088
|
+
const control = this.statements.getOccurrenceControl.get(id);
|
|
4089
|
+
this._appendEvent({
|
|
4090
|
+
jobId: occurrence.job_id,
|
|
4091
|
+
occurrenceId: id,
|
|
4092
|
+
type: "occurrence_pause_requested",
|
|
4093
|
+
occurredAt: now,
|
|
4094
|
+
fence,
|
|
4095
|
+
data: {
|
|
4096
|
+
requestId: request,
|
|
4097
|
+
controlRevision: control.revision,
|
|
4098
|
+
capabilityDigest,
|
|
4099
|
+
},
|
|
4100
|
+
});
|
|
4101
|
+
return { ...mapOccurrenceControl(control), deduplicated: false };
|
|
4102
|
+
});
|
|
4103
|
+
}
|
|
4104
|
+
|
|
4105
|
+
ackOccurrencePause({
|
|
4106
|
+
occurrenceId,
|
|
4107
|
+
ownerId,
|
|
4108
|
+
fence,
|
|
4109
|
+
requestId,
|
|
4110
|
+
expectedRevision,
|
|
4111
|
+
safePoint,
|
|
4112
|
+
checkpoint,
|
|
4113
|
+
} = {}) {
|
|
4114
|
+
const id = normalizeIdentifier(occurrenceId, "occurrenceId");
|
|
4115
|
+
const owner = normalizeIdentifier(ownerId, "ownerId");
|
|
4116
|
+
const token = assertPositiveSafeInteger(fence, "fence");
|
|
4117
|
+
const request = normalizeIdentifier(requestId, "requestId", {
|
|
4118
|
+
maxLength: 128,
|
|
4119
|
+
});
|
|
4120
|
+
const revision = assertPositiveSafeInteger(
|
|
4121
|
+
expectedRevision,
|
|
4122
|
+
"expectedRevision",
|
|
4123
|
+
);
|
|
4124
|
+
const point = normalizeIdentifier(safePoint, "safePoint", {
|
|
4125
|
+
maxLength: 64,
|
|
4126
|
+
});
|
|
4127
|
+
if (!Object.values(RUNTIME_CONTROL_SAFE_POINTS).includes(point)) {
|
|
4128
|
+
throw invalidArgument("safePoint is not a scheduler runtime safe point");
|
|
4129
|
+
}
|
|
4130
|
+
const normalizedCheckpoint = normalizeRuntimeCheckpoint({
|
|
4131
|
+
schemaVersion: 1,
|
|
4132
|
+
safePoint: point,
|
|
4133
|
+
data: checkpoint ?? null,
|
|
4134
|
+
});
|
|
4135
|
+
const now = this._now();
|
|
4136
|
+
return this._write(() => {
|
|
4137
|
+
const occurrence = this.statements.getOccurrence.get(id);
|
|
4138
|
+
const controlRow = this.statements.getOccurrenceControl.get(id);
|
|
4139
|
+
if (!occurrence || !controlRow) {
|
|
4140
|
+
throw new SchedulerKernelError(
|
|
4141
|
+
"SCHEDULER_NOT_FOUND",
|
|
4142
|
+
`Scheduler occurrence control does not exist: ${id}`,
|
|
4143
|
+
);
|
|
4144
|
+
}
|
|
4145
|
+
const control = mapOccurrenceControl(controlRow);
|
|
4146
|
+
if (
|
|
4147
|
+
control.state !== "pause_requested" ||
|
|
4148
|
+
control.pauseRequestId !== request
|
|
4149
|
+
) {
|
|
4150
|
+
throw new SchedulerKernelError(
|
|
4151
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4152
|
+
`Scheduler pause request is no longer pending: ${id}`,
|
|
4153
|
+
{ state: control.state, revision: control.revision },
|
|
4154
|
+
);
|
|
4155
|
+
}
|
|
4156
|
+
if (control.revision !== revision) {
|
|
4157
|
+
throw new SchedulerKernelError(
|
|
4158
|
+
"SCHEDULER_REVISION_CONFLICT",
|
|
4159
|
+
`Scheduler occurrence control revision changed: ${id}`,
|
|
4160
|
+
{ expectedRevision: revision, actualRevision: control.revision },
|
|
4161
|
+
);
|
|
4162
|
+
}
|
|
4163
|
+
if (
|
|
4164
|
+
control.expectedFence !== token ||
|
|
4165
|
+
occurrence.status !== OCCURRENCE_STATUS.RUNNING ||
|
|
4166
|
+
occurrence.fence !== token ||
|
|
4167
|
+
occurrence.lease_owner !== owner ||
|
|
4168
|
+
occurrence.lease_expires_at <= now
|
|
4169
|
+
) {
|
|
4170
|
+
throw leaseLost(id);
|
|
4171
|
+
}
|
|
4172
|
+
if (!control.capability.safePoints.includes(point)) {
|
|
4173
|
+
throw new SchedulerKernelError(
|
|
4174
|
+
"SCHEDULER_PAUSE_UNSUPPORTED_SAFE_POINT",
|
|
4175
|
+
`Scheduler adapter did not declare pause safety at ${point}`,
|
|
4176
|
+
);
|
|
4177
|
+
}
|
|
4178
|
+
const occurrenceUpdate = this.db
|
|
4179
|
+
.prepare(
|
|
4180
|
+
`UPDATE occurrences
|
|
4181
|
+
SET status = 'retry_wait', available_at = @now,
|
|
4182
|
+
lease_owner = NULL, lease_expires_at = NULL,
|
|
4183
|
+
updated_at = @now, settled_at = NULL
|
|
4184
|
+
WHERE occurrence_id = @occurrenceId AND status = 'running'
|
|
4185
|
+
AND lease_owner = @ownerId AND fence = @fence
|
|
4186
|
+
AND lease_expires_at > @now`,
|
|
4187
|
+
)
|
|
4188
|
+
.run({ occurrenceId: id, ownerId: owner, fence: token, now });
|
|
4189
|
+
if (occurrenceUpdate.changes !== 1) throw leaseLost(id);
|
|
4190
|
+
const controlUpdate = this.db
|
|
4191
|
+
.prepare(
|
|
4192
|
+
`UPDATE scheduler_occurrence_controls
|
|
4193
|
+
SET state = 'paused', revision = revision + 1,
|
|
4194
|
+
checkpoint_json = @checkpointJson,
|
|
4195
|
+
paused_at = @now, updated_at = @now
|
|
4196
|
+
WHERE occurrence_id = @occurrenceId
|
|
4197
|
+
AND state = 'pause_requested' AND revision = @revision
|
|
4198
|
+
AND pause_request_id = @requestId AND expected_fence = @fence`,
|
|
4199
|
+
)
|
|
4200
|
+
.run({
|
|
4201
|
+
occurrenceId: id,
|
|
4202
|
+
checkpointJson: normalizedCheckpoint.encoded,
|
|
4203
|
+
now,
|
|
4204
|
+
revision,
|
|
4205
|
+
requestId: request,
|
|
4206
|
+
fence: token,
|
|
4207
|
+
});
|
|
4208
|
+
if (controlUpdate.changes !== 1) {
|
|
4209
|
+
throw new SchedulerKernelError(
|
|
4210
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4211
|
+
`Scheduler pause acknowledgement lost its CAS: ${id}`,
|
|
4212
|
+
);
|
|
4213
|
+
}
|
|
4214
|
+
this._appendEvent({
|
|
4215
|
+
jobId: occurrence.job_id,
|
|
4216
|
+
occurrenceId: id,
|
|
4217
|
+
type: "occurrence_paused",
|
|
4218
|
+
occurredAt: now,
|
|
4219
|
+
fence: token,
|
|
4220
|
+
data: {
|
|
4221
|
+
requestId: request,
|
|
4222
|
+
safePoint: point,
|
|
4223
|
+
checkpointDigest: sha256PayloadDigest(
|
|
4224
|
+
normalizedCheckpoint.checkpoint,
|
|
4225
|
+
"runtimeControl.checkpoint",
|
|
4226
|
+
),
|
|
4227
|
+
},
|
|
4228
|
+
});
|
|
4229
|
+
return {
|
|
4230
|
+
control: mapOccurrenceControl(
|
|
4231
|
+
this.statements.getOccurrenceControl.get(id),
|
|
4232
|
+
),
|
|
4233
|
+
occurrence: mapOccurrence(this.statements.getOccurrence.get(id)),
|
|
4234
|
+
};
|
|
4235
|
+
});
|
|
4236
|
+
}
|
|
4237
|
+
|
|
4238
|
+
resumeOccurrence({ occurrenceId, expectedRevision, requestId } = {}) {
|
|
4239
|
+
const id = normalizeIdentifier(occurrenceId, "occurrenceId");
|
|
4240
|
+
const revision = assertPositiveSafeInteger(
|
|
4241
|
+
expectedRevision,
|
|
4242
|
+
"expectedRevision",
|
|
4243
|
+
);
|
|
4244
|
+
const request = normalizeIdentifier(requestId, "requestId", {
|
|
4245
|
+
maxLength: 128,
|
|
4246
|
+
});
|
|
4247
|
+
const now = this._now();
|
|
4248
|
+
return this._write(() => {
|
|
4249
|
+
const occurrence = this.statements.getOccurrence.get(id);
|
|
4250
|
+
const controlRow = this.statements.getOccurrenceControl.get(id);
|
|
4251
|
+
if (!occurrence || !controlRow) {
|
|
4252
|
+
throw new SchedulerKernelError(
|
|
4253
|
+
"SCHEDULER_NOT_FOUND",
|
|
4254
|
+
`Scheduler occurrence control does not exist: ${id}`,
|
|
4255
|
+
);
|
|
4256
|
+
}
|
|
4257
|
+
const requestOwner =
|
|
4258
|
+
this.statements.getOccurrenceControlByResumeRequest.get(request);
|
|
4259
|
+
if (requestOwner && requestOwner.occurrence_id !== id) {
|
|
4260
|
+
throw new SchedulerKernelError(
|
|
4261
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4262
|
+
"Scheduler resume request is already bound to another occurrence",
|
|
4263
|
+
);
|
|
4264
|
+
}
|
|
4265
|
+
const control = mapOccurrenceControl(controlRow);
|
|
4266
|
+
if (control.resumeRequestId === request) {
|
|
4267
|
+
return {
|
|
4268
|
+
control: { ...control, deduplicated: true },
|
|
4269
|
+
occurrence: mapOccurrence(occurrence),
|
|
4270
|
+
};
|
|
4271
|
+
}
|
|
4272
|
+
if (control.state !== "paused") {
|
|
4273
|
+
throw new SchedulerKernelError(
|
|
4274
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4275
|
+
`Scheduler occurrence is not paused: ${id}`,
|
|
4276
|
+
{ state: control.state, revision: control.revision },
|
|
4277
|
+
);
|
|
4278
|
+
}
|
|
4279
|
+
if (control.revision !== revision) {
|
|
4280
|
+
throw new SchedulerKernelError(
|
|
4281
|
+
"SCHEDULER_REVISION_CONFLICT",
|
|
4282
|
+
`Scheduler occurrence control revision changed: ${id}`,
|
|
4283
|
+
{ expectedRevision: revision, actualRevision: control.revision },
|
|
4284
|
+
);
|
|
4285
|
+
}
|
|
4286
|
+
if (
|
|
4287
|
+
occurrence.status !== OCCURRENCE_STATUS.RETRY_WAIT ||
|
|
4288
|
+
occurrence.lease_owner !== null ||
|
|
4289
|
+
occurrence.lease_expires_at !== null
|
|
4290
|
+
) {
|
|
4291
|
+
throw new SchedulerKernelError(
|
|
4292
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4293
|
+
`Paused occurrence state is inconsistent: ${id}`,
|
|
4294
|
+
);
|
|
4295
|
+
}
|
|
4296
|
+
const update = this.db
|
|
4297
|
+
.prepare(
|
|
4298
|
+
`UPDATE scheduler_occurrence_controls
|
|
4299
|
+
SET state = 'resumed', revision = revision + 1,
|
|
4300
|
+
resume_request_id = @requestId, resumed_at = @now,
|
|
4301
|
+
updated_at = @now
|
|
4302
|
+
WHERE occurrence_id = @occurrenceId AND state = 'paused'
|
|
4303
|
+
AND revision = @revision`,
|
|
4304
|
+
)
|
|
4305
|
+
.run({ occurrenceId: id, requestId: request, now, revision });
|
|
4306
|
+
if (update.changes !== 1) {
|
|
4307
|
+
throw new SchedulerKernelError(
|
|
4308
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4309
|
+
`Scheduler resume request lost its CAS: ${id}`,
|
|
4310
|
+
);
|
|
4311
|
+
}
|
|
4312
|
+
this.db
|
|
4313
|
+
.prepare(
|
|
4314
|
+
`UPDATE occurrences SET available_at = ?, updated_at = ?
|
|
4315
|
+
WHERE occurrence_id = ? AND status = 'retry_wait'`,
|
|
4316
|
+
)
|
|
4317
|
+
.run(now, now, id);
|
|
4318
|
+
this._appendEvent({
|
|
4319
|
+
jobId: occurrence.job_id,
|
|
4320
|
+
occurrenceId: id,
|
|
4321
|
+
type: "occurrence_resume_requested",
|
|
4322
|
+
occurredAt: now,
|
|
4323
|
+
fence: occurrence.fence,
|
|
4324
|
+
data: { requestId: request, controlRevision: revision + 1 },
|
|
4325
|
+
});
|
|
4326
|
+
return {
|
|
4327
|
+
control: mapOccurrenceControl(
|
|
4328
|
+
this.statements.getOccurrenceControl.get(id),
|
|
4329
|
+
),
|
|
4330
|
+
occurrence: mapOccurrence(this.statements.getOccurrence.get(id)),
|
|
4331
|
+
};
|
|
4332
|
+
});
|
|
4333
|
+
}
|
|
4334
|
+
|
|
4335
|
+
requeueDeadLetter({
|
|
4336
|
+
occurrenceId,
|
|
4337
|
+
expectedFence,
|
|
4338
|
+
expectedErrorCode,
|
|
4339
|
+
requestId,
|
|
1280
4340
|
} = {}) {
|
|
1281
|
-
const id = normalizeIdentifier(
|
|
1282
|
-
const
|
|
1283
|
-
const
|
|
1284
|
-
|
|
4341
|
+
const id = normalizeIdentifier(occurrenceId, "occurrenceId");
|
|
4342
|
+
const fence = assertPositiveSafeInteger(expectedFence, "expectedFence");
|
|
4343
|
+
const errorCode = normalizeIdentifier(
|
|
4344
|
+
expectedErrorCode,
|
|
4345
|
+
"expectedErrorCode",
|
|
4346
|
+
{
|
|
4347
|
+
maxLength: 128,
|
|
4348
|
+
},
|
|
4349
|
+
);
|
|
4350
|
+
const request = normalizeIdentifier(requestId, "requestId", {
|
|
4351
|
+
maxLength: 128,
|
|
4352
|
+
});
|
|
1285
4353
|
const now = this._now();
|
|
1286
4354
|
return this._write(() => {
|
|
1287
|
-
const
|
|
1288
|
-
if (
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
4355
|
+
const existing = this.statements.getOccurrenceRetryByRequest.get(request);
|
|
4356
|
+
if (existing) {
|
|
4357
|
+
if (
|
|
4358
|
+
existing.occurrence_id !== id ||
|
|
4359
|
+
existing.expected_fence !== fence ||
|
|
4360
|
+
existing.expected_error_code !== errorCode
|
|
4361
|
+
) {
|
|
4362
|
+
throw new SchedulerKernelError(
|
|
4363
|
+
"SCHEDULER_REQUEUE_CONFLICT",
|
|
4364
|
+
"Scheduler retry request evidence changed",
|
|
4365
|
+
);
|
|
4366
|
+
}
|
|
4367
|
+
return {
|
|
4368
|
+
occurrence: mapOccurrence(this.statements.getOccurrence.get(id)),
|
|
4369
|
+
requestId: request,
|
|
4370
|
+
deduplicated: true,
|
|
4371
|
+
};
|
|
4372
|
+
}
|
|
4373
|
+
const occurrence = this.statements.getOccurrence.get(id);
|
|
4374
|
+
if (!occurrence) {
|
|
4375
|
+
throw new SchedulerKernelError(
|
|
4376
|
+
"SCHEDULER_NOT_FOUND",
|
|
4377
|
+
`Scheduler occurrence does not exist: ${id}`,
|
|
1292
4378
|
);
|
|
1293
4379
|
}
|
|
1294
|
-
|
|
1295
|
-
|
|
4380
|
+
const lastError =
|
|
4381
|
+
occurrence.last_error_json === null
|
|
4382
|
+
? null
|
|
4383
|
+
: readStoredJson(occurrence.last_error_json, "occurrence error");
|
|
4384
|
+
if (
|
|
4385
|
+
occurrence.status !== OCCURRENCE_STATUS.DEAD_LETTER ||
|
|
4386
|
+
occurrence.fence !== fence ||
|
|
4387
|
+
lastError?.code !== errorCode
|
|
4388
|
+
) {
|
|
4389
|
+
throw new SchedulerKernelError(
|
|
4390
|
+
"SCHEDULER_REQUEUE_EVIDENCE_CONFLICT",
|
|
4391
|
+
`Scheduler dead-letter evidence changed: ${id}`,
|
|
4392
|
+
{
|
|
4393
|
+
status: occurrence.status,
|
|
4394
|
+
expectedFence: fence,
|
|
4395
|
+
actualFence: occurrence.fence,
|
|
4396
|
+
expectedErrorCode: errorCode,
|
|
4397
|
+
actualErrorCode: lastError?.code ?? null,
|
|
4398
|
+
},
|
|
4399
|
+
);
|
|
1296
4400
|
}
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
4401
|
+
this.db
|
|
4402
|
+
.prepare(
|
|
4403
|
+
`INSERT INTO scheduler_occurrence_retries
|
|
4404
|
+
(occurrence_id, request_id, expected_fence,
|
|
4405
|
+
expected_error_code, created_at, claimed_at)
|
|
4406
|
+
VALUES (?, ?, ?, ?, ?, NULL)`,
|
|
4407
|
+
)
|
|
4408
|
+
.run(id, request, fence, errorCode, now);
|
|
4409
|
+
const update = this.db
|
|
4410
|
+
.prepare(
|
|
4411
|
+
`UPDATE occurrences
|
|
4412
|
+
SET status = 'retry_wait', available_at = ?, updated_at = ?,
|
|
4413
|
+
settled_at = NULL
|
|
4414
|
+
WHERE occurrence_id = ? AND status = 'dead_letter' AND fence = ?`,
|
|
4415
|
+
)
|
|
4416
|
+
.run(now, now, id, fence);
|
|
4417
|
+
if (update.changes !== 1) {
|
|
4418
|
+
throw new SchedulerKernelError(
|
|
4419
|
+
"SCHEDULER_REQUEUE_CONFLICT",
|
|
4420
|
+
`Scheduler dead letter changed during requeue: ${id}`,
|
|
4421
|
+
);
|
|
4422
|
+
}
|
|
4423
|
+
this._appendEvent({
|
|
4424
|
+
jobId: occurrence.job_id,
|
|
4425
|
+
occurrenceId: id,
|
|
4426
|
+
type: "occurrence_requeued",
|
|
4427
|
+
occurredAt: now,
|
|
4428
|
+
fence,
|
|
4429
|
+
data: { requestId: request, expectedErrorCode: errorCode },
|
|
1304
4430
|
});
|
|
4431
|
+
return {
|
|
4432
|
+
occurrence: mapOccurrence(this.statements.getOccurrence.get(id)),
|
|
4433
|
+
requestId: request,
|
|
4434
|
+
deduplicated: false,
|
|
4435
|
+
};
|
|
1305
4436
|
});
|
|
1306
4437
|
}
|
|
1307
4438
|
|
|
1308
|
-
getOccurrence(occurrenceId) {
|
|
1309
|
-
this._assertOpen();
|
|
1310
|
-
return mapOccurrence(
|
|
1311
|
-
this.statements.getOccurrence.get(
|
|
1312
|
-
normalizeIdentifier(occurrenceId, "occurrenceId"),
|
|
1313
|
-
),
|
|
1314
|
-
);
|
|
1315
|
-
}
|
|
1316
|
-
|
|
1317
4439
|
listOccurrencesByTrigger({ jobId, triggerKey, limit = 2 } = {}) {
|
|
1318
4440
|
this._assertOpen();
|
|
1319
4441
|
const id = normalizeIdentifier(jobId, "jobId");
|
|
@@ -1336,6 +4458,11 @@ export class SchedulerStore {
|
|
|
1336
4458
|
WHERE status = 'running'
|
|
1337
4459
|
AND lease_expires_at <= ?
|
|
1338
4460
|
AND attempt >= max_attempts
|
|
4461
|
+
AND NOT EXISTS (
|
|
4462
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4463
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4464
|
+
AND c.state IN ('pause_requested', 'resumed')
|
|
4465
|
+
)
|
|
1339
4466
|
ORDER BY lease_expires_at, occurrence_id
|
|
1340
4467
|
LIMIT 100
|
|
1341
4468
|
`,
|
|
@@ -1355,6 +4482,11 @@ export class SchedulerStore {
|
|
|
1355
4482
|
AND fence = @fence
|
|
1356
4483
|
AND lease_expires_at <= @now
|
|
1357
4484
|
AND attempt >= max_attempts
|
|
4485
|
+
AND NOT EXISTS (
|
|
4486
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4487
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4488
|
+
AND c.state IN ('pause_requested', 'resumed')
|
|
4489
|
+
)
|
|
1358
4490
|
`);
|
|
1359
4491
|
for (const row of expired) {
|
|
1360
4492
|
const error = {
|
|
@@ -1369,6 +4501,42 @@ export class SchedulerStore {
|
|
|
1369
4501
|
now,
|
|
1370
4502
|
});
|
|
1371
4503
|
if (result.changes === 1) {
|
|
4504
|
+
const adjudication = this.db
|
|
4505
|
+
.prepare(
|
|
4506
|
+
`SELECT * FROM scheduler_occurrence_adjudications
|
|
4507
|
+
WHERE occurrence_id = ? AND status = 'pending'`,
|
|
4508
|
+
)
|
|
4509
|
+
.get(row.occurrence_id);
|
|
4510
|
+
if (adjudication) {
|
|
4511
|
+
const retryOutcome = {
|
|
4512
|
+
status: OCCURRENCE_STATUS.DEAD_LETTER,
|
|
4513
|
+
error,
|
|
4514
|
+
};
|
|
4515
|
+
this.db
|
|
4516
|
+
.prepare(
|
|
4517
|
+
`UPDATE scheduler_occurrence_adjudications
|
|
4518
|
+
SET status = 'applied', applied_at = ?, retry_settled_at = ?,
|
|
4519
|
+
retry_outcome_json = ?
|
|
4520
|
+
WHERE occurrence_id = ? AND status = 'pending'`,
|
|
4521
|
+
)
|
|
4522
|
+
.run(
|
|
4523
|
+
now,
|
|
4524
|
+
now,
|
|
4525
|
+
canonicalJson(retryOutcome, "adjudication.retryOutcome"),
|
|
4526
|
+
row.occurrence_id,
|
|
4527
|
+
);
|
|
4528
|
+
this.db
|
|
4529
|
+
.prepare(
|
|
4530
|
+
`UPDATE scheduler_authority_reservations
|
|
4531
|
+
SET status = 'failed', outcome_json = ?, settled_at = ?
|
|
4532
|
+
WHERE occurrence_id = ? AND status = 'reserved'`,
|
|
4533
|
+
)
|
|
4534
|
+
.run(
|
|
4535
|
+
canonicalJson(retryOutcome, "authoritySettlement"),
|
|
4536
|
+
now,
|
|
4537
|
+
row.occurrence_id,
|
|
4538
|
+
);
|
|
4539
|
+
}
|
|
1372
4540
|
this._appendEvent({
|
|
1373
4541
|
jobId: row.job_id,
|
|
1374
4542
|
occurrenceId: row.occurrence_id,
|
|
@@ -1380,6 +4548,9 @@ export class SchedulerStore {
|
|
|
1380
4548
|
attempt: row.attempt,
|
|
1381
4549
|
maxAttempts: row.max_attempts,
|
|
1382
4550
|
reason: "lease_expired",
|
|
4551
|
+
...(adjudication
|
|
4552
|
+
? { adjudicationRequestId: adjudication.request_id }
|
|
4553
|
+
: {}),
|
|
1383
4554
|
},
|
|
1384
4555
|
});
|
|
1385
4556
|
}
|
|
@@ -1397,26 +4568,43 @@ export class SchedulerStore {
|
|
|
1397
4568
|
workspaceId === undefined
|
|
1398
4569
|
? null
|
|
1399
4570
|
: normalizeIdentifier(workspaceId, "workspaceId", { maxLength: 256 });
|
|
1400
|
-
const now = this._now();
|
|
1401
|
-
const leaseExpiresAt = now + lease;
|
|
1402
|
-
if (!Number.isSafeInteger(leaseExpiresAt)) {
|
|
1403
|
-
throw invalidArgument("lease expiry exceeds the safe integer range");
|
|
1404
|
-
}
|
|
1405
4571
|
return this._write(() => {
|
|
4572
|
+
const now = this._now();
|
|
4573
|
+
const leaseExpiresAt = now + lease;
|
|
4574
|
+
if (!Number.isSafeInteger(leaseExpiresAt)) {
|
|
4575
|
+
throw invalidArgument("lease expiry exceeds the safe integer range");
|
|
4576
|
+
}
|
|
1406
4577
|
this._deadLetterExpiredLeases(now);
|
|
1407
4578
|
const candidate = this.db
|
|
1408
4579
|
.prepare(
|
|
1409
4580
|
`
|
|
1410
|
-
SELECT o
|
|
4581
|
+
SELECT o.*, c.state AS control_state,
|
|
4582
|
+
c.expected_fence AS control_expected_fence,
|
|
4583
|
+
r.request_id AS retry_request_id
|
|
1411
4584
|
FROM occurrences o
|
|
1412
4585
|
JOIN jobs j ON j.job_id = o.job_id
|
|
4586
|
+
LEFT JOIN scheduler_occurrence_controls c
|
|
4587
|
+
ON c.occurrence_id = o.occurrence_id
|
|
4588
|
+
LEFT JOIN scheduler_occurrence_retries r
|
|
4589
|
+
ON r.occurrence_id = o.occurrence_id
|
|
4590
|
+
AND r.claimed_at IS NULL
|
|
4591
|
+
AND r.expected_fence = o.fence
|
|
1413
4592
|
WHERE j.enabled = 1
|
|
1414
4593
|
AND (@jobKind IS NULL OR j.kind = @jobKind)
|
|
1415
4594
|
AND (
|
|
1416
4595
|
@workspaceId IS NULL
|
|
1417
4596
|
OR json_extract(j.authority_json, '$.workspaceId') = @workspaceId
|
|
1418
4597
|
)
|
|
1419
|
-
AND
|
|
4598
|
+
AND (
|
|
4599
|
+
o.attempt < o.max_attempts
|
|
4600
|
+
OR c.state IN ('pause_requested', 'resumed')
|
|
4601
|
+
OR r.request_id IS NOT NULL
|
|
4602
|
+
)
|
|
4603
|
+
AND (c.state IS NULL OR c.state <> 'paused')
|
|
4604
|
+
AND (
|
|
4605
|
+
c.state IS NULL OR c.state <> 'pause_requested'
|
|
4606
|
+
OR o.status = 'running'
|
|
4607
|
+
)
|
|
1420
4608
|
AND (
|
|
1421
4609
|
(o.status IN ('queued', 'retry_wait') AND o.available_at <= @now)
|
|
1422
4610
|
OR
|
|
@@ -1438,7 +4626,25 @@ export class SchedulerStore {
|
|
|
1438
4626
|
`
|
|
1439
4627
|
UPDATE occurrences
|
|
1440
4628
|
SET status = 'running',
|
|
1441
|
-
attempt = attempt +
|
|
4629
|
+
attempt = attempt + CASE
|
|
4630
|
+
WHEN EXISTS (
|
|
4631
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4632
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4633
|
+
AND c.state = 'pause_requested'
|
|
4634
|
+
) THEN 0
|
|
4635
|
+
WHEN EXISTS (
|
|
4636
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4637
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4638
|
+
AND c.state = 'resumed'
|
|
4639
|
+
AND c.expected_fence = occurrences.fence
|
|
4640
|
+
) THEN 0
|
|
4641
|
+
WHEN EXISTS (
|
|
4642
|
+
SELECT 1 FROM scheduler_occurrence_retries r
|
|
4643
|
+
WHERE r.occurrence_id = occurrences.occurrence_id
|
|
4644
|
+
AND r.claimed_at IS NULL
|
|
4645
|
+
AND r.expected_fence = occurrences.fence
|
|
4646
|
+
) THEN 0
|
|
4647
|
+
ELSE 1 END,
|
|
1442
4648
|
fence = fence + 1,
|
|
1443
4649
|
lease_owner = @ownerId,
|
|
1444
4650
|
lease_expires_at = @leaseExpiresAt,
|
|
@@ -1446,7 +4652,25 @@ export class SchedulerStore {
|
|
|
1446
4652
|
settled_at = NULL
|
|
1447
4653
|
WHERE occurrence_id = @occurrenceId
|
|
1448
4654
|
AND fence = @expectedFence
|
|
1449
|
-
AND
|
|
4655
|
+
AND (
|
|
4656
|
+
attempt < max_attempts
|
|
4657
|
+
OR EXISTS (
|
|
4658
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4659
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4660
|
+
AND c.state IN ('pause_requested', 'resumed')
|
|
4661
|
+
)
|
|
4662
|
+
OR EXISTS (
|
|
4663
|
+
SELECT 1 FROM scheduler_occurrence_retries r
|
|
4664
|
+
WHERE r.occurrence_id = occurrences.occurrence_id
|
|
4665
|
+
AND r.claimed_at IS NULL
|
|
4666
|
+
AND r.expected_fence = occurrences.fence
|
|
4667
|
+
)
|
|
4668
|
+
)
|
|
4669
|
+
AND NOT EXISTS (
|
|
4670
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4671
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4672
|
+
AND c.state = 'paused'
|
|
4673
|
+
)
|
|
1450
4674
|
AND (
|
|
1451
4675
|
(status IN ('queued', 'retry_wait') AND available_at <= @now)
|
|
1452
4676
|
OR
|
|
@@ -1470,6 +4694,49 @@ export class SchedulerStore {
|
|
|
1470
4694
|
const claimed = this.statements.getOccurrence.get(
|
|
1471
4695
|
candidate.occurrence_id,
|
|
1472
4696
|
);
|
|
4697
|
+
if (candidate.control_state === "pause_requested") {
|
|
4698
|
+
this.db
|
|
4699
|
+
.prepare(
|
|
4700
|
+
`UPDATE scheduler_occurrence_controls
|
|
4701
|
+
SET expected_fence = ?, revision = revision + 1, updated_at = ?
|
|
4702
|
+
WHERE occurrence_id = ? AND state = 'pause_requested'
|
|
4703
|
+
AND expected_fence = ?`,
|
|
4704
|
+
)
|
|
4705
|
+
.run(
|
|
4706
|
+
claimed.fence,
|
|
4707
|
+
now,
|
|
4708
|
+
candidate.occurrence_id,
|
|
4709
|
+
candidate.control_expected_fence,
|
|
4710
|
+
);
|
|
4711
|
+
} else if (candidate.control_state === "resumed") {
|
|
4712
|
+
const rebound = this.db
|
|
4713
|
+
.prepare(
|
|
4714
|
+
`UPDATE scheduler_occurrence_controls
|
|
4715
|
+
SET expected_fence = ?, revision = revision + 1, updated_at = ?
|
|
4716
|
+
WHERE occurrence_id = ? AND state = 'resumed'
|
|
4717
|
+
AND expected_fence = ?`,
|
|
4718
|
+
)
|
|
4719
|
+
.run(
|
|
4720
|
+
claimed.fence,
|
|
4721
|
+
now,
|
|
4722
|
+
candidate.occurrence_id,
|
|
4723
|
+
candidate.control_expected_fence,
|
|
4724
|
+
);
|
|
4725
|
+
if (rebound.changes !== 1) {
|
|
4726
|
+
throw new SchedulerKernelError(
|
|
4727
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4728
|
+
`Scheduler resume checkpoint lost its claim fence: ${candidate.occurrence_id}`,
|
|
4729
|
+
);
|
|
4730
|
+
}
|
|
4731
|
+
}
|
|
4732
|
+
if (candidate.retry_request_id !== null) {
|
|
4733
|
+
this.db
|
|
4734
|
+
.prepare(
|
|
4735
|
+
`UPDATE scheduler_occurrence_retries SET claimed_at = ?
|
|
4736
|
+
WHERE occurrence_id = ? AND request_id = ? AND claimed_at IS NULL`,
|
|
4737
|
+
)
|
|
4738
|
+
.run(now, candidate.occurrence_id, candidate.retry_request_id);
|
|
4739
|
+
}
|
|
1473
4740
|
this._appendEvent({
|
|
1474
4741
|
jobId: candidate.job_id,
|
|
1475
4742
|
occurrenceId: candidate.occurrence_id,
|
|
@@ -1484,6 +4751,15 @@ export class SchedulerStore {
|
|
|
1484
4751
|
attempt: claimed.attempt,
|
|
1485
4752
|
previousOwner,
|
|
1486
4753
|
previousStatus,
|
|
4754
|
+
...(candidate.control_state === "pause_requested"
|
|
4755
|
+
? { pauseRequestRebound: true }
|
|
4756
|
+
: {}),
|
|
4757
|
+
...(candidate.control_state === "resumed"
|
|
4758
|
+
? { resumedWithoutAttempt: true }
|
|
4759
|
+
: {}),
|
|
4760
|
+
...(candidate.retry_request_id === null
|
|
4761
|
+
? {}
|
|
4762
|
+
: { retryRequestId: candidate.retry_request_id }),
|
|
1487
4763
|
},
|
|
1488
4764
|
});
|
|
1489
4765
|
return mapOccurrence(claimed);
|
|
@@ -1503,12 +4779,12 @@ export class SchedulerStore {
|
|
|
1503
4779
|
const id = normalizeIdentifier(occurrenceId, "occurrenceId");
|
|
1504
4780
|
const owner = normalizeIdentifier(ownerId, "ownerId");
|
|
1505
4781
|
const lease = normalizeLeaseMs(leaseMs);
|
|
1506
|
-
const now = this._now();
|
|
1507
|
-
const leaseExpiresAt = now + lease;
|
|
1508
|
-
if (!Number.isSafeInteger(leaseExpiresAt)) {
|
|
1509
|
-
throw invalidArgument("lease expiry exceeds the safe integer range");
|
|
1510
|
-
}
|
|
1511
4782
|
return this._write(() => {
|
|
4783
|
+
const now = this._now();
|
|
4784
|
+
const leaseExpiresAt = now + lease;
|
|
4785
|
+
if (!Number.isSafeInteger(leaseExpiresAt)) {
|
|
4786
|
+
throw invalidArgument("lease expiry exceeds the safe integer range");
|
|
4787
|
+
}
|
|
1512
4788
|
this._deadLetterExpiredLeases(now);
|
|
1513
4789
|
const candidate = this.statements.getOccurrence.get(id);
|
|
1514
4790
|
if (!candidate) {
|
|
@@ -1518,9 +4794,24 @@ export class SchedulerStore {
|
|
|
1518
4794
|
);
|
|
1519
4795
|
}
|
|
1520
4796
|
const job = this.statements.getJob.get(candidate.job_id);
|
|
4797
|
+
const controlRow = this.statements.getOccurrenceControl.get(id);
|
|
4798
|
+
const retryRow = this.db
|
|
4799
|
+
.prepare(
|
|
4800
|
+
`SELECT * FROM scheduler_occurrence_retries
|
|
4801
|
+
WHERE occurrence_id = ? AND claimed_at IS NULL
|
|
4802
|
+
AND expected_fence = ? ORDER BY created_at, request_id LIMIT 1`,
|
|
4803
|
+
)
|
|
4804
|
+
.get(id, candidate.fence);
|
|
1521
4805
|
const eligible =
|
|
1522
4806
|
job?.enabled === 1 &&
|
|
1523
|
-
candidate.attempt < candidate.max_attempts
|
|
4807
|
+
(candidate.attempt < candidate.max_attempts ||
|
|
4808
|
+
["pause_requested", "resumed"].includes(controlRow?.state) ||
|
|
4809
|
+
retryRow !== undefined) &&
|
|
4810
|
+
controlRow?.state !== "paused" &&
|
|
4811
|
+
!(
|
|
4812
|
+
controlRow?.state === "pause_requested" &&
|
|
4813
|
+
candidate.status !== OCCURRENCE_STATUS.RUNNING
|
|
4814
|
+
) &&
|
|
1524
4815
|
(([OCCURRENCE_STATUS.QUEUED, OCCURRENCE_STATUS.RETRY_WAIT].includes(
|
|
1525
4816
|
candidate.status,
|
|
1526
4817
|
) &&
|
|
@@ -1536,7 +4827,25 @@ export class SchedulerStore {
|
|
|
1536
4827
|
`
|
|
1537
4828
|
UPDATE occurrences
|
|
1538
4829
|
SET status = 'running',
|
|
1539
|
-
attempt = attempt +
|
|
4830
|
+
attempt = attempt + CASE
|
|
4831
|
+
WHEN EXISTS (
|
|
4832
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4833
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4834
|
+
AND c.state = 'pause_requested'
|
|
4835
|
+
) THEN 0
|
|
4836
|
+
WHEN EXISTS (
|
|
4837
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4838
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4839
|
+
AND c.state = 'resumed'
|
|
4840
|
+
AND c.expected_fence = occurrences.fence
|
|
4841
|
+
) THEN 0
|
|
4842
|
+
WHEN EXISTS (
|
|
4843
|
+
SELECT 1 FROM scheduler_occurrence_retries r
|
|
4844
|
+
WHERE r.occurrence_id = occurrences.occurrence_id
|
|
4845
|
+
AND r.claimed_at IS NULL
|
|
4846
|
+
AND r.expected_fence = occurrences.fence
|
|
4847
|
+
) THEN 0
|
|
4848
|
+
ELSE 1 END,
|
|
1540
4849
|
fence = fence + 1,
|
|
1541
4850
|
lease_owner = @ownerId,
|
|
1542
4851
|
lease_expires_at = @leaseExpiresAt,
|
|
@@ -1544,7 +4853,25 @@ export class SchedulerStore {
|
|
|
1544
4853
|
settled_at = NULL
|
|
1545
4854
|
WHERE occurrence_id = @occurrenceId
|
|
1546
4855
|
AND fence = @expectedFence
|
|
1547
|
-
AND
|
|
4856
|
+
AND (
|
|
4857
|
+
attempt < max_attempts
|
|
4858
|
+
OR EXISTS (
|
|
4859
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4860
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4861
|
+
AND c.state IN ('pause_requested', 'resumed')
|
|
4862
|
+
)
|
|
4863
|
+
OR EXISTS (
|
|
4864
|
+
SELECT 1 FROM scheduler_occurrence_retries r
|
|
4865
|
+
WHERE r.occurrence_id = occurrences.occurrence_id
|
|
4866
|
+
AND r.claimed_at IS NULL
|
|
4867
|
+
AND r.expected_fence = occurrences.fence
|
|
4868
|
+
)
|
|
4869
|
+
)
|
|
4870
|
+
AND NOT EXISTS (
|
|
4871
|
+
SELECT 1 FROM scheduler_occurrence_controls c
|
|
4872
|
+
WHERE c.occurrence_id = occurrences.occurrence_id
|
|
4873
|
+
AND c.state = 'paused'
|
|
4874
|
+
)
|
|
1548
4875
|
AND (
|
|
1549
4876
|
(status IN ('queued', 'retry_wait') AND available_at <= @now)
|
|
1550
4877
|
OR
|
|
@@ -1566,6 +4893,39 @@ export class SchedulerStore {
|
|
|
1566
4893
|
);
|
|
1567
4894
|
}
|
|
1568
4895
|
const claimed = this.statements.getOccurrence.get(id);
|
|
4896
|
+
if (controlRow?.state === "pause_requested") {
|
|
4897
|
+
this.db
|
|
4898
|
+
.prepare(
|
|
4899
|
+
`UPDATE scheduler_occurrence_controls
|
|
4900
|
+
SET expected_fence = ?, revision = revision + 1, updated_at = ?
|
|
4901
|
+
WHERE occurrence_id = ? AND state = 'pause_requested'
|
|
4902
|
+
AND expected_fence = ?`,
|
|
4903
|
+
)
|
|
4904
|
+
.run(claimed.fence, now, id, controlRow.expected_fence);
|
|
4905
|
+
} else if (controlRow?.state === "resumed") {
|
|
4906
|
+
const rebound = this.db
|
|
4907
|
+
.prepare(
|
|
4908
|
+
`UPDATE scheduler_occurrence_controls
|
|
4909
|
+
SET expected_fence = ?, revision = revision + 1, updated_at = ?
|
|
4910
|
+
WHERE occurrence_id = ? AND state = 'resumed'
|
|
4911
|
+
AND expected_fence = ?`,
|
|
4912
|
+
)
|
|
4913
|
+
.run(claimed.fence, now, id, controlRow.expected_fence);
|
|
4914
|
+
if (rebound.changes !== 1) {
|
|
4915
|
+
throw new SchedulerKernelError(
|
|
4916
|
+
"SCHEDULER_OCCURRENCE_CONTROL_CONFLICT",
|
|
4917
|
+
`Scheduler resume checkpoint lost its claim fence: ${id}`,
|
|
4918
|
+
);
|
|
4919
|
+
}
|
|
4920
|
+
}
|
|
4921
|
+
if (retryRow) {
|
|
4922
|
+
this.db
|
|
4923
|
+
.prepare(
|
|
4924
|
+
`UPDATE scheduler_occurrence_retries SET claimed_at = ?
|
|
4925
|
+
WHERE occurrence_id = ? AND request_id = ? AND claimed_at IS NULL`,
|
|
4926
|
+
)
|
|
4927
|
+
.run(now, id, retryRow.request_id);
|
|
4928
|
+
}
|
|
1569
4929
|
this._appendEvent({
|
|
1570
4930
|
jobId: candidate.job_id,
|
|
1571
4931
|
occurrenceId: id,
|
|
@@ -1581,6 +4941,13 @@ export class SchedulerStore {
|
|
|
1581
4941
|
previousOwner,
|
|
1582
4942
|
previousStatus,
|
|
1583
4943
|
targeted: true,
|
|
4944
|
+
...(controlRow?.state === "pause_requested"
|
|
4945
|
+
? { pauseRequestRebound: true }
|
|
4946
|
+
: {}),
|
|
4947
|
+
...(controlRow?.state === "resumed"
|
|
4948
|
+
? { resumedWithoutAttempt: true }
|
|
4949
|
+
: {}),
|
|
4950
|
+
...(retryRow ? { retryRequestId: retryRow.request_id } : {}),
|
|
1584
4951
|
},
|
|
1585
4952
|
});
|
|
1586
4953
|
return mapOccurrence(claimed);
|
|
@@ -1592,12 +4959,12 @@ export class SchedulerStore {
|
|
|
1592
4959
|
const owner = normalizeIdentifier(ownerId, "ownerId");
|
|
1593
4960
|
const token = assertPositiveSafeInteger(fence, "fence");
|
|
1594
4961
|
const lease = normalizeLeaseMs(leaseMs);
|
|
1595
|
-
const now = this._now();
|
|
1596
|
-
const leaseExpiresAt = now + lease;
|
|
1597
|
-
if (!Number.isSafeInteger(leaseExpiresAt)) {
|
|
1598
|
-
throw invalidArgument("lease expiry exceeds the safe integer range");
|
|
1599
|
-
}
|
|
1600
4962
|
return this._write(() => {
|
|
4963
|
+
const now = this._now();
|
|
4964
|
+
const leaseExpiresAt = now + lease;
|
|
4965
|
+
if (!Number.isSafeInteger(leaseExpiresAt)) {
|
|
4966
|
+
throw invalidArgument("lease expiry exceeds the safe integer range");
|
|
4967
|
+
}
|
|
1601
4968
|
const result = this.db
|
|
1602
4969
|
.prepare(
|
|
1603
4970
|
`
|
|
@@ -1642,6 +5009,7 @@ export class SchedulerStore {
|
|
|
1642
5009
|
error,
|
|
1643
5010
|
retryable = true,
|
|
1644
5011
|
retryAt,
|
|
5012
|
+
adjudicationRequestId,
|
|
1645
5013
|
} = {}) {
|
|
1646
5014
|
const id = normalizeIdentifier(occurrenceId, "occurrenceId");
|
|
1647
5015
|
const owner = normalizeIdentifier(ownerId, "ownerId");
|
|
@@ -1664,13 +5032,21 @@ export class SchedulerStore {
|
|
|
1664
5032
|
"error",
|
|
1665
5033
|
)
|
|
1666
5034
|
: null;
|
|
1667
|
-
const
|
|
1668
|
-
const availableAt =
|
|
5035
|
+
const normalizedRetryAt =
|
|
1669
5036
|
outcome !== "failed" || retryAt === undefined
|
|
1670
|
-
?
|
|
1671
|
-
:
|
|
5037
|
+
? null
|
|
5038
|
+
: normalizeEpochMs(retryAt, "retryAt");
|
|
5039
|
+
const adjudicationRequest =
|
|
5040
|
+
adjudicationRequestId === undefined
|
|
5041
|
+
? null
|
|
5042
|
+
: normalizeIdentifier(adjudicationRequestId, "adjudicationRequestId", {
|
|
5043
|
+
maxLength: 128,
|
|
5044
|
+
});
|
|
1672
5045
|
|
|
1673
5046
|
return this._write(() => {
|
|
5047
|
+
const now = this._now();
|
|
5048
|
+
const availableAt =
|
|
5049
|
+
normalizedRetryAt === null ? now : Math.max(now, normalizedRetryAt);
|
|
1674
5050
|
const current = this.statements.getOccurrence.get(id);
|
|
1675
5051
|
if (
|
|
1676
5052
|
!current ||
|
|
@@ -1736,6 +5112,35 @@ export class SchedulerStore {
|
|
|
1736
5112
|
settledAt,
|
|
1737
5113
|
});
|
|
1738
5114
|
if (update.changes !== 1) throw leaseLost(id);
|
|
5115
|
+
if (nextStatus === OCCURRENCE_STATUS.RETRY_WAIT) {
|
|
5116
|
+
this.db
|
|
5117
|
+
.prepare(
|
|
5118
|
+
`UPDATE scheduler_occurrence_controls
|
|
5119
|
+
SET expected_fence = @nextFence, revision = revision + 1,
|
|
5120
|
+
updated_at = @now
|
|
5121
|
+
WHERE occurrence_id = @occurrenceId
|
|
5122
|
+
AND state = 'resumed'
|
|
5123
|
+
AND expected_fence = @fence`,
|
|
5124
|
+
)
|
|
5125
|
+
.run({
|
|
5126
|
+
occurrenceId: id,
|
|
5127
|
+
fence: token,
|
|
5128
|
+
nextFence: token + 1,
|
|
5129
|
+
now,
|
|
5130
|
+
});
|
|
5131
|
+
}
|
|
5132
|
+
const controlTerminalized = this.db
|
|
5133
|
+
.prepare(
|
|
5134
|
+
`UPDATE scheduler_occurrence_controls
|
|
5135
|
+
SET state = 'terminal', revision = revision + 1,
|
|
5136
|
+
terminal_at = @now, updated_at = @now
|
|
5137
|
+
WHERE occurrence_id = @occurrenceId
|
|
5138
|
+
AND (
|
|
5139
|
+
state = 'pause_requested'
|
|
5140
|
+
OR (state = 'resumed' AND @status <> 'retry_wait')
|
|
5141
|
+
)`,
|
|
5142
|
+
)
|
|
5143
|
+
.run({ occurrenceId: id, status: nextStatus, now }).changes;
|
|
1739
5144
|
const reservation = this.db
|
|
1740
5145
|
.prepare(
|
|
1741
5146
|
`SELECT * FROM scheduler_authority_reservations
|
|
@@ -1772,6 +5177,51 @@ export class SchedulerStore {
|
|
|
1772
5177
|
);
|
|
1773
5178
|
}
|
|
1774
5179
|
}
|
|
5180
|
+
const adjudication = this.db
|
|
5181
|
+
.prepare(
|
|
5182
|
+
`SELECT * FROM scheduler_occurrence_adjudications
|
|
5183
|
+
WHERE occurrence_id = ?`,
|
|
5184
|
+
)
|
|
5185
|
+
.get(id);
|
|
5186
|
+
if (
|
|
5187
|
+
adjudicationRequest !== null &&
|
|
5188
|
+
adjudication?.status === "pending" &&
|
|
5189
|
+
adjudication.request_id !== adjudicationRequest
|
|
5190
|
+
) {
|
|
5191
|
+
throw new SchedulerKernelError(
|
|
5192
|
+
"SCHEDULER_ADJUDICATION_SETTLEMENT_CONFLICT",
|
|
5193
|
+
`Scheduler adjudication request does not match settlement: ${id}`,
|
|
5194
|
+
);
|
|
5195
|
+
}
|
|
5196
|
+
if (
|
|
5197
|
+
adjudication?.status === "pending" &&
|
|
5198
|
+
(nextStatus === OCCURRENCE_STATUS.SUCCEEDED ||
|
|
5199
|
+
nextStatus === OCCURRENCE_STATUS.DEAD_LETTER)
|
|
5200
|
+
) {
|
|
5201
|
+
const retryOutcome =
|
|
5202
|
+
nextStatus === OCCURRENCE_STATUS.SUCCEEDED
|
|
5203
|
+
? { status: nextStatus, result: normalizedResult }
|
|
5204
|
+
: { status: nextStatus, error: normalizedError };
|
|
5205
|
+
const adjudicationUpdate = this.db
|
|
5206
|
+
.prepare(
|
|
5207
|
+
`UPDATE scheduler_occurrence_adjudications
|
|
5208
|
+
SET status = 'applied', applied_at = ?, retry_settled_at = ?,
|
|
5209
|
+
retry_outcome_json = ?
|
|
5210
|
+
WHERE occurrence_id = ? AND status = 'pending'`,
|
|
5211
|
+
)
|
|
5212
|
+
.run(
|
|
5213
|
+
now,
|
|
5214
|
+
now,
|
|
5215
|
+
canonicalJson(retryOutcome, "adjudication.retryOutcome"),
|
|
5216
|
+
id,
|
|
5217
|
+
);
|
|
5218
|
+
if (adjudicationUpdate.changes !== 1) {
|
|
5219
|
+
throw new SchedulerKernelError(
|
|
5220
|
+
"SCHEDULER_ADJUDICATION_SETTLEMENT_CONFLICT",
|
|
5221
|
+
`Scheduler adjudication could not be settled: ${id}`,
|
|
5222
|
+
);
|
|
5223
|
+
}
|
|
5224
|
+
}
|
|
1775
5225
|
this._appendEvent({
|
|
1776
5226
|
jobId: current.job_id,
|
|
1777
5227
|
occurrenceId: id,
|
|
@@ -1791,6 +5241,12 @@ export class SchedulerStore {
|
|
|
1791
5241
|
...(nextStatus === OCCURRENCE_STATUS.RETRY_WAIT
|
|
1792
5242
|
? { retryAt: nextAvailableAt }
|
|
1793
5243
|
: {}),
|
|
5244
|
+
...(adjudication?.status === "pending"
|
|
5245
|
+
? { adjudicationRequestId: adjudication.request_id }
|
|
5246
|
+
: {}),
|
|
5247
|
+
...(controlTerminalized === 1
|
|
5248
|
+
? { runtimeControlTerminalized: true }
|
|
5249
|
+
: {}),
|
|
1794
5250
|
},
|
|
1795
5251
|
});
|
|
1796
5252
|
return mapOccurrence(this.statements.getOccurrence.get(id));
|
|
@@ -1843,6 +5299,259 @@ export class SchedulerStore {
|
|
|
1843
5299
|
.map(mapOccurrence);
|
|
1844
5300
|
}
|
|
1845
5301
|
|
|
5302
|
+
_adjudicationCase(occurrenceId) {
|
|
5303
|
+
const row = this.statements.getOccurrence.get(occurrenceId);
|
|
5304
|
+
if (!row) return null;
|
|
5305
|
+
const occurrence = mapOccurrence(row);
|
|
5306
|
+
const reservation = mapAuthorityReservation(
|
|
5307
|
+
this.db
|
|
5308
|
+
.prepare(
|
|
5309
|
+
`SELECT * FROM scheduler_authority_reservations
|
|
5310
|
+
WHERE occurrence_id = ?`,
|
|
5311
|
+
)
|
|
5312
|
+
.get(occurrenceId),
|
|
5313
|
+
);
|
|
5314
|
+
const adjudication = mapAdjudication(
|
|
5315
|
+
this.db
|
|
5316
|
+
.prepare(
|
|
5317
|
+
`SELECT * FROM scheduler_occurrence_adjudications
|
|
5318
|
+
WHERE occurrence_id = ?`,
|
|
5319
|
+
)
|
|
5320
|
+
.get(occurrenceId),
|
|
5321
|
+
);
|
|
5322
|
+
const eligible =
|
|
5323
|
+
occurrence.status === OCCURRENCE_STATUS.DEAD_LETTER &&
|
|
5324
|
+
isOutcomeUnknownError(occurrence.lastError) &&
|
|
5325
|
+
adjudication === null;
|
|
5326
|
+
const evidenceDigest = eligible
|
|
5327
|
+
? sha256PayloadDigest(
|
|
5328
|
+
adjudicationEvidence(occurrence, reservation),
|
|
5329
|
+
"adjudication.evidence",
|
|
5330
|
+
)
|
|
5331
|
+
: (adjudication?.evidenceDigest ?? null);
|
|
5332
|
+
return {
|
|
5333
|
+
occurrenceId: occurrence.id,
|
|
5334
|
+
jobId: occurrence.jobId,
|
|
5335
|
+
status: occurrence.status,
|
|
5336
|
+
attempt: occurrence.attempt,
|
|
5337
|
+
maxAttempts: occurrence.maxAttempts,
|
|
5338
|
+
fence: occurrence.fence,
|
|
5339
|
+
settledAt: occurrence.settledAt,
|
|
5340
|
+
errorCode:
|
|
5341
|
+
typeof occurrence.lastError?.code === "string"
|
|
5342
|
+
? occurrence.lastError.code
|
|
5343
|
+
: null,
|
|
5344
|
+
authorityPrincipal: occurrence.authority.principal,
|
|
5345
|
+
reservation:
|
|
5346
|
+
reservation === null
|
|
5347
|
+
? null
|
|
5348
|
+
: {
|
|
5349
|
+
policyRevision: reservation.policyRevision,
|
|
5350
|
+
units: reservation.units,
|
|
5351
|
+
status: reservation.status,
|
|
5352
|
+
},
|
|
5353
|
+
eligible,
|
|
5354
|
+
evidenceDigest,
|
|
5355
|
+
adjudication,
|
|
5356
|
+
};
|
|
5357
|
+
}
|
|
5358
|
+
|
|
5359
|
+
getAdjudicationCase(occurrenceId) {
|
|
5360
|
+
this._assertOpen();
|
|
5361
|
+
return this._adjudicationCase(
|
|
5362
|
+
normalizeIdentifier(occurrenceId, "occurrenceId"),
|
|
5363
|
+
);
|
|
5364
|
+
}
|
|
5365
|
+
|
|
5366
|
+
listAdjudicationCases({ limit } = {}) {
|
|
5367
|
+
this._assertOpen();
|
|
5368
|
+
const boundedLimit = normalizeHistoryLimit(limit);
|
|
5369
|
+
return this.db
|
|
5370
|
+
.prepare(
|
|
5371
|
+
`SELECT o.occurrence_id
|
|
5372
|
+
FROM occurrences o
|
|
5373
|
+
LEFT JOIN scheduler_occurrence_adjudications a
|
|
5374
|
+
ON a.occurrence_id = o.occurrence_id
|
|
5375
|
+
WHERE o.status = 'dead_letter'
|
|
5376
|
+
AND a.occurrence_id IS NULL
|
|
5377
|
+
AND json_extract(o.last_error_json, '$.code') GLOB '*_OUTCOME_UNKNOWN'
|
|
5378
|
+
ORDER BY o.settled_at DESC, o.occurrence_id
|
|
5379
|
+
LIMIT 1000`,
|
|
5380
|
+
)
|
|
5381
|
+
.all()
|
|
5382
|
+
.map((row) => this._adjudicationCase(row.occurrence_id))
|
|
5383
|
+
.filter((entry) => entry.eligible)
|
|
5384
|
+
.slice(0, boundedLimit);
|
|
5385
|
+
}
|
|
5386
|
+
|
|
5387
|
+
getOccurrenceAdjudication(occurrenceId) {
|
|
5388
|
+
this._assertOpen();
|
|
5389
|
+
const id = normalizeIdentifier(occurrenceId, "occurrenceId");
|
|
5390
|
+
return mapAdjudication(
|
|
5391
|
+
this.db
|
|
5392
|
+
.prepare(
|
|
5393
|
+
`SELECT * FROM scheduler_occurrence_adjudications
|
|
5394
|
+
WHERE occurrence_id = ?`,
|
|
5395
|
+
)
|
|
5396
|
+
.get(id),
|
|
5397
|
+
);
|
|
5398
|
+
}
|
|
5399
|
+
|
|
5400
|
+
adjudicateOccurrence({
|
|
5401
|
+
occurrenceId,
|
|
5402
|
+
decision,
|
|
5403
|
+
expectedEvidenceDigest,
|
|
5404
|
+
expectedAttempt,
|
|
5405
|
+
expectedFence,
|
|
5406
|
+
reasonDigest,
|
|
5407
|
+
operatorDigest,
|
|
5408
|
+
} = {}) {
|
|
5409
|
+
const id = normalizeIdentifier(occurrenceId, "occurrenceId");
|
|
5410
|
+
const normalizedDecision = normalizeAdjudicationDecision(decision);
|
|
5411
|
+
const evidenceDigest = normalizeSha256Digest(
|
|
5412
|
+
expectedEvidenceDigest,
|
|
5413
|
+
"expectedEvidenceDigest",
|
|
5414
|
+
);
|
|
5415
|
+
const reason = normalizeSha256Digest(reasonDigest, "reasonDigest");
|
|
5416
|
+
const operator = normalizeSha256Digest(operatorDigest, "operatorDigest");
|
|
5417
|
+
const attempt = assertPositiveSafeInteger(
|
|
5418
|
+
expectedAttempt,
|
|
5419
|
+
"expectedAttempt",
|
|
5420
|
+
);
|
|
5421
|
+
const fence = assertPositiveSafeInteger(expectedFence, "expectedFence");
|
|
5422
|
+
const now = this._now();
|
|
5423
|
+
return this._write(() => {
|
|
5424
|
+
const adjudicationCase = this._adjudicationCase(id);
|
|
5425
|
+
if (!adjudicationCase) {
|
|
5426
|
+
throw schedulerAdjudicationError(
|
|
5427
|
+
"SCHEDULER_NOT_FOUND",
|
|
5428
|
+
`Scheduler occurrence does not exist: ${id}`,
|
|
5429
|
+
);
|
|
5430
|
+
}
|
|
5431
|
+
if (adjudicationCase.adjudication) {
|
|
5432
|
+
throw schedulerAdjudicationError(
|
|
5433
|
+
"SCHEDULER_ADJUDICATION_ALREADY_RECORDED",
|
|
5434
|
+
`Scheduler occurrence already has a monotonic adjudication: ${id}`,
|
|
5435
|
+
{ requestId: adjudicationCase.adjudication.requestId },
|
|
5436
|
+
);
|
|
5437
|
+
}
|
|
5438
|
+
if (!adjudicationCase.eligible) {
|
|
5439
|
+
throw schedulerAdjudicationError(
|
|
5440
|
+
"SCHEDULER_ADJUDICATION_NOT_ELIGIBLE",
|
|
5441
|
+
`Scheduler occurrence is not an outcome-unknown dead letter: ${id}`,
|
|
5442
|
+
{
|
|
5443
|
+
status: adjudicationCase.status,
|
|
5444
|
+
errorCode: adjudicationCase.errorCode,
|
|
5445
|
+
},
|
|
5446
|
+
);
|
|
5447
|
+
}
|
|
5448
|
+
if (
|
|
5449
|
+
adjudicationCase.evidenceDigest !== evidenceDigest ||
|
|
5450
|
+
adjudicationCase.attempt !== attempt ||
|
|
5451
|
+
adjudicationCase.fence !== fence
|
|
5452
|
+
) {
|
|
5453
|
+
throw schedulerAdjudicationError(
|
|
5454
|
+
"SCHEDULER_ADJUDICATION_EVIDENCE_CONFLICT",
|
|
5455
|
+
`Scheduler adjudication evidence changed: ${id}`,
|
|
5456
|
+
{
|
|
5457
|
+
actualEvidenceDigest: adjudicationCase.evidenceDigest,
|
|
5458
|
+
actualAttempt: adjudicationCase.attempt,
|
|
5459
|
+
actualFence: adjudicationCase.fence,
|
|
5460
|
+
},
|
|
5461
|
+
);
|
|
5462
|
+
}
|
|
5463
|
+
const grantedMaxAttempts =
|
|
5464
|
+
normalizedDecision ===
|
|
5465
|
+
SCHEDULER_ADJUDICATION_DECISIONS.CONFIRMED_APPLIED
|
|
5466
|
+
? attempt + 2
|
|
5467
|
+
: attempt + 1;
|
|
5468
|
+
if (grantedMaxAttempts > 32) {
|
|
5469
|
+
throw schedulerAdjudicationError(
|
|
5470
|
+
"SCHEDULER_ADJUDICATION_ATTEMPT_LIMIT",
|
|
5471
|
+
`Scheduler occurrence cannot receive a bounded adjudication claim: ${id}`,
|
|
5472
|
+
);
|
|
5473
|
+
}
|
|
5474
|
+
const requestId = deterministicAdjudicationRequestId({
|
|
5475
|
+
occurrenceId: id,
|
|
5476
|
+
decision: normalizedDecision,
|
|
5477
|
+
evidenceDigest,
|
|
5478
|
+
reasonDigest: reason,
|
|
5479
|
+
});
|
|
5480
|
+
this.db
|
|
5481
|
+
.prepare(
|
|
5482
|
+
`INSERT INTO scheduler_occurrence_adjudications
|
|
5483
|
+
(occurrence_id, job_id, request_id, decision, authority,
|
|
5484
|
+
evidence_digest, expected_attempt, expected_fence,
|
|
5485
|
+
reason_digest, operator_digest, status, created_at, applied_at,
|
|
5486
|
+
retry_settled_at, retry_outcome_json)
|
|
5487
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, NULL, NULL, NULL)`,
|
|
5488
|
+
)
|
|
5489
|
+
.run(
|
|
5490
|
+
id,
|
|
5491
|
+
adjudicationCase.jobId,
|
|
5492
|
+
requestId,
|
|
5493
|
+
normalizedDecision,
|
|
5494
|
+
SCHEDULER_ADJUDICATION_AUTHORITY,
|
|
5495
|
+
evidenceDigest,
|
|
5496
|
+
attempt,
|
|
5497
|
+
fence,
|
|
5498
|
+
reason,
|
|
5499
|
+
operator,
|
|
5500
|
+
now,
|
|
5501
|
+
);
|
|
5502
|
+
const update = this.db
|
|
5503
|
+
.prepare(
|
|
5504
|
+
`UPDATE occurrences
|
|
5505
|
+
SET status = 'retry_wait',
|
|
5506
|
+
available_at = ?,
|
|
5507
|
+
max_attempts = ?,
|
|
5508
|
+
lease_owner = NULL,
|
|
5509
|
+
lease_expires_at = NULL,
|
|
5510
|
+
result_json = NULL,
|
|
5511
|
+
updated_at = ?,
|
|
5512
|
+
settled_at = NULL
|
|
5513
|
+
WHERE occurrence_id = ?
|
|
5514
|
+
AND status = 'dead_letter'
|
|
5515
|
+
AND attempt = ?
|
|
5516
|
+
AND fence = ?`,
|
|
5517
|
+
)
|
|
5518
|
+
.run(now, grantedMaxAttempts, now, id, attempt, fence);
|
|
5519
|
+
if (update.changes !== 1) {
|
|
5520
|
+
throw schedulerAdjudicationError(
|
|
5521
|
+
"SCHEDULER_ADJUDICATION_EVIDENCE_CONFLICT",
|
|
5522
|
+
`Scheduler occurrence changed during adjudication: ${id}`,
|
|
5523
|
+
);
|
|
5524
|
+
}
|
|
5525
|
+
this.db
|
|
5526
|
+
.prepare(
|
|
5527
|
+
`UPDATE scheduler_authority_reservations
|
|
5528
|
+
SET status = 'reserved', outcome_json = NULL, settled_at = NULL
|
|
5529
|
+
WHERE occurrence_id = ? AND status = 'failed'`,
|
|
5530
|
+
)
|
|
5531
|
+
.run(id);
|
|
5532
|
+
this._appendEvent({
|
|
5533
|
+
jobId: adjudicationCase.jobId,
|
|
5534
|
+
occurrenceId: id,
|
|
5535
|
+
type: "occurrence_adjudication_recorded",
|
|
5536
|
+
occurredAt: now,
|
|
5537
|
+
fence,
|
|
5538
|
+
data: {
|
|
5539
|
+
requestId,
|
|
5540
|
+
decision: normalizedDecision,
|
|
5541
|
+
authority: SCHEDULER_ADJUDICATION_AUTHORITY,
|
|
5542
|
+
evidenceDigest,
|
|
5543
|
+
expectedAttempt: attempt,
|
|
5544
|
+
expectedFence: fence,
|
|
5545
|
+
reasonDigest: reason,
|
|
5546
|
+
operatorDigest: operator,
|
|
5547
|
+
retryAttempt: attempt + 1,
|
|
5548
|
+
maxAttempts: grantedMaxAttempts,
|
|
5549
|
+
},
|
|
5550
|
+
});
|
|
5551
|
+
return this._adjudicationCase(id);
|
|
5552
|
+
});
|
|
5553
|
+
}
|
|
5554
|
+
|
|
1846
5555
|
setAuthorityPolicy(
|
|
1847
5556
|
principal,
|
|
1848
5557
|
{
|