deepline 0.3.460 → 0.3.462
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client-core.ts +7593 -0
- package/dist/bundling-sources/sdk/src/client.ts +8 -7366
- package/dist/bundling-sources/sdk/src/http-core.ts +1239 -0
- package/dist/bundling-sources/sdk/src/http.ts +6 -1231
- package/dist/bundling-sources/sdk/src/index.ts +4 -1
- package/dist/bundling-sources/sdk/src/play.ts +5 -5
- package/dist/bundling-sources/sdk/src/plays/client-core.ts +437 -0
- package/dist/bundling-sources/sdk/src/plays/client.ts +4 -421
- package/dist/bundling-sources/sdk/src/plays/input-example.ts +3 -4
- package/dist/bundling-sources/sdk/src/plays/play-description.ts +1 -1
- package/dist/bundling-sources/sdk/src/plays/run-request.ts +1 -1
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/server.ts +43 -0
- package/dist/bundling-sources/sdk/src/types.ts +15 -5
- package/dist/bundling-sources/shared_libs/integration-contracts/billing-entitlements.ts +15 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/state.ts +28 -0
- package/dist/bundling-sources/shared_libs/monitors/callback-security.ts +0 -307
- package/dist/bundling-sources/shared_libs/monitors/http-error-response.ts +3 -0
- package/dist/bundling-sources/shared_libs/monitors/lifecycle-notifications.ts +3 -2
- package/dist/bundling-sources/shared_libs/monitors/package.json +2 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-callback-verification.ts +34 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +59 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-upstream-identity.ts +94 -1
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts +35 -1
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-failure-codes.ts +24 -0
- package/dist/bundling-sources/shared_libs/observability/dlq.ts +32 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +57 -3
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-data-plane/sheet-column-storage.ts +36 -7
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +250 -11
- package/dist/bundling-sources/shared_libs/play-runtime/array-input-store.ts +319 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +991 -359
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +110 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +155 -6
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +110 -6
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +127 -13
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +14 -1
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +57 -1
- package/dist/bundling-sources/shared_libs/play-runtime/free-company-lookup-batching.ts +218 -0
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +42 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-budget-state-backend.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +92 -31
- package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +69 -14
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +93 -23
- package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +15 -4
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +22 -18
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +190 -14
- package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +131 -0
- package/dist/bundling-sources/shared_libs/play-runtime/modal-runtime-config.ts +77 -2
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +271 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +51 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +27 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-text-for-log.ts +63 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +112 -16
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +104 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +265 -151
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-gateway-transport.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +35 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-sheet-persistence-retry.ts +12 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-control-transport.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-labels.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +136 -70
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +237 -36
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +148 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +285 -32
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +251 -56
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +29 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/sandbox-create-outcome.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +22 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +1397 -382
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-array-input-receipts.ts +920 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +38 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts +26 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +14 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +31 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts +20 -7
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/index.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +1496 -320
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +31 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1389 -129
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +162 -23
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +499 -115
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +377 -9
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts +117 -8
- package/dist/bundling-sources/shared_libs/play-runtime/secret-resolution-retry-policy.ts +25 -0
- package/dist/bundling-sources/shared_libs/play-runtime/terminal-runtime-performance.ts +391 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/mapped-calls.ts +171 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +16 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +26 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +16 -2
- package/dist/bundling-sources/shared_libs/play-runtime/transport-error-diagnostics.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +55 -5
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +41 -1
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +4 -0
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +9 -3
- package/dist/bundling-sources/shared_libs/plays/deploy/manifest-contract.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/deploy/manifest.ts +3 -2
- package/dist/bundling-sources/shared_libs/plays/deploy/project-serializer.ts +2 -2
- package/dist/bundling-sources/shared_libs/plays/run-output-summary.ts +66 -0
- package/dist/bundling-sources/shared_libs/plays/script-entry.ts +1 -1
- package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +43 -1
- package/dist/bundling-sources/shared_libs/product-notifications/content.ts +9 -0
- package/dist/bundling-sources/shared_libs/product-notifications/email.ts +91 -13
- package/dist/bundling-sources/shared_libs/product-notifications/events.ts +3 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-delivery.ts +41 -24
- package/dist/bundling-sources/shared_libs/security/attested-client-ip.ts +91 -0
- package/dist/bundling-sources/shared_libs/security/internal-actor.ts +0 -11
- package/dist/cli/{auth-HKA6ZVHQ.mjs → auth-JDLEY7VU.mjs} +5 -4
- package/dist/cli/auth-UXKY6TCC.js +41 -0
- package/dist/cli/{billing-MTO2ZZ3G.js → billing-HXAXB3XZ.js} +152 -78
- package/dist/cli/{billing-26G44S5N.mjs → billing-SDHLBLMV.mjs} +84 -10
- package/dist/cli/{chunk-BHGII2D6.js → chunk-3ACMRZRV.js} +8 -6
- package/dist/cli/{chunk-S4Q2XCGM.mjs → chunk-3KZCFOZ6.mjs} +1 -1
- package/dist/cli/chunk-4KQ6SIBC.mjs +1038 -0
- package/dist/cli/{chunk-Z2GUFYT6.js → chunk-5LEAXRHV.js} +11 -11
- package/dist/cli/{chunk-XAYO3Q2X.mjs → chunk-6NNVJDVR.mjs} +7 -5
- package/dist/cli/{chunk-NMJ4D2WO.js → chunk-6PMCRYDA.js} +216 -91
- package/dist/cli/{chunk-OMM54B3X.mjs → chunk-6RNQGWFD.mjs} +16 -806
- package/dist/cli/{chunk-EBU7LQ64.js → chunk-6V7KVRR5.js} +3 -3
- package/dist/cli/{chunk-RDPTDDKX.js → chunk-7LFIMWMW.js} +2 -2
- package/dist/cli/chunk-7MJ6VI2T.mjs +19 -0
- package/dist/cli/chunk-7SWNPNML.js +1039 -0
- package/dist/cli/{chunk-CTRPCARW.mjs → chunk-7U3XNPOA.mjs} +267 -38
- package/dist/cli/{chunk-RBTTUYZZ.js → chunk-7VZHXVOQ.js} +1 -1
- package/dist/cli/{chunk-YVBWO647.mjs → chunk-AT636LLU.mjs} +2 -2
- package/dist/cli/{chunk-VTODQWQA.js → chunk-BGTPIMSM.js} +392 -261
- package/dist/cli/{chunk-KADIT5NV.mjs → chunk-CKSOY47X.mjs} +2 -2
- package/dist/cli/{chunk-7DH2FD7Y.mjs → chunk-D464JW3N.mjs} +5 -85
- package/dist/cli/chunk-D6DXOUTO.mjs +827 -0
- package/dist/cli/{describe-QW54QQNB.js → chunk-EM4TXCCI.js} +27 -44
- package/dist/cli/chunk-F6QV42MC.mjs +91 -0
- package/dist/cli/chunk-FKVYR3QA.mjs +1039 -0
- package/dist/cli/{chunk-LMAULZLF.js → chunk-GEGW5HPZ.js} +29 -27
- package/dist/cli/{chunk-M52JQSLK.js → chunk-HUB3CCM2.js} +13 -4
- package/dist/cli/{chunk-YHTQM5XI.js → chunk-I6VHOU5O.js} +27 -27
- package/dist/cli/{chunk-Z2XUZC34.js → chunk-IAYSWZDJ.js} +188 -76
- package/dist/cli/chunk-JF3TLIMQ.js +19 -0
- package/dist/cli/{chunk-HDFRJ4IT.mjs → chunk-JVHOS3K3.mjs} +177 -65
- package/dist/cli/{chunk-LAWEURCT.mjs → chunk-LFXLQEZN.mjs} +1 -1
- package/dist/cli/{chunk-2PBN4SZE.js → chunk-LROWFPGN.js} +16 -16
- package/dist/cli/{chunk-SLEUWLJE.mjs → chunk-MSY22NQW.mjs} +5 -5
- package/dist/cli/{chunk-LVJOFUU7.mjs → chunk-MUUQZIOF.mjs} +23 -49
- package/dist/cli/{chunk-2XHWOQF4.js → chunk-NBLSCHKK.js} +52 -50
- package/dist/cli/{chunk-526R2HON.mjs → chunk-NHRWZPWZ.mjs} +4 -2
- package/dist/cli/{chunk-KXMNM4AQ.js → chunk-NMU3VR5U.js} +4 -5
- package/dist/cli/{chunk-W3BE3CHK.mjs → chunk-O2RX5A66.mjs} +14 -12
- package/dist/cli/{chunk-UKFZT7RF.js → chunk-OMQ23IXL.js} +2 -2
- package/dist/cli/{chunk-LJTFZOHU.js → chunk-OPJAJBD4.js} +28 -54
- package/dist/cli/{chunk-MLFR6DVF.mjs → chunk-PHSEINNH.mjs} +5 -5
- package/dist/cli/{chunk-3DNKFH2G.mjs → chunk-PKHVSIJG.mjs} +19 -1
- package/dist/cli/{chunk-WIYWSYDS.mjs → chunk-Q2EIZKVX.mjs} +3 -3
- package/dist/cli/{chunk-VJAEA4CS.js → chunk-QJZ6WNWD.js} +8 -8
- package/dist/cli/{chunk-EIMG75BT.mjs → chunk-S3A4IXBQ.mjs} +4 -5
- package/dist/cli/{chunk-RZSTT45F.mjs → chunk-S42YIQNE.mjs} +13 -8
- package/dist/cli/{chunk-YULBYKAK.mjs → chunk-S6UAJR6V.mjs} +3 -3
- package/dist/cli/{chunk-G7L27KZB.mjs → chunk-TKAQR3IB.mjs} +31 -21
- package/dist/cli/{chunk-QWRHGGUH.js → chunk-UAJFWWZE.js} +20 -2
- package/dist/cli/{chunk-ZR2RLI4O.mjs → chunk-UDRPEZ5F.mjs} +223 -92
- package/dist/cli/{chunk-QGAWTBIR.js → chunk-ULK7JGGI.js} +511 -282
- package/dist/cli/chunk-V335HGX5.js +1038 -0
- package/dist/cli/{chunk-NH6JSJW6.mjs → chunk-V3MEA7NB.mjs} +7 -5
- package/dist/cli/{chunk-NRGGFVVB.js → chunk-VFGVXXZP.js} +37 -35
- package/dist/cli/chunk-VLEVVDWG.js +91 -0
- package/dist/cli/{chunk-R2GLGLDR.mjs → chunk-VPHN2HAL.mjs} +216 -91
- package/dist/cli/{chunk-CFUTOWE4.js → chunk-WG3KGAII.js} +8 -88
- package/dist/cli/{describe-TOMIAT37.mjs → chunk-WIY6U6GD.mjs} +19 -36
- package/dist/cli/{chunk-YS2YBSJB.js → chunk-XANRM3QA.js} +6 -6
- package/dist/cli/{chunk-KGQNIA37.js → chunk-XQCTG5OO.js} +44 -34
- package/dist/cli/{chunk-EJGBLXDZ.mjs → chunk-XZAFIOUQ.mjs} +1 -1
- package/dist/cli/chunk-Y4DPU4OM.js +827 -0
- package/dist/cli/chunk-YT5EODJO.js +920 -0
- package/dist/cli/{chunk-2WRO2UJ5.js → chunk-ZGQSFYD2.js} +56 -51
- package/dist/cli/{chunk-OJ6RUKVT.mjs → chunk-ZUK6LIJB.mjs} +10 -1
- package/dist/cli/command-docs.d.mts +5 -0
- package/dist/cli/command-docs.d.ts +5 -0
- package/dist/cli/command-docs.js +2 -2
- package/dist/cli/command-docs.mjs +1 -1
- package/dist/cli/{csv-6FNFYJUC.js → csv-C56GSCEU.js} +13 -12
- package/dist/cli/{csv-MDHT4BKK.mjs → csv-L54JS7TF.mjs} +4 -3
- package/dist/cli/{db-D6AMVOEW.js → db-MOGRMZTA.js} +30 -27
- package/dist/cli/{db-74O3RFWY.mjs → db-YVBUW2DX.mjs} +10 -7
- package/dist/cli/describe-KYEULWPG.js +48 -0
- package/dist/cli/describe-NLZKKT7C.mjs +48 -0
- package/dist/cli/describe-core.d.mts +153 -5
- package/dist/cli/describe-core.d.ts +153 -5
- package/dist/cli/describe-core.js +9 -5
- package/dist/cli/describe-core.mjs +10 -6
- package/dist/cli/{executions-N4FMGVSC.js → executions-A7L4ULPP.js} +9 -7
- package/dist/cli/{executions-PVSF4HUF.mjs → executions-ZBGSTT37.mjs} +7 -5
- package/dist/cli/{feedback-OB7NANZW.mjs → feedback-G6GU37FD.mjs} +6 -5
- package/dist/cli/{feedback-WFSPCAI2.js → feedback-OHLY7RYD.js} +10 -9
- package/dist/cli/getting-started.d.mts +6 -6
- package/dist/cli/getting-started.d.ts +6 -6
- package/dist/cli/getting-started.js +2 -2
- package/dist/cli/getting-started.mjs +1 -1
- package/dist/cli/git-P76N4NDQ.js +25 -0
- package/dist/cli/{git-FHUSPAEU.mjs → git-PKZW3QD5.mjs} +5 -4
- package/dist/cli/index.js +152 -142
- package/dist/cli/index.mjs +56 -46
- package/dist/cli/{legacy-noops-ECHNA5HN.js → legacy-noops-5LM3D724.js} +8 -7
- package/dist/cli/{legacy-noops-HMK4BSCI.mjs → legacy-noops-LJJYG6YA.mjs} +5 -4
- package/dist/cli/{monitors-U7VQ3EZM.mjs → monitors-72QV4ALO.mjs} +19 -6
- package/dist/cli/{monitors-XLLFNENH.js → monitors-UX3H5DC2.js} +103 -90
- package/dist/cli/{org-6H3USO5M.mjs → org-6RRRKOCP.mjs} +9 -6
- package/dist/cli/{org-QX6ER2HN.js → org-IF7ISUVB.js} +70 -67
- package/dist/cli/pages-3SU3HKL7.js +252 -0
- package/dist/cli/pages-DXDYYKEK.mjs +252 -0
- package/dist/cli/{play-MOB2WYOR.mjs → play-AHASKRLA.mjs} +512 -1508
- package/dist/cli/{play-WNB7HBZK.js → play-EUIUVLUH.js} +1090 -2086
- package/dist/cli/{providers-CXEZ23E2.mjs → providers-GWGE3WY3.mjs} +7 -5
- package/dist/cli/{providers-7M3DISKX.js → providers-V5WU4VXJ.js} +10 -8
- package/dist/cli/{quickstart-ZFCHKHF6.mjs → quickstart-OXKQ5ZST.mjs} +5 -4
- package/dist/cli/{quickstart-CLJT4AZL.js → quickstart-QP65I5PF.js} +13 -12
- package/dist/cli/run-6MM72IGL.mjs +443 -0
- package/dist/cli/run-F3FTYS56.js +443 -0
- package/dist/cli/search-OYKOOI7A.js +32 -0
- package/dist/cli/{search-FVYWYIC3.mjs → search-ZMEBVJZB.mjs} +8 -6
- package/dist/cli/{secrets-YRLKZQS5.js → secrets-ACND5N6A.js} +16 -14
- package/dist/cli/{secrets-WAMPTQFC.mjs → secrets-XRCJFF3L.mjs} +7 -5
- package/dist/cli/{sessions-IQ332BJH.mjs → sessions-RTK3CEW6.mjs} +5 -4
- package/dist/cli/{sessions-44J6WIVU.js → sessions-ZGRJPCFF.js} +6 -5
- package/dist/cli/{settings-4IJR2I2T.js → settings-PD5KWQQZ.js} +80 -77
- package/dist/cli/{settings-22S6NIFQ.mjs → settings-XDRBPRDE.mjs} +14 -11
- package/dist/cli/{setup-OZUPKKPW.mjs → setup-3HPYAOCC.mjs} +12 -10
- package/dist/cli/{setup-3FWSRNOX.js → setup-E2Q6PXHC.js} +53 -51
- package/dist/cli/{skills-5CW3NFP3.mjs → skills-OOQFMPOV.mjs} +7 -6
- package/dist/cli/skills-XYWV52KB.js +31 -0
- package/dist/cli/tools-ASCDZJCK.js +55 -0
- package/dist/cli/tools-C5ZT3UIC.mjs +55 -0
- package/dist/cli/update-5RQSEIFM.js +33 -0
- package/dist/cli/{update-IBULOV7P.mjs → update-ZWJTADIW.mjs} +9 -8
- package/dist/cli/{workflow-MVZT7MWT.mjs → workflow-23NMPAJ7.mjs} +10 -8
- package/dist/cli/{workflow-Z7FW6GT2.js → workflow-TCXU5YJV.js} +20 -18
- package/dist/client-DS2IWYZF.d.ts +10 -0
- package/dist/client-DTwnz_AW.d.mts +10 -0
- package/dist/{client-CYqG23K1.d.ts → client-core-B_6HAIBy.d.ts} +385 -12
- package/dist/{client-BbU5_j7n.d.mts → client-core-DzBSlY1U.d.mts} +385 -12
- package/dist/{compiler-manifest-CWz_sNOQ.d.mts → compiler-manifest-Dcyd3-Zd.d.mts} +43 -1
- package/dist/{compiler-manifest-CWz_sNOQ.d.ts → compiler-manifest-Dcyd3-Zd.d.ts} +43 -1
- package/dist/errors-d-sTrZhe.d.mts +142 -0
- package/dist/errors-yQwfgrcX.d.ts +142 -0
- package/dist/{index-BF2JB_Ug.d.mts → index-C6oXchjN.d.mts} +1 -1
- package/dist/{index-4NPXDzlD.d.ts → index-P56zy_z5.d.ts} +1 -1
- package/dist/index.d.mts +11 -147
- package/dist/index.d.ts +11 -147
- package/dist/index.js +15 -11
- package/dist/index.mjs +14 -10
- package/dist/install-integrity.json +141 -104
- package/dist/plays/bundle-play-file.d.mts +3 -3
- package/dist/plays/bundle-play-file.d.ts +3 -3
- package/dist/plays/bundle-play-file.mjs +2 -1
- package/dist/plays/play-description.d.mts +2 -2
- package/dist/plays/play-description.d.ts +2 -2
- package/dist/plays/play-description.js +2 -2
- package/dist/plays/play-description.mjs +1 -1
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/dist/server.d.mts +17 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.js +76 -0
- package/dist/server.mjs +76 -0
- package/package.json +6 -1
- package/dist/cli/auth-VDOCNR7H.js +0 -40
- package/dist/cli/chunk-PQUUWFXY.js +0 -95
- package/dist/cli/chunk-QJLKYYPX.js +0 -1710
- package/dist/cli/chunk-YBWAG72P.mjs +0 -95
- package/dist/cli/git-4PMBJSH5.js +0 -24
- package/dist/cli/search-4CCQCTVD.js +0 -30
- package/dist/cli/skills-LMNM6B5F.js +0 -30
- package/dist/cli/tools-AGBBDJ7P.js +0 -45
- package/dist/cli/tools-NOPBDRGW.mjs +0 -45
- package/dist/cli/update-U3W6USVD.js +0 -32
|
@@ -1,4 +1,10 @@
|
|
|
1
|
+
import {
|
|
2
|
+
terminalRuntimePerformanceFromStats,
|
|
3
|
+
normalizeTerminalRuntimePerformance,
|
|
4
|
+
normalizeTerminalRuntimePerformanceUnavailableReason,
|
|
5
|
+
} from '../terminal-runtime-performance';
|
|
1
6
|
import { escapeLiteral } from 'pg';
|
|
7
|
+
import { assertWorkReceiptKeyMutable } from '../work-receipt-storage-contract';
|
|
2
8
|
import {
|
|
3
9
|
readRowFailureCounts,
|
|
4
10
|
type RowFailureCounts,
|
|
@@ -179,6 +185,7 @@ import {
|
|
|
179
185
|
resolveLateRuntimeSandboxComputeItems,
|
|
180
186
|
resolveRuntimeSandboxComputeItems,
|
|
181
187
|
} from '../sandbox-compute-usage';
|
|
188
|
+
import { PLAY_RUNNER_TERMINAL_GRACE_SECONDS } from '../runtime-constants';
|
|
182
189
|
import { assertNodeRunnableArtifact } from '../../plays/artifact-kind-guard';
|
|
183
190
|
import {
|
|
184
191
|
artifactKindForBackend,
|
|
@@ -375,14 +382,18 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
|
|
|
375
382
|
// Failed facts remain indexed: their live leases still fence the run.
|
|
376
383
|
// v63 adds optional rate-probe count and clock baselines. Older writers ignore them;
|
|
377
384
|
// unexpanded acquires retain their existing operation-local measurement.
|
|
385
|
+
// v64 adds the optional per-run array-input intent used by terminal retirement.
|
|
386
|
+
// Existing runs default to false and older workers ignore the column.
|
|
378
387
|
// This marker records deployment work, never a runtime schema floor.
|
|
379
|
-
export const POSTGRES_SCHEDULER_SCHEMA_VERSION =
|
|
388
|
+
export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 64;
|
|
380
389
|
const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
|
|
381
390
|
'scheduler_schema_migrations';
|
|
382
391
|
export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
|
|
383
392
|
'runtime_scheduler_runs_open_recovery_idx';
|
|
384
393
|
export const POSTGRES_SCHEDULER_SANDBOX_CUSTODY_INDEX =
|
|
385
394
|
'runtime_scheduler_sandbox_capacity_leases_custody_idx';
|
|
395
|
+
export const POSTGRES_SCHEDULER_LEGACY_CLEANUP_PENDING_INDEX =
|
|
396
|
+
'runtime_scheduler_sandbox_cleanup_legacy_pending_idx';
|
|
386
397
|
const POSTGRES_SCHEDULER_DEPENDENCY_CIRCUITS_TABLE =
|
|
387
398
|
'runtime_dependency_circuits';
|
|
388
399
|
/**
|
|
@@ -661,6 +672,10 @@ export type PostgresSchedulerRunStatusSnapshot = {
|
|
|
661
672
|
/** Included only for callers that explicitly request admission identity. */
|
|
662
673
|
definitionScope?: 'org' | 'system';
|
|
663
674
|
status: string;
|
|
675
|
+
/** Immutable log-store marker from the org-owned scheduler run row. */
|
|
676
|
+
runLogStore?: 'planet_scale';
|
|
677
|
+
/** Schema actually queried for this run; never inferred from its name. */
|
|
678
|
+
schedulerSchema?: string;
|
|
664
679
|
playName: string;
|
|
665
680
|
graphHash: string | null;
|
|
666
681
|
runtimeBackend: string | null;
|
|
@@ -683,6 +698,7 @@ export type PostgresSchedulerRunStatusSnapshot = {
|
|
|
683
698
|
type RunStatusRow = RunRow & {
|
|
684
699
|
play_name: string;
|
|
685
700
|
org_id: string | null;
|
|
701
|
+
run_log_store: string | null;
|
|
686
702
|
graph_hash: string | null;
|
|
687
703
|
runtime_profile: string | null;
|
|
688
704
|
launch_json: unknown;
|
|
@@ -1266,6 +1282,8 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
|
|
|
1266
1282
|
rateScopeRetryAfter?: boolean;
|
|
1267
1283
|
/** v48 expands compact log storage; prior readers ignore these columns. */
|
|
1268
1284
|
runtimeLogStorage?: boolean;
|
|
1285
|
+
/** v64 records whether terminal projection owns array-input retirement. */
|
|
1286
|
+
arrayInputIntent?: boolean;
|
|
1269
1287
|
}>;
|
|
1270
1288
|
|
|
1271
1289
|
export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
@@ -1905,6 +1923,17 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1905
1923
|
) = 2
|
|
1906
1924
|
)
|
|
1907
1925
|
)
|
|
1926
|
+
AND (
|
|
1927
|
+
$40::boolean = false
|
|
1928
|
+
OR EXISTS (
|
|
1929
|
+
SELECT 1 FROM information_schema.columns
|
|
1930
|
+
WHERE table_schema = $1 AND table_name = 'runs'
|
|
1931
|
+
AND column_name = 'array_input_started'
|
|
1932
|
+
AND data_type = 'boolean'
|
|
1933
|
+
AND is_nullable = 'NO'
|
|
1934
|
+
AND column_default IS NOT NULL
|
|
1935
|
+
)
|
|
1936
|
+
)
|
|
1908
1937
|
AS ready
|
|
1909
1938
|
`,
|
|
1910
1939
|
[
|
|
@@ -1947,6 +1976,7 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1947
1976
|
`${quotedSchema}.${quoteIdent('rate_scope_holds')}`,
|
|
1948
1977
|
requirements.runtimeLogStorage !== false,
|
|
1949
1978
|
requirements.gatewayReaderFence !== false,
|
|
1979
|
+
requirements.arrayInputIntent !== false,
|
|
1950
1980
|
],
|
|
1951
1981
|
);
|
|
1952
1982
|
if (currentShape.rows[0]?.ready !== true) return false;
|
|
@@ -2254,9 +2284,19 @@ export async function ensurePostgresSchedulerOnlineAdditiveExpansion(
|
|
|
2254
2284
|
)} (state, updated_at)
|
|
2255
2285
|
`);
|
|
2256
2286
|
await ensurePostgresSchedulerRunLogStorageExpansion(client, options);
|
|
2287
|
+
await ensurePostgresSchedulerArrayInputIntentExpansion(client, options);
|
|
2257
2288
|
await ensurePostgresSchedulerGatewayReaderFenceSchema(client, options);
|
|
2258
2289
|
}
|
|
2259
2290
|
|
|
2291
|
+
/** Deployment-owned v64 expansion; absent values retain legacy behavior. */
|
|
2292
|
+
async function ensurePostgresSchedulerArrayInputIntentExpansion(
|
|
2293
|
+
client: PostgresSchedulerQueryClient,
|
|
2294
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
2295
|
+
): Promise<void> {
|
|
2296
|
+
await client.query(`ALTER TABLE ${tableName('runs', options)}
|
|
2297
|
+
ADD COLUMN IF NOT EXISTS array_input_started boolean NOT NULL DEFAULT false`);
|
|
2298
|
+
}
|
|
2299
|
+
|
|
2260
2300
|
/** Deployment-only expansion; absent values keep legacy storage authoritative. */
|
|
2261
2301
|
async function ensurePostgresSchedulerRunLogStorageExpansion(
|
|
2262
2302
|
client: PostgresSchedulerQueryClient,
|
|
@@ -2556,6 +2596,8 @@ export async function ensurePostgresSchedulerLaunchAdmissionFunctions(
|
|
|
2556
2596
|
'coordinatorInternalToken',
|
|
2557
2597
|
'baseUrl',
|
|
2558
2598
|
'runtimeApiBaseUrl',
|
|
2599
|
+
'arrayInputTerminalRetirementSupported',
|
|
2600
|
+
'arrayInputTerminalProjectionOrigin',
|
|
2559
2601
|
'vercelProtectionBypassToken'
|
|
2560
2602
|
]::text[],
|
|
2561
2603
|
'{maxConcurrentExternalCalls}',
|
|
@@ -2726,6 +2768,7 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
2726
2768
|
terminalComputeBillingOutboxTrigger: false,
|
|
2727
2769
|
outboxWakeTrigger: false,
|
|
2728
2770
|
cronScheduleSchema: false,
|
|
2771
|
+
arrayInputIntent: false,
|
|
2729
2772
|
}));
|
|
2730
2773
|
const schemaCoreIsCurrent =
|
|
2731
2774
|
schemaIsCurrent || schemaIsCurrentWithoutAdditiveFastPathObjects;
|
|
@@ -2746,6 +2789,9 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
2746
2789
|
const hasLegacyAdmissionRows =
|
|
2747
2790
|
legacyAdmissionRows?.rows[0]?.present === true;
|
|
2748
2791
|
if (schemaCoreIsCurrent) {
|
|
2792
|
+
if (!schemaIsCurrent) {
|
|
2793
|
+
await ensurePostgresSchedulerArrayInputIntentExpansion(client, options);
|
|
2794
|
+
}
|
|
2749
2795
|
if (!options?.deferRuntimeAdminIndexes) {
|
|
2750
2796
|
await ensurePostgresSchedulerSandboxCustodyIndex(client, {
|
|
2751
2797
|
schema,
|
|
@@ -2800,6 +2846,9 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
2800
2846
|
finished_at timestamptz
|
|
2801
2847
|
)
|
|
2802
2848
|
`);
|
|
2849
|
+
// Reader-first expansion. A registered array-input intent makes terminal
|
|
2850
|
+
// projection responsible for retiring tenant input pages before outbox ACK.
|
|
2851
|
+
await ensurePostgresSchedulerArrayInputIntentExpansion(client, options);
|
|
2803
2852
|
await client.query(`
|
|
2804
2853
|
ALTER TABLE ${tableName('runs', options)}
|
|
2805
2854
|
ADD COLUMN IF NOT EXISTS runtime_timing_json jsonb
|
|
@@ -3734,6 +3783,8 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
3734
3783
|
'coordinatorInternalToken',
|
|
3735
3784
|
'baseUrl',
|
|
3736
3785
|
'runtimeApiBaseUrl',
|
|
3786
|
+
'arrayInputTerminalRetirementSupported',
|
|
3787
|
+
'arrayInputTerminalProjectionOrigin',
|
|
3737
3788
|
'vercelProtectionBypassToken'
|
|
3738
3789
|
]::text[],
|
|
3739
3790
|
'{maxConcurrentExternalCalls}',
|
|
@@ -4201,6 +4252,14 @@ const POSTGRES_SCHEDULER_SANDBOX_CUSTODY_INDEX_DEFINITION = {
|
|
|
4201
4252
|
definition: '(run_id, lease_id)',
|
|
4202
4253
|
} as const;
|
|
4203
4254
|
|
|
4255
|
+
// Optional deploy-owned optimization for the v12 compatibility read. The
|
|
4256
|
+
// claimant still scans correctly if this index has not been built yet.
|
|
4257
|
+
const POSTGRES_SCHEDULER_LEGACY_CLEANUP_PENDING_INDEX_DEFINITION = {
|
|
4258
|
+
name: POSTGRES_SCHEDULER_LEGACY_CLEANUP_PENDING_INDEX,
|
|
4259
|
+
table: 'sandbox_cleanup_jobs',
|
|
4260
|
+
definition: "(sandbox_id) WHERE status <> 'deleted'",
|
|
4261
|
+
} as const;
|
|
4262
|
+
|
|
4204
4263
|
const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
|
|
4205
4264
|
POSTGRES_SCHEDULER_SANDBOX_CUSTODY_INDEX_DEFINITION,
|
|
4206
4265
|
{
|
|
@@ -4490,6 +4549,19 @@ export async function ensurePostgresSchedulerSandboxCustodyIndex(
|
|
|
4490
4549
|
);
|
|
4491
4550
|
}
|
|
4492
4551
|
|
|
4552
|
+
/** Build outside live-schema transactions; absence never blocks cleanup. */
|
|
4553
|
+
export async function ensurePostgresSchedulerLegacyCleanupPendingIndex(
|
|
4554
|
+
client: PostgresSchedulerQueryClient,
|
|
4555
|
+
options: Pick<PostgresSchedulerOptions, 'schema'> & { concurrently: boolean },
|
|
4556
|
+
): Promise<void> {
|
|
4557
|
+
await ensurePostgresSchedulerIndex(
|
|
4558
|
+
client,
|
|
4559
|
+
POSTGRES_SCHEDULER_LEGACY_CLEANUP_PENDING_INDEX_DEFINITION,
|
|
4560
|
+
options,
|
|
4561
|
+
options.concurrently,
|
|
4562
|
+
);
|
|
4563
|
+
}
|
|
4564
|
+
|
|
4493
4565
|
export async function readPostgresSchedulerOpenRecoveryIndexState(
|
|
4494
4566
|
client: PostgresSchedulerQueryClient,
|
|
4495
4567
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
@@ -6844,7 +6916,6 @@ export async function repairPostgresSchedulerCronFire(
|
|
|
6844
6916
|
|
|
6845
6917
|
function runtimeReleaseAdmissionKeys(input: {
|
|
6846
6918
|
releaseId: string;
|
|
6847
|
-
admissionEpoch: number;
|
|
6848
6919
|
environment?: 'preview' | 'production';
|
|
6849
6920
|
}) {
|
|
6850
6921
|
const releaseId = input.releaseId.trim();
|
|
@@ -6864,14 +6935,53 @@ function runtimeReleaseAdmissionKeys(input: {
|
|
|
6864
6935
|
: [releaseAdmissionKey];
|
|
6865
6936
|
if (!releaseId)
|
|
6866
6937
|
throw new Error('Runtime release admission requires releaseId.');
|
|
6867
|
-
|
|
6938
|
+
return { releaseAdmissionLockKeys, releaseAdmissionWriteKeys };
|
|
6939
|
+
}
|
|
6940
|
+
|
|
6941
|
+
function runtimeReleaseAdmissionEpoch(value: unknown): number {
|
|
6942
|
+
const epoch = value;
|
|
6943
|
+
if (typeof epoch !== 'number' || !Number.isSafeInteger(epoch) || epoch < 0) {
|
|
6868
6944
|
throw new Error(
|
|
6869
6945
|
'Runtime release admission requires a non-negative safe-integer admissionEpoch.',
|
|
6870
6946
|
);
|
|
6871
6947
|
}
|
|
6872
|
-
return
|
|
6948
|
+
return epoch;
|
|
6949
|
+
}
|
|
6950
|
+
|
|
6951
|
+
/** Read the same highest-epoch, closed-wins fence used by run admission. */
|
|
6952
|
+
export async function readPostgresSchedulerRuntimeReleaseAdmission(
|
|
6953
|
+
client: PostgresSchedulerQueryClient,
|
|
6954
|
+
input: { releaseId: string; environment?: 'preview' | 'production' },
|
|
6955
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
6956
|
+
): Promise<{ accepting: boolean; admissionEpoch: number } | null> {
|
|
6957
|
+
const { releaseAdmissionWriteKeys } = runtimeReleaseAdmissionKeys(input);
|
|
6958
|
+
const result = await client.query<{
|
|
6959
|
+
accepting: boolean;
|
|
6960
|
+
admission_epoch: string;
|
|
6961
|
+
}>(
|
|
6962
|
+
`SELECT admission.accepting, admission.admission_epoch::text AS admission_epoch
|
|
6963
|
+
FROM ${tableName('runtime_release_admission', options)} AS admission
|
|
6964
|
+
WHERE admission.release_id = ANY($1::text[])
|
|
6965
|
+
ORDER BY admission.admission_epoch DESC, admission.accepting ASC
|
|
6966
|
+
LIMIT 1`,
|
|
6967
|
+
[releaseAdmissionWriteKeys],
|
|
6968
|
+
);
|
|
6969
|
+
const row = result.rows[0];
|
|
6970
|
+
return row
|
|
6971
|
+
? {
|
|
6972
|
+
accepting: row.accepting,
|
|
6973
|
+
admissionEpoch: runtimeReleaseAdmissionEpoch(
|
|
6974
|
+
Number(row.admission_epoch),
|
|
6975
|
+
),
|
|
6976
|
+
}
|
|
6977
|
+
: null;
|
|
6873
6978
|
}
|
|
6874
6979
|
|
|
6980
|
+
/**
|
|
6981
|
+
* Opening callers must own a transaction and roll it back on refusal: a
|
|
6982
|
+
* concurrent higher fence can reject one key after another was updated.
|
|
6983
|
+
* Closing remains a monotonic write-if-newer and permits a stale no-op.
|
|
6984
|
+
*/
|
|
6875
6985
|
export async function setPostgresSchedulerRuntimeReleaseAdmission(
|
|
6876
6986
|
client: PostgresSchedulerQueryClient,
|
|
6877
6987
|
input: {
|
|
@@ -6884,7 +6994,12 @@ export async function setPostgresSchedulerRuntimeReleaseAdmission(
|
|
|
6884
6994
|
): Promise<void> {
|
|
6885
6995
|
const { releaseAdmissionLockKeys, releaseAdmissionWriteKeys } =
|
|
6886
6996
|
runtimeReleaseAdmissionKeys(input);
|
|
6887
|
-
|
|
6997
|
+
runtimeReleaseAdmissionEpoch(input.admissionEpoch);
|
|
6998
|
+
const result = await client.query<{
|
|
6999
|
+
release_id: string;
|
|
7000
|
+
accepting: boolean;
|
|
7001
|
+
admission_epoch: string;
|
|
7002
|
+
}>(
|
|
6888
7003
|
`
|
|
6889
7004
|
WITH admission_lock AS MATERIALIZED (
|
|
6890
7005
|
SELECT pg_advisory_xact_lock(
|
|
@@ -6909,6 +7024,7 @@ export async function setPostgresSchedulerRuntimeReleaseAdmission(
|
|
|
6909
7024
|
admission_epoch = excluded.admission_epoch,
|
|
6910
7025
|
updated_at = now()
|
|
6911
7026
|
WHERE excluded.admission_epoch >= admission.admission_epoch
|
|
7027
|
+
RETURNING release_id, accepting, admission_epoch::text AS admission_epoch
|
|
6912
7028
|
`,
|
|
6913
7029
|
[
|
|
6914
7030
|
releaseAdmissionLockKeys,
|
|
@@ -6917,6 +7033,23 @@ export async function setPostgresSchedulerRuntimeReleaseAdmission(
|
|
|
6917
7033
|
releaseAdmissionWriteKeys,
|
|
6918
7034
|
],
|
|
6919
7035
|
);
|
|
7036
|
+
if (
|
|
7037
|
+
input.accepting &&
|
|
7038
|
+
(result.rows.length !== releaseAdmissionWriteKeys.length ||
|
|
7039
|
+
new Set(result.rows.map((row) => row.release_id)).size !==
|
|
7040
|
+
releaseAdmissionWriteKeys.length ||
|
|
7041
|
+
result.rows.some(
|
|
7042
|
+
(row) =>
|
|
7043
|
+
!releaseAdmissionWriteKeys.includes(row.release_id) ||
|
|
7044
|
+
row.accepting !== true ||
|
|
7045
|
+
runtimeReleaseAdmissionEpoch(Number(row.admission_epoch)) !==
|
|
7046
|
+
input.admissionEpoch,
|
|
7047
|
+
))
|
|
7048
|
+
) {
|
|
7049
|
+
throw new Error(
|
|
7050
|
+
`Runtime release admission opening refused for "${input.releaseId}" at epoch ${input.admissionEpoch}.`,
|
|
7051
|
+
);
|
|
7052
|
+
}
|
|
6920
7053
|
}
|
|
6921
7054
|
|
|
6922
7055
|
/**
|
|
@@ -6935,6 +7068,7 @@ export async function withPostgresSchedulerClosedReleaseAdmission<T>(
|
|
|
6935
7068
|
operation: () => Promise<T>,
|
|
6936
7069
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
6937
7070
|
): Promise<{ guarded: false } | { guarded: true; value: T }> {
|
|
7071
|
+
runtimeReleaseAdmissionEpoch(input.admissionEpoch);
|
|
6938
7072
|
const { releaseAdmissionLockKeys } = runtimeReleaseAdmissionKeys(input);
|
|
6939
7073
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
6940
7074
|
try {
|
|
@@ -10748,7 +10882,8 @@ export async function settlePostgresSchedulerRuntimeResources(
|
|
|
10748
10882
|
* retried push is idempotent and a duplicate can never rewrite the recorded
|
|
10749
10883
|
* outcome. Returns true when a terminal is durably recorded for the attempt
|
|
10750
10884
|
* (either by this call or an earlier one); false when the attempt row does not
|
|
10751
|
-
* exist
|
|
10885
|
+
* exist or a synthetic loss presents an obsolete heartbeat identity. Ordinary
|
|
10886
|
+
* ingress must reject an unknown attempt rather than invent state.
|
|
10752
10887
|
*/
|
|
10753
10888
|
export async function recordPostgresSchedulerRunnerTerminal(
|
|
10754
10889
|
client: PostgresSchedulerQueryClient,
|
|
@@ -10757,6 +10892,9 @@ export async function recordPostgresSchedulerRunnerTerminal(
|
|
|
10757
10892
|
attempt: number;
|
|
10758
10893
|
result: unknown;
|
|
10759
10894
|
terminalProgressEvents?: readonly unknown[];
|
|
10895
|
+
/** Only synthetic runner-loss outcomes use this exact liveness fence.
|
|
10896
|
+
* Undefined preserves ordinary terminal ingress; null confirms no heartbeat. */
|
|
10897
|
+
expectedHeartbeatIdentity?: string | null;
|
|
10760
10898
|
},
|
|
10761
10899
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
10762
10900
|
): Promise<boolean> {
|
|
@@ -10771,7 +10909,9 @@ export async function recordPostgresSchedulerRunnerTerminal(
|
|
|
10771
10909
|
)
|
|
10772
10910
|
UPDATE ${tableName('attempts', options)} AS target
|
|
10773
10911
|
SET runner_result_json = COALESCE(target.runner_result_json, $3::jsonb),
|
|
10774
|
-
runner_log_payload_json = CASE
|
|
10912
|
+
runner_log_payload_json = CASE
|
|
10913
|
+
WHEN target.runner_result_json IS NOT NULL THEN target.runner_log_payload_json
|
|
10914
|
+
WHEN EXISTS (
|
|
10775
10915
|
SELECT 1 FROM ${tableName('runs', options)} AS log_run
|
|
10776
10916
|
WHERE log_run.run_id = target.run_id AND log_run.run_log_store = 'planet_scale'
|
|
10777
10917
|
) THEN $4::jsonb ELSE NULL END,
|
|
@@ -10830,6 +10970,11 @@ export async function recordPostgresSchedulerRunnerTerminal(
|
|
|
10830
10970
|
FROM accepted
|
|
10831
10971
|
WHERE target.run_id = $1
|
|
10832
10972
|
AND target.attempt = $2
|
|
10973
|
+
AND (
|
|
10974
|
+
NOT $5::boolean
|
|
10975
|
+
OR target.runner_result_json IS NOT NULL
|
|
10976
|
+
OR target.heartbeat_at IS NOT DISTINCT FROM $6::timestamptz
|
|
10977
|
+
)
|
|
10833
10978
|
RETURNING target.run_id
|
|
10834
10979
|
`,
|
|
10835
10980
|
[
|
|
@@ -10837,6 +10982,8 @@ export async function recordPostgresSchedulerRunnerTerminal(
|
|
|
10837
10982
|
input.attempt,
|
|
10838
10983
|
stringifyPostgresJson(input.result ?? null),
|
|
10839
10984
|
stringifyPostgresJson(runnerTerminalLogPayload(input)),
|
|
10985
|
+
input.expectedHeartbeatIdentity !== undefined,
|
|
10986
|
+
input.expectedHeartbeatIdentity ?? null,
|
|
10840
10987
|
],
|
|
10841
10988
|
);
|
|
10842
10989
|
return recorded.rows.length > 0;
|
|
@@ -10871,6 +11018,29 @@ function terminalWorkCounterSegmentFromRunnerResult(
|
|
|
10871
11018
|
});
|
|
10872
11019
|
}
|
|
10873
11020
|
|
|
11021
|
+
/** Only the authenticated raw runner envelope supplies this fact. */
|
|
11022
|
+
function terminalRuntimePerformanceFacts(result: unknown, attempt: number) {
|
|
11023
|
+
const envelope =
|
|
11024
|
+
result && typeof result === 'object' && !Array.isArray(result)
|
|
11025
|
+
? (result as { stats?: unknown })
|
|
11026
|
+
: undefined;
|
|
11027
|
+
const identity = terminalWorkCounterSegmentFromRunnerResult(result, attempt);
|
|
11028
|
+
const performance = identity
|
|
11029
|
+
? terminalRuntimePerformanceFromStats(envelope?.stats, identity)
|
|
11030
|
+
: undefined;
|
|
11031
|
+
const hasResources =
|
|
11032
|
+
envelope?.stats &&
|
|
11033
|
+
typeof envelope.stats === 'object' &&
|
|
11034
|
+
Object.hasOwn(envelope.stats, 'runtimeResources');
|
|
11035
|
+
return {
|
|
11036
|
+
performance,
|
|
11037
|
+
unavailableReason:
|
|
11038
|
+
hasResources && !performance
|
|
11039
|
+
? ('invalid_runner_stats' as const)
|
|
11040
|
+
: undefined,
|
|
11041
|
+
};
|
|
11042
|
+
}
|
|
11043
|
+
|
|
10874
11044
|
/** Close the admitted invocation in the same durable fact as its terminal. */
|
|
10875
11045
|
function terminalWorkCounterProgressEventsFromRunnerResult(
|
|
10876
11046
|
result: unknown,
|
|
@@ -10972,6 +11142,10 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
10972
11142
|
input.result,
|
|
10973
11143
|
input.attempt,
|
|
10974
11144
|
);
|
|
11145
|
+
const terminalPerformanceFacts = terminalRuntimePerformanceFacts(
|
|
11146
|
+
input.result,
|
|
11147
|
+
input.attempt,
|
|
11148
|
+
);
|
|
10975
11149
|
// `workCompletedAtMs` is useful lifecycle telemetry, but it comes from the
|
|
10976
11150
|
// sandbox process where play code can alter Date.now. Billing must stop on
|
|
10977
11151
|
// the first database receipt, whose clock and first-write semantics are
|
|
@@ -11198,7 +11372,8 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
11198
11372
|
finished_at = now(),
|
|
11199
11373
|
updated_at = now()
|
|
11200
11374
|
WHERE run_id IN (SELECT run_id FROM completed_attempt)
|
|
11201
|
-
RETURNING run_id
|
|
11375
|
+
RETURNING run_id, launch_json#>>'{contractSnapshot,compatibility,terminalRuntimePerformanceSupported}' = 'true' AS terminal_runtime_performance_supported,
|
|
11376
|
+
launch_json#>>'{contractSnapshot,compatibility,terminalRuntimeBodyAdmissionSupported}' = 'true' AS terminal_runtime_body_admission_supported
|
|
11202
11377
|
),
|
|
11203
11378
|
compute_billing_job AS (
|
|
11204
11379
|
INSERT INTO ${tableName('compute_billing_jobs_v2', options)} (run_id)
|
|
@@ -11242,7 +11417,18 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
11242
11417
|
), $8::jsonb),
|
|
11243
11418
|
'resultSummary', $16::jsonb,
|
|
11244
11419
|
'terminalWorkCounterSegment', $18::jsonb
|
|
11245
|
-
)
|
|
11420
|
+
) || CASE WHEN run_terminal.terminal_runtime_performance_supported THEN jsonb_build_object(
|
|
11421
|
+
'terminalRuntimePerformance', CASE
|
|
11422
|
+
WHEN run_terminal.terminal_runtime_body_admission_supported THEN $21::jsonb
|
|
11423
|
+
WHEN jsonb_typeof($21::jsonb->'memoryAdmission') = 'object' THEN
|
|
11424
|
+
CASE WHEN ($21::jsonb->'memoryAdmission') - 'toolResponseBody' = '{}'::jsonb
|
|
11425
|
+
THEN $21::jsonb - 'memoryAdmission'
|
|
11426
|
+
ELSE jsonb_set($21::jsonb, '{memoryAdmission}', ($21::jsonb->'memoryAdmission') - 'toolResponseBody')
|
|
11427
|
+
END
|
|
11428
|
+
ELSE $21::jsonb
|
|
11429
|
+
END,
|
|
11430
|
+
'terminalRuntimePerformanceUnavailableReason', $22::text
|
|
11431
|
+
) ELSE '{}'::jsonb END,
|
|
11246
11432
|
${causalOutboxCreatedAt('run_terminal.run_id', options)}
|
|
11247
11433
|
FROM run_terminal
|
|
11248
11434
|
ON CONFLICT (event_id) DO NOTHING
|
|
@@ -11282,6 +11468,8 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
11282
11468
|
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
11283
11469
|
stringifyPostgresJson(runnerTerminalLogPayload(input)),
|
|
11284
11470
|
JSON.stringify(terminalLogProjection.lineMetadata),
|
|
11471
|
+
stringifyPostgresJson(terminalPerformanceFacts.performance ?? null),
|
|
11472
|
+
terminalPerformanceFacts.unavailableReason ?? null,
|
|
11285
11473
|
],
|
|
11286
11474
|
);
|
|
11287
11475
|
if (completed.rows.length > 0) return 'completed';
|
|
@@ -12426,10 +12614,171 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
|
|
|
12426
12614
|
(provider) => provider === 'daytona' || provider === 'modal',
|
|
12427
12615
|
);
|
|
12428
12616
|
if (providers?.length === 0) return [];
|
|
12429
|
-
//
|
|
12430
|
-
//
|
|
12431
|
-
|
|
12432
|
-
|
|
12617
|
+
// The v12 table retains deleted history. Avoid planning the lineage repair
|
|
12618
|
+
// when neither a legacy row nor a v2 NULL-domain duplicate needs adoption.
|
|
12619
|
+
// A row arriving after this read is picked up by the next claim poll; the
|
|
12620
|
+
// provider-qualified v2 claim below still runs on every call.
|
|
12621
|
+
const adoptionWork = await client.query<{ pending: boolean }>(`
|
|
12622
|
+
SELECT EXISTS (
|
|
12623
|
+
SELECT 1 FROM ${tableName('sandbox_cleanup_jobs_v2', options)}
|
|
12624
|
+
WHERE provider = 'daytona' AND routing_domain IS NULL
|
|
12625
|
+
AND status <> 'deleted'
|
|
12626
|
+
) OR EXISTS (
|
|
12627
|
+
SELECT 1 FROM ${tableName('sandbox_cleanup_jobs', options)}
|
|
12628
|
+
WHERE status <> 'deleted'
|
|
12629
|
+
) AS pending
|
|
12630
|
+
`);
|
|
12631
|
+
if (typeof adoptionWork.rows[0]?.pending !== 'boolean') {
|
|
12632
|
+
throw new Error(
|
|
12633
|
+
'Sandbox cleanup adoption preflight returned no pending proof.',
|
|
12634
|
+
);
|
|
12635
|
+
}
|
|
12636
|
+
// A v12 NULL-domain row can be copied to v2 before provider deletion gives
|
|
12637
|
+
// the v2 tombstone its observed domain. The moved primary key would
|
|
12638
|
+
// otherwise make every later adoption recreate a pending NULL-domain row.
|
|
12639
|
+
// Reconcile only the exact original identity proved by the immutable
|
|
12640
|
+
// provider confirmation and the matching deleted v2 tombstone.
|
|
12641
|
+
if (adoptionWork.rows[0].pending)
|
|
12642
|
+
await client.query(
|
|
12643
|
+
`
|
|
12644
|
+
WITH pending_legacy AS MATERIALIZED (
|
|
12645
|
+
SELECT sandbox_id, run_id, attempt, environment, routing_domain,
|
|
12646
|
+
attempt_count, failure_count, available_at, last_error,
|
|
12647
|
+
created_at, blocked_at, block_reason
|
|
12648
|
+
FROM ${tableName('sandbox_cleanup_jobs', options)}
|
|
12649
|
+
WHERE status <> 'deleted'
|
|
12650
|
+
), unresolved_null_domain AS MATERIALIZED (
|
|
12651
|
+
SELECT sandbox_id, run_id, attempt, environment
|
|
12652
|
+
FROM ${tableName('sandbox_cleanup_jobs_v2', options)}
|
|
12653
|
+
WHERE provider = 'daytona'
|
|
12654
|
+
AND routing_domain IS NULL
|
|
12655
|
+
AND status <> 'deleted'
|
|
12656
|
+
UNION
|
|
12657
|
+
SELECT sandbox_id, run_id, attempt, environment
|
|
12658
|
+
FROM pending_legacy
|
|
12659
|
+
WHERE routing_domain IS NULL
|
|
12660
|
+
), confirmed_null_domain AS MATERIALIZED (
|
|
12661
|
+
SELECT
|
|
12662
|
+
deleted.sandbox_id, deleted.run_id, deleted.attempt,
|
|
12663
|
+
deleted.environment,
|
|
12664
|
+
MIN(COALESCE(deleted.deleted_at, confirmation.created_at)) AS confirmed_deleted_at
|
|
12665
|
+
FROM unresolved_null_domain AS original
|
|
12666
|
+
JOIN ${tableName('sandbox_cleanup_jobs_v2', options)} AS deleted
|
|
12667
|
+
ON deleted.sandbox_id = original.sandbox_id
|
|
12668
|
+
AND deleted.run_id = original.run_id
|
|
12669
|
+
AND deleted.attempt = original.attempt
|
|
12670
|
+
AND deleted.environment = original.environment
|
|
12671
|
+
JOIN ${tableName('sandbox_cleanup_events_v2', options)} AS confirmation
|
|
12672
|
+
ON confirmation.provider = deleted.provider
|
|
12673
|
+
AND confirmation.sandbox_id = deleted.sandbox_id
|
|
12674
|
+
AND confirmation.run_id = deleted.run_id
|
|
12675
|
+
AND confirmation.attempt = deleted.attempt
|
|
12676
|
+
AND confirmation.routing_domain IS NULL
|
|
12677
|
+
AND confirmation.event_type = 'provider_delete_confirmed'
|
|
12678
|
+
AND confirmation.details_json->>'providerOutcome' = 'deleted'
|
|
12679
|
+
AND confirmation.details_json->>'routingDomain' = deleted.routing_domain
|
|
12680
|
+
WHERE deleted.provider = 'daytona'
|
|
12681
|
+
AND deleted.status = 'deleted'
|
|
12682
|
+
AND deleted.routing_domain IS NOT NULL
|
|
12683
|
+
GROUP BY deleted.sandbox_id, deleted.run_id, deleted.attempt,
|
|
12684
|
+
deleted.environment
|
|
12685
|
+
), settled_duplicate AS (
|
|
12686
|
+
UPDATE ${tableName('sandbox_cleanup_jobs_v2', options)} AS pending
|
|
12687
|
+
SET status = 'deleted',
|
|
12688
|
+
deleted_at = confirmed.confirmed_deleted_at,
|
|
12689
|
+
lease_owner = NULL,
|
|
12690
|
+
lease_expires_at = NULL,
|
|
12691
|
+
last_error = NULL,
|
|
12692
|
+
updated_at = clock_timestamp()
|
|
12693
|
+
FROM confirmed_null_domain AS confirmed
|
|
12694
|
+
WHERE pending.provider = 'daytona'
|
|
12695
|
+
AND pending.sandbox_id = confirmed.sandbox_id
|
|
12696
|
+
AND pending.run_id = confirmed.run_id
|
|
12697
|
+
AND pending.attempt = confirmed.attempt
|
|
12698
|
+
AND pending.environment = confirmed.environment
|
|
12699
|
+
AND pending.routing_domain IS NULL
|
|
12700
|
+
AND pending.status <> 'deleted'
|
|
12701
|
+
RETURNING pending.sandbox_id
|
|
12702
|
+
), proven_legacy AS MATERIALIZED (
|
|
12703
|
+
SELECT legacy.sandbox_id, legacy.run_id, legacy.attempt,
|
|
12704
|
+
legacy.environment, legacy.routing_domain,
|
|
12705
|
+
COALESCE(
|
|
12706
|
+
(
|
|
12707
|
+
SELECT deleted.deleted_at
|
|
12708
|
+
FROM ${tableName('sandbox_cleanup_jobs_v2', options)} AS deleted
|
|
12709
|
+
WHERE deleted.provider = 'daytona'
|
|
12710
|
+
AND deleted.sandbox_id = legacy.sandbox_id
|
|
12711
|
+
AND deleted.run_id = legacy.run_id
|
|
12712
|
+
AND deleted.attempt = legacy.attempt
|
|
12713
|
+
AND deleted.environment = legacy.environment
|
|
12714
|
+
AND deleted.routing_domain IS NOT DISTINCT FROM legacy.routing_domain
|
|
12715
|
+
AND deleted.status = 'deleted'
|
|
12716
|
+
),
|
|
12717
|
+
(
|
|
12718
|
+
SELECT confirmed.confirmed_deleted_at
|
|
12719
|
+
FROM confirmed_null_domain AS confirmed
|
|
12720
|
+
WHERE legacy.routing_domain IS NULL
|
|
12721
|
+
AND confirmed.sandbox_id = legacy.sandbox_id
|
|
12722
|
+
AND confirmed.run_id = legacy.run_id
|
|
12723
|
+
AND confirmed.attempt = legacy.attempt
|
|
12724
|
+
AND confirmed.environment = legacy.environment
|
|
12725
|
+
),
|
|
12726
|
+
(
|
|
12727
|
+
SELECT MIN(confirmation.created_at)
|
|
12728
|
+
FROM ${tableName('sandbox_cleanup_events_v2', options)} AS confirmation
|
|
12729
|
+
WHERE confirmation.provider = 'daytona'
|
|
12730
|
+
AND confirmation.sandbox_id = legacy.sandbox_id
|
|
12731
|
+
AND confirmation.run_id = legacy.run_id
|
|
12732
|
+
AND confirmation.attempt = legacy.attempt
|
|
12733
|
+
AND confirmation.routing_domain IS NOT DISTINCT FROM legacy.routing_domain
|
|
12734
|
+
AND confirmation.event_type = 'provider_delete_confirmed'
|
|
12735
|
+
AND confirmation.details_json->>'providerOutcome' IN ('deleted', 'already_absent')
|
|
12736
|
+
AND confirmation.details_json->>'routingDomain' IS NOT DISTINCT FROM legacy.routing_domain
|
|
12737
|
+
)
|
|
12738
|
+
) AS confirmed_deleted_at
|
|
12739
|
+
FROM pending_legacy AS legacy
|
|
12740
|
+
WHERE (
|
|
12741
|
+
EXISTS (
|
|
12742
|
+
SELECT 1
|
|
12743
|
+
FROM ${tableName('sandbox_cleanup_jobs_v2', options)} AS deleted
|
|
12744
|
+
WHERE deleted.provider = 'daytona'
|
|
12745
|
+
AND deleted.sandbox_id = legacy.sandbox_id
|
|
12746
|
+
AND deleted.run_id = legacy.run_id
|
|
12747
|
+
AND deleted.attempt = legacy.attempt
|
|
12748
|
+
AND deleted.environment = legacy.environment
|
|
12749
|
+
AND deleted.routing_domain IS NOT DISTINCT FROM legacy.routing_domain
|
|
12750
|
+
AND deleted.status = 'deleted'
|
|
12751
|
+
)
|
|
12752
|
+
OR (
|
|
12753
|
+
legacy.routing_domain IS NULL
|
|
12754
|
+
AND EXISTS (
|
|
12755
|
+
SELECT 1 FROM confirmed_null_domain AS confirmed
|
|
12756
|
+
WHERE confirmed.sandbox_id = legacy.sandbox_id
|
|
12757
|
+
AND confirmed.run_id = legacy.run_id
|
|
12758
|
+
AND confirmed.attempt = legacy.attempt
|
|
12759
|
+
AND confirmed.environment = legacy.environment
|
|
12760
|
+
)
|
|
12761
|
+
)
|
|
12762
|
+
)
|
|
12763
|
+
), settled_legacy AS (
|
|
12764
|
+
UPDATE ${tableName('sandbox_cleanup_jobs', options)} AS legacy
|
|
12765
|
+
SET status = 'deleted',
|
|
12766
|
+
deleted_at = COALESCE(proven.confirmed_deleted_at, legacy.deleted_at),
|
|
12767
|
+
lease_owner = NULL,
|
|
12768
|
+
lease_expires_at = NULL,
|
|
12769
|
+
last_error = NULL,
|
|
12770
|
+
updated_at = clock_timestamp()
|
|
12771
|
+
FROM proven_legacy AS proven
|
|
12772
|
+
-- Completion locks v2 before legacy. Consume the v2 update first so
|
|
12773
|
+
-- concurrent adoption cannot acquire the same rows in reverse order.
|
|
12774
|
+
CROSS JOIN (SELECT count(*) FROM settled_duplicate) AS duplicate_fence
|
|
12775
|
+
WHERE legacy.sandbox_id = proven.sandbox_id
|
|
12776
|
+
AND legacy.run_id = proven.run_id
|
|
12777
|
+
AND legacy.attempt = proven.attempt
|
|
12778
|
+
AND legacy.environment = proven.environment
|
|
12779
|
+
AND legacy.routing_domain IS NOT DISTINCT FROM proven.routing_domain
|
|
12780
|
+
RETURNING legacy.sandbox_id
|
|
12781
|
+
)
|
|
12433
12782
|
INSERT INTO ${tableName('sandbox_cleanup_jobs_v2', options)} (
|
|
12434
12783
|
provider,
|
|
12435
12784
|
sandbox_id,
|
|
@@ -12463,11 +12812,18 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
|
|
|
12463
12812
|
now(),
|
|
12464
12813
|
blocked_at,
|
|
12465
12814
|
block_reason
|
|
12466
|
-
FROM
|
|
12467
|
-
WHERE
|
|
12815
|
+
FROM pending_legacy AS legacy
|
|
12816
|
+
WHERE NOT EXISTS (
|
|
12817
|
+
SELECT 1 FROM proven_legacy AS proven
|
|
12818
|
+
WHERE proven.sandbox_id = legacy.sandbox_id
|
|
12819
|
+
AND proven.run_id = legacy.run_id
|
|
12820
|
+
AND proven.attempt = legacy.attempt
|
|
12821
|
+
AND proven.environment = legacy.environment
|
|
12822
|
+
AND proven.routing_domain IS NOT DISTINCT FROM legacy.routing_domain
|
|
12823
|
+
)
|
|
12468
12824
|
ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
|
|
12469
12825
|
`,
|
|
12470
|
-
|
|
12826
|
+
);
|
|
12471
12827
|
const claimed = await client.query<{
|
|
12472
12828
|
provider: 'daytona' | 'modal';
|
|
12473
12829
|
sandbox_id: string;
|
|
@@ -12768,6 +13124,62 @@ export async function recordPostgresSchedulerSandboxCleanupEvent(
|
|
|
12768
13124
|
);
|
|
12769
13125
|
}
|
|
12770
13126
|
|
|
13127
|
+
export type PostgresSchedulerSandboxDeleteAcceptance = {
|
|
13128
|
+
acceptedAtMs: number;
|
|
13129
|
+
/** The organization of the sandbox Daytona accepted the delete for. */
|
|
13130
|
+
organizationId: string | null;
|
|
13131
|
+
};
|
|
13132
|
+
|
|
13133
|
+
/**
|
|
13134
|
+
* The first delete the provider accepted for this job. A cleanup that was
|
|
13135
|
+
* accepted and is awaiting its read-back reads this between claims.
|
|
13136
|
+
*/
|
|
13137
|
+
export async function readPostgresSchedulerSandboxDeleteAcceptance(
|
|
13138
|
+
client: PostgresSchedulerQueryClient,
|
|
13139
|
+
input: {
|
|
13140
|
+
provider: 'daytona' | 'modal';
|
|
13141
|
+
sandboxId: string;
|
|
13142
|
+
routingDomain: string | null;
|
|
13143
|
+
runId: string;
|
|
13144
|
+
attempt: number;
|
|
13145
|
+
},
|
|
13146
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
13147
|
+
): Promise<PostgresSchedulerSandboxDeleteAcceptance | null> {
|
|
13148
|
+
const accepted = await client.query<{
|
|
13149
|
+
accepted_at_ms: string;
|
|
13150
|
+
organization_id: string | null;
|
|
13151
|
+
}>(
|
|
13152
|
+
`
|
|
13153
|
+
SELECT
|
|
13154
|
+
(extract(epoch FROM created_at) * 1000)::bigint::text AS accepted_at_ms,
|
|
13155
|
+
details_json->>'routingDomain' AS organization_id
|
|
13156
|
+
FROM ${tableName('sandbox_cleanup_events_v2', options)}
|
|
13157
|
+
WHERE run_id = $1
|
|
13158
|
+
AND attempt = $2
|
|
13159
|
+
AND provider = $3
|
|
13160
|
+
AND sandbox_id = $4
|
|
13161
|
+
AND routing_domain IS NOT DISTINCT FROM $5
|
|
13162
|
+
AND event_type = 'provider_delete_accepted'
|
|
13163
|
+
ORDER BY created_at, event_id
|
|
13164
|
+
LIMIT 1
|
|
13165
|
+
`,
|
|
13166
|
+
[
|
|
13167
|
+
input.runId,
|
|
13168
|
+
input.attempt,
|
|
13169
|
+
input.provider,
|
|
13170
|
+
input.sandboxId,
|
|
13171
|
+
input.routingDomain?.trim() || null,
|
|
13172
|
+
],
|
|
13173
|
+
);
|
|
13174
|
+
const row = accepted.rows[0];
|
|
13175
|
+
return row
|
|
13176
|
+
? {
|
|
13177
|
+
acceptedAtMs: Number(row.accepted_at_ms),
|
|
13178
|
+
organizationId: row.organization_id,
|
|
13179
|
+
}
|
|
13180
|
+
: null;
|
|
13181
|
+
}
|
|
13182
|
+
|
|
12771
13183
|
export async function completePostgresSchedulerSandboxCleanupJob(
|
|
12772
13184
|
client: PostgresSchedulerQueryClient,
|
|
12773
13185
|
input: {
|
|
@@ -12789,21 +13201,39 @@ export async function completePostgresSchedulerSandboxCleanupJob(
|
|
|
12789
13201
|
| 'missing_obligation'
|
|
12790
13202
|
| 'wrong_routing_domain'
|
|
12791
13203
|
> {
|
|
13204
|
+
// `deleted_at` feeds billing end times, so it stays the moment the provider
|
|
13205
|
+
// took the delete. Waiting for the read-back must not bill the customer for
|
|
13206
|
+
// Daytona's record outliving it; `delete_confirmed_at` keeps the confirmation.
|
|
12792
13207
|
const completed = await client.query<{ outcome: string }>(
|
|
12793
13208
|
`
|
|
12794
13209
|
WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $3', options)}, current_job AS MATERIALIZED (
|
|
12795
|
-
SELECT status, attempt_count, lease_owner, routing_domain
|
|
13210
|
+
SELECT status, attempt_count, lease_owner, routing_domain, environment
|
|
12796
13211
|
FROM ${tableName('sandbox_cleanup_jobs_v2', options)} AS sandbox_cleanup_jobs_v2
|
|
12797
13212
|
CROSS JOIN (SELECT count(*) FROM custody_operations) AS operation_locks
|
|
12798
13213
|
WHERE provider = $8
|
|
12799
13214
|
AND sandbox_id = $1
|
|
13215
|
+
AND run_id = $3
|
|
13216
|
+
AND attempt = $4
|
|
12800
13217
|
AND routing_domain IS NOT DISTINCT FROM $9
|
|
12801
13218
|
FOR UPDATE OF sandbox_cleanup_jobs_v2
|
|
12802
13219
|
),
|
|
13220
|
+
delete_acceptance AS MATERIALIZED (
|
|
13221
|
+
SELECT min(event.created_at) AS accepted_at
|
|
13222
|
+
FROM ${tableName('sandbox_cleanup_events_v2', options)} AS event
|
|
13223
|
+
WHERE event.run_id = $3
|
|
13224
|
+
AND event.attempt = $4
|
|
13225
|
+
AND event.provider = $8
|
|
13226
|
+
AND event.sandbox_id = $1
|
|
13227
|
+
AND event.routing_domain IS NOT DISTINCT FROM $9
|
|
13228
|
+
AND event.event_type = 'provider_delete_accepted'
|
|
13229
|
+
),
|
|
12803
13230
|
completed AS (
|
|
12804
13231
|
UPDATE ${tableName('sandbox_cleanup_jobs_v2', options)} AS target
|
|
12805
13232
|
SET status = 'deleted',
|
|
12806
|
-
deleted_at =
|
|
13233
|
+
deleted_at = COALESCE(
|
|
13234
|
+
(SELECT accepted_at FROM delete_acceptance),
|
|
13235
|
+
clock_timestamp()
|
|
13236
|
+
),
|
|
12807
13237
|
routing_domain = COALESCE(
|
|
12808
13238
|
target.routing_domain,
|
|
12809
13239
|
NULLIF($7::text, '')
|
|
@@ -12815,6 +13245,8 @@ export async function completePostgresSchedulerSandboxCleanupJob(
|
|
|
12815
13245
|
FROM current_job
|
|
12816
13246
|
WHERE target.provider = $8
|
|
12817
13247
|
AND target.sandbox_id = $1
|
|
13248
|
+
AND target.run_id = $3
|
|
13249
|
+
AND target.attempt = $4
|
|
12818
13250
|
AND target.routing_domain IS NOT DISTINCT FROM $9
|
|
12819
13251
|
AND current_job.status <> 'deleted'
|
|
12820
13252
|
AND (
|
|
@@ -12884,7 +13316,12 @@ export async function completePostgresSchedulerSandboxCleanupJob(
|
|
|
12884
13316
|
)
|
|
12885
13317
|
THEN resource || jsonb_build_object(
|
|
12886
13318
|
'billingEndedAt',
|
|
12887
|
-
extract(
|
|
13319
|
+
extract(
|
|
13320
|
+
epoch FROM COALESCE(
|
|
13321
|
+
(SELECT accepted_at FROM delete_acceptance),
|
|
13322
|
+
clock_timestamp()
|
|
13323
|
+
)
|
|
13324
|
+
) * 1000
|
|
12888
13325
|
)
|
|
12889
13326
|
ELSE resource
|
|
12890
13327
|
END
|
|
@@ -13012,6 +13449,25 @@ export async function completePostgresSchedulerSandboxCleanupJob(
|
|
|
13012
13449
|
)
|
|
13013
13450
|
RETURNING provider, sandbox_id, routing_domain
|
|
13014
13451
|
),
|
|
13452
|
+
legacy_completion AS (
|
|
13453
|
+
UPDATE ${tableName('sandbox_cleanup_jobs', options)} AS legacy
|
|
13454
|
+
SET status = 'deleted',
|
|
13455
|
+
deleted_at = COALESCE(legacy.deleted_at, clock_timestamp()),
|
|
13456
|
+
lease_owner = NULL,
|
|
13457
|
+
lease_expires_at = NULL,
|
|
13458
|
+
last_error = NULL,
|
|
13459
|
+
updated_at = clock_timestamp()
|
|
13460
|
+
FROM provider_confirmation AS confirmation
|
|
13461
|
+
CROSS JOIN current_job
|
|
13462
|
+
WHERE confirmation.provider = 'daytona'
|
|
13463
|
+
AND legacy.sandbox_id = confirmation.sandbox_id
|
|
13464
|
+
AND legacy.run_id = $3
|
|
13465
|
+
AND legacy.attempt = $4
|
|
13466
|
+
AND legacy.environment = current_job.environment
|
|
13467
|
+
AND legacy.routing_domain IS NOT DISTINCT FROM confirmation.routing_domain
|
|
13468
|
+
AND legacy.status <> 'deleted'
|
|
13469
|
+
RETURNING legacy.sandbox_id
|
|
13470
|
+
),
|
|
13015
13471
|
durable_completion AS (
|
|
13016
13472
|
INSERT INTO ${tableName('sandbox_cleanup_events_v2', options)} (
|
|
13017
13473
|
provider,
|
|
@@ -13272,6 +13728,8 @@ export type PostgresSchedulerCompletedRunnerTerminal = {
|
|
|
13272
13728
|
attempt: number;
|
|
13273
13729
|
leaseToken: string;
|
|
13274
13730
|
result: unknown;
|
|
13731
|
+
/** The detached suspension still on the run, when there is one. */
|
|
13732
|
+
suspension?: unknown;
|
|
13275
13733
|
};
|
|
13276
13734
|
|
|
13277
13735
|
export async function listPostgresSchedulerCompletedRunnerTerminals(
|
|
@@ -13311,14 +13769,16 @@ export async function listPostgresSchedulerCompletedRunnerTerminals(
|
|
|
13311
13769
|
terminals.run_id,
|
|
13312
13770
|
terminals.attempt,
|
|
13313
13771
|
terminals.lease_token,
|
|
13314
|
-
terminals.runner_result_json
|
|
13772
|
+
terminals.runner_result_json,
|
|
13773
|
+
terminals.suspension_json
|
|
13315
13774
|
FROM requested_lanes AS lanes
|
|
13316
13775
|
CROSS JOIN LATERAL (
|
|
13317
13776
|
SELECT runs.launch_json ->> 'absurdReleaseId' AS release_id,
|
|
13318
13777
|
attempts.run_id,
|
|
13319
13778
|
attempts.attempt,
|
|
13320
13779
|
attempts.lease_token,
|
|
13321
|
-
attempts.runner_result_json
|
|
13780
|
+
attempts.runner_result_json,
|
|
13781
|
+
runs.suspension_json
|
|
13322
13782
|
FROM ${tableName('runs', options)} AS runs
|
|
13323
13783
|
JOIN ${tableName('attempts', options)} AS attempts
|
|
13324
13784
|
ON attempts.run_id = runs.run_id
|
|
@@ -13344,7 +13804,8 @@ export async function listPostgresSchedulerCompletedRunnerTerminals(
|
|
|
13344
13804
|
attempts.run_id,
|
|
13345
13805
|
attempts.attempt,
|
|
13346
13806
|
attempts.lease_token,
|
|
13347
|
-
attempts.runner_result_json
|
|
13807
|
+
attempts.runner_result_json,
|
|
13808
|
+
runs.suspension_json
|
|
13348
13809
|
FROM ${tableName('attempts', options)} AS attempts
|
|
13349
13810
|
JOIN ${tableName('runs', options)} AS runs
|
|
13350
13811
|
ON runs.run_id = attempts.run_id
|
|
@@ -13365,6 +13826,7 @@ export async function listPostgresSchedulerCompletedRunnerTerminals(
|
|
|
13365
13826
|
attempt: number;
|
|
13366
13827
|
lease_token: string;
|
|
13367
13828
|
runner_result_json: unknown;
|
|
13829
|
+
suspension_json: unknown;
|
|
13368
13830
|
}>(terminalQuery.sql, terminalQuery.params);
|
|
13369
13831
|
return terminals.rows.map((row) => ({
|
|
13370
13832
|
releaseId: row.release_id,
|
|
@@ -13372,56 +13834,319 @@ export async function listPostgresSchedulerCompletedRunnerTerminals(
|
|
|
13372
13834
|
attempt: row.attempt,
|
|
13373
13835
|
leaseToken: row.lease_token,
|
|
13374
13836
|
result: row.runner_result_json,
|
|
13837
|
+
suspension: row.suspension_json,
|
|
13375
13838
|
}));
|
|
13376
13839
|
}
|
|
13377
13840
|
|
|
13841
|
+
/** A parked detached runner whose heartbeat is stale beyond its own lease. */
|
|
13842
|
+
export type PostgresSchedulerStaleDetachedRunner = {
|
|
13843
|
+
provider: 'daytona' | 'modal';
|
|
13844
|
+
releaseId: string | null;
|
|
13845
|
+
runId: string;
|
|
13846
|
+
attempt: number;
|
|
13847
|
+
leaseToken: string;
|
|
13848
|
+
/** Postgres text, so the verdict fence compares every microsecond. */
|
|
13849
|
+
heartbeatAt: string | null;
|
|
13850
|
+
heartbeatAtMs: number | null;
|
|
13851
|
+
suspension: unknown;
|
|
13852
|
+
};
|
|
13853
|
+
|
|
13854
|
+
/** A runner terminal on a live attempt, on any lane. Identity only: its result
|
|
13855
|
+
* can be large, so settlement reads it one run at a time. */
|
|
13856
|
+
export type PostgresSchedulerUnsettledRunnerTerminal = {
|
|
13857
|
+
releaseId: string | null;
|
|
13858
|
+
runId: string;
|
|
13859
|
+
attempt: number;
|
|
13860
|
+
leaseToken: string;
|
|
13861
|
+
suspension: unknown;
|
|
13862
|
+
};
|
|
13863
|
+
|
|
13378
13864
|
/**
|
|
13379
|
-
* Read
|
|
13380
|
-
*
|
|
13381
|
-
*
|
|
13865
|
+
* Read-only discovery for the liveness sweep, across every release lane. It
|
|
13866
|
+
* selects only runners that need action: a stale parked runner to inspect, or a
|
|
13867
|
+
* recorded terminal to settle. A live heartbeat, a resumable `suspended`
|
|
13868
|
+
* receipt, a passed hard ceiling (its sweep owns that), a newer attempt, and a
|
|
13869
|
+
* queued cancellation all leave a runner out.
|
|
13870
|
+
*
|
|
13871
|
+
* Each provider pages separately, so one provider's backlog cannot crowd out
|
|
13872
|
+
* the other, and `afterRunIds` lets a caller walk past candidates it has read.
|
|
13382
13873
|
*/
|
|
13383
|
-
export async function
|
|
13874
|
+
export async function listPostgresSchedulerDetachedRunnerLivenessWork(
|
|
13384
13875
|
client: PostgresSchedulerQueryClient,
|
|
13385
|
-
input: {
|
|
13876
|
+
input: {
|
|
13877
|
+
limit: number;
|
|
13878
|
+
defaultHeartbeatTimeoutMs: number;
|
|
13879
|
+
afterRunIds: Partial<Record<'daytona' | 'modal', string | null>>;
|
|
13880
|
+
},
|
|
13386
13881
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
13387
13882
|
): Promise<{
|
|
13388
|
-
|
|
13389
|
-
|
|
13390
|
-
|
|
13391
|
-
|
|
13392
|
-
|
|
13393
|
-
|
|
13394
|
-
|
|
13395
|
-
|
|
13883
|
+
stale: PostgresSchedulerStaleDetachedRunner[];
|
|
13884
|
+
terminals: PostgresSchedulerUnsettledRunnerTerminal[];
|
|
13885
|
+
}> {
|
|
13886
|
+
const rows = await client.query<{
|
|
13887
|
+
work: 'inspect' | 'settle';
|
|
13888
|
+
provider: 'daytona' | 'modal';
|
|
13889
|
+
absurd_release_id: string | null;
|
|
13890
|
+
run_id: string;
|
|
13891
|
+
attempt: number;
|
|
13892
|
+
lease_token: string;
|
|
13893
|
+
heartbeat_at: string | null;
|
|
13894
|
+
heartbeat_at_ms: string | null;
|
|
13895
|
+
suspension_json: unknown;
|
|
13396
13896
|
}>(
|
|
13397
13897
|
`
|
|
13398
|
-
|
|
13399
|
-
|
|
13400
|
-
|
|
13898
|
+
WITH detached AS MATERIALIZED (
|
|
13899
|
+
SELECT
|
|
13900
|
+
runs.run_id,
|
|
13901
|
+
attempts.attempt,
|
|
13902
|
+
attempts.lease_token,
|
|
13903
|
+
attempts.status AS attempt_status,
|
|
13904
|
+
runs.status AS run_status,
|
|
13905
|
+
attempts.heartbeat_at,
|
|
13906
|
+
attempts.runner_result_json IS NULL AS no_runner_result,
|
|
13907
|
+
attempts.runner_result_json ->> 'status' AS runner_result_status,
|
|
13908
|
+
runs.suspension_json,
|
|
13909
|
+
runs.wait_deadline,
|
|
13910
|
+
NULLIF(runs.launch_json ->> 'absurdReleaseId', '') AS absurd_release_id,
|
|
13911
|
+
COALESCE(
|
|
13912
|
+
NULLIF(runs.suspension_json -> 'runtimeSandboxRef' ->> 'provider', ''),
|
|
13913
|
+
NULLIF(runs.suspension_json ->> 'sandboxProvider', ''),
|
|
13914
|
+
'daytona'
|
|
13915
|
+
) AS provider
|
|
13916
|
+
FROM ${tableName('runs', options)} AS runs
|
|
13917
|
+
JOIN ${tableName('attempts', options)} AS attempts
|
|
13918
|
+
ON attempts.run_id = runs.run_id
|
|
13919
|
+
AND attempts.attempt = (runs.suspension_json ->> 'runnerAttempt')::integer
|
|
13920
|
+
WHERE runs.status IN ('running', 'waiting')
|
|
13921
|
+
AND runs.suspension_json ->> 'kind' = 'detached_runner'
|
|
13922
|
+
AND attempts.status IN ('running', 'waiting')
|
|
13923
|
+
AND attempts.lease_token IS NOT NULL
|
|
13924
|
+
AND NOT EXISTS (
|
|
13925
|
+
SELECT 1
|
|
13926
|
+
FROM ${tableName('attempts', options)} AS newer
|
|
13927
|
+
WHERE newer.run_id = attempts.run_id
|
|
13928
|
+
AND newer.attempt > attempts.attempt
|
|
13929
|
+
AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed')
|
|
13930
|
+
)
|
|
13931
|
+
),
|
|
13932
|
+
unsettled AS (
|
|
13933
|
+
SELECT *
|
|
13934
|
+
FROM detached
|
|
13935
|
+
WHERE runner_result_status IN ('completed', 'failed')
|
|
13936
|
+
ORDER BY run_id ASC
|
|
13937
|
+
LIMIT $1
|
|
13938
|
+
),
|
|
13939
|
+
stale AS (
|
|
13940
|
+
SELECT
|
|
13941
|
+
detached.*,
|
|
13942
|
+
row_number() OVER (PARTITION BY provider ORDER BY run_id ASC) AS provider_rank
|
|
13943
|
+
FROM detached
|
|
13944
|
+
WHERE provider IN ('daytona', 'modal')
|
|
13945
|
+
AND no_runner_result
|
|
13946
|
+
AND attempt_status = 'waiting'
|
|
13947
|
+
AND run_status = 'waiting'
|
|
13948
|
+
AND (wait_deadline IS NULL OR wait_deadline > clock_timestamp())
|
|
13949
|
+
AND run_id > COALESCE($2::jsonb ->> provider, '')
|
|
13950
|
+
AND NOT EXISTS (
|
|
13951
|
+
SELECT 1
|
|
13952
|
+
FROM ${tableName('control', options)} AS cancel
|
|
13953
|
+
WHERE cancel.key = 'runtime_cancel:' || detached.run_id
|
|
13954
|
+
)
|
|
13955
|
+
AND COALESCE(
|
|
13956
|
+
heartbeat_at,
|
|
13957
|
+
CASE
|
|
13958
|
+
WHEN jsonb_typeof(suspension_json -> 'startedAtMs') = 'number'
|
|
13959
|
+
THEN to_timestamp((suspension_json ->> 'startedAtMs')::double precision / 1000)
|
|
13960
|
+
END
|
|
13961
|
+
) + (
|
|
13962
|
+
CASE
|
|
13963
|
+
WHEN jsonb_typeof(suspension_json -> 'heartbeatTimeoutMs') = 'number'
|
|
13964
|
+
AND (suspension_json ->> 'heartbeatTimeoutMs')::double precision >= 0
|
|
13965
|
+
THEN (suspension_json ->> 'heartbeatTimeoutMs')::double precision
|
|
13966
|
+
ELSE $3::double precision
|
|
13967
|
+
END
|
|
13968
|
+
) * interval '1 millisecond' <= clock_timestamp()
|
|
13969
|
+
)
|
|
13970
|
+
SELECT
|
|
13971
|
+
'settle' AS work, provider, absurd_release_id, run_id, attempt,
|
|
13972
|
+
lease_token, heartbeat_at::text AS heartbeat_at,
|
|
13973
|
+
(extract(epoch FROM heartbeat_at) * 1000)::bigint::text AS heartbeat_at_ms,
|
|
13974
|
+
suspension_json
|
|
13975
|
+
FROM unsettled
|
|
13976
|
+
UNION ALL
|
|
13977
|
+
SELECT
|
|
13978
|
+
'inspect', provider, absurd_release_id, run_id, attempt,
|
|
13979
|
+
lease_token, heartbeat_at::text,
|
|
13980
|
+
(extract(epoch FROM heartbeat_at) * 1000)::bigint::text,
|
|
13981
|
+
suspension_json
|
|
13982
|
+
FROM stale
|
|
13983
|
+
WHERE provider_rank <= $1
|
|
13984
|
+
ORDER BY run_id ASC
|
|
13401
13985
|
`,
|
|
13402
|
-
[
|
|
13986
|
+
[
|
|
13987
|
+
Math.min(100, Math.max(1, Math.floor(input.limit))),
|
|
13988
|
+
JSON.stringify(input.afterRunIds),
|
|
13989
|
+
input.defaultHeartbeatTimeoutMs,
|
|
13990
|
+
],
|
|
13403
13991
|
);
|
|
13404
|
-
const
|
|
13405
|
-
|
|
13406
|
-
|
|
13407
|
-
|
|
13992
|
+
const identity = (row: (typeof rows.rows)[number]) => ({
|
|
13993
|
+
releaseId: row.absurd_release_id,
|
|
13994
|
+
runId: row.run_id,
|
|
13995
|
+
attempt: row.attempt,
|
|
13996
|
+
leaseToken: row.lease_token,
|
|
13997
|
+
suspension: row.suspension_json,
|
|
13998
|
+
});
|
|
13408
13999
|
return {
|
|
13409
|
-
|
|
13410
|
-
|
|
13411
|
-
|
|
14000
|
+
stale: rows.rows
|
|
14001
|
+
.filter((row) => row.work === 'inspect')
|
|
14002
|
+
.map((row) => ({
|
|
14003
|
+
...identity(row),
|
|
14004
|
+
provider: row.provider,
|
|
14005
|
+
heartbeatAt: row.heartbeat_at,
|
|
14006
|
+
heartbeatAtMs:
|
|
14007
|
+
row.heartbeat_at_ms === null ? null : Number(row.heartbeat_at_ms),
|
|
14008
|
+
})),
|
|
14009
|
+
terminals: rows.rows
|
|
14010
|
+
.filter((row) => row.work === 'settle')
|
|
14011
|
+
.map((row) => identity(row)),
|
|
13412
14012
|
};
|
|
13413
14013
|
}
|
|
13414
14014
|
|
|
13415
14015
|
/**
|
|
13416
|
-
*
|
|
14016
|
+
* The sweep's first write. It records the same runner terminal the parked
|
|
14017
|
+
* attempt would, so whichever of the sweep and the wake path settles it commits
|
|
14018
|
+
* one verdict. Everything the provider read could have outdated is fenced in
|
|
14019
|
+
* this one statement: the run must still be parked on this exact attempt, with
|
|
14020
|
+
* the same lease and the same heartbeat, no terminal, no newer attempt, and no
|
|
14021
|
+
* queued cancellation. A false return means something moved during the read,
|
|
14022
|
+
* and it wins.
|
|
13417
14023
|
*
|
|
13418
|
-
*
|
|
13419
|
-
* Require this exact attempt to remain active, the run to remain nonterminal,
|
|
13420
|
-
* and no newer attempt to exist. Gateway replicas must never infer this fence
|
|
13421
|
-
* from process memory.
|
|
14024
|
+
* Takes the Run lock before the attempt lock, matching detached restore.
|
|
13422
14025
|
*/
|
|
13423
|
-
function
|
|
13424
|
-
|
|
14026
|
+
export async function recordPostgresSchedulerDetachedRunnerLivenessVerdict(
|
|
14027
|
+
client: PostgresSchedulerQueryClient,
|
|
14028
|
+
input: {
|
|
14029
|
+
runId: string;
|
|
14030
|
+
attempt: number;
|
|
14031
|
+
leaseToken: string;
|
|
14032
|
+
observedHeartbeatAt: string | null;
|
|
14033
|
+
result: unknown;
|
|
14034
|
+
},
|
|
14035
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
14036
|
+
): Promise<boolean> {
|
|
14037
|
+
const recorded = await client.query<{ run_id: string }>(
|
|
14038
|
+
`
|
|
14039
|
+
WITH parked_run AS MATERIALIZED (
|
|
14040
|
+
SELECT run_id, run_log_store
|
|
14041
|
+
FROM ${tableName('runs', options)}
|
|
14042
|
+
WHERE run_id = $1
|
|
14043
|
+
AND status = 'waiting'
|
|
14044
|
+
AND suspension_json ->> 'kind' = 'detached_runner'
|
|
14045
|
+
AND suspension_json -> 'runnerAttempt' = to_jsonb($2::integer)
|
|
14046
|
+
FOR NO KEY UPDATE
|
|
14047
|
+
),
|
|
14048
|
+
accepted AS MATERIALIZED (
|
|
14049
|
+
SELECT clock_timestamp() AS accepted_at
|
|
14050
|
+
)
|
|
14051
|
+
UPDATE ${tableName('attempts', options)} AS target
|
|
14052
|
+
SET runner_result_json = $5::jsonb,
|
|
14053
|
+
runner_log_payload_json = CASE
|
|
14054
|
+
WHEN parked_run.run_log_store = 'planet_scale' THEN $6::jsonb
|
|
14055
|
+
ELSE NULL
|
|
14056
|
+
END,
|
|
14057
|
+
runner_terminal_received_at = accepted.accepted_at,
|
|
14058
|
+
heartbeat_at = now(),
|
|
14059
|
+
updated_at = now()
|
|
14060
|
+
FROM parked_run, accepted
|
|
14061
|
+
WHERE target.run_id = parked_run.run_id
|
|
14062
|
+
AND target.attempt = $2
|
|
14063
|
+
AND target.status = 'waiting'
|
|
14064
|
+
AND target.lease_token = $3
|
|
14065
|
+
AND target.heartbeat_at IS NOT DISTINCT FROM $4::timestamptz
|
|
14066
|
+
AND target.runner_result_json IS NULL
|
|
14067
|
+
AND NOT EXISTS (
|
|
14068
|
+
SELECT 1
|
|
14069
|
+
FROM ${tableName('attempts', options)} AS newer
|
|
14070
|
+
WHERE newer.run_id = target.run_id
|
|
14071
|
+
AND newer.attempt > target.attempt
|
|
14072
|
+
AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed')
|
|
14073
|
+
)
|
|
14074
|
+
AND NOT EXISTS (
|
|
14075
|
+
SELECT 1
|
|
14076
|
+
FROM ${tableName('control', options)} AS cancel
|
|
14077
|
+
WHERE cancel.key = 'runtime_cancel:' || target.run_id
|
|
14078
|
+
)
|
|
14079
|
+
RETURNING target.run_id
|
|
14080
|
+
`,
|
|
14081
|
+
[
|
|
14082
|
+
input.runId,
|
|
14083
|
+
input.attempt,
|
|
14084
|
+
input.leaseToken,
|
|
14085
|
+
input.observedHeartbeatAt,
|
|
14086
|
+
stringifyPostgresJson(input.result ?? null),
|
|
14087
|
+
stringifyPostgresJson(
|
|
14088
|
+
runnerTerminalLogPayload({
|
|
14089
|
+
attempt: input.attempt,
|
|
14090
|
+
result: input.result,
|
|
14091
|
+
}),
|
|
14092
|
+
),
|
|
14093
|
+
],
|
|
14094
|
+
);
|
|
14095
|
+
return recorded.rows.length > 0;
|
|
14096
|
+
}
|
|
14097
|
+
|
|
14098
|
+
/**
|
|
14099
|
+
* Read the detached runner's most recent liveness fact for timeout diagnosis.
|
|
14100
|
+
* The raw timestamp identity retains PostgreSQL precision for synthetic
|
|
14101
|
+
* runner-loss arbitration; the normalized timestamp remains diagnostic.
|
|
14102
|
+
*/
|
|
14103
|
+
export async function readPostgresSchedulerRunnerLiveness(
|
|
14104
|
+
client: PostgresSchedulerQueryClient,
|
|
14105
|
+
input: { runId: string; attempt: number },
|
|
14106
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
14107
|
+
): Promise<{
|
|
14108
|
+
heartbeatAt: string | null;
|
|
14109
|
+
heartbeatIdentity: string | null;
|
|
14110
|
+
updatedAt: string | null;
|
|
14111
|
+
status: string;
|
|
14112
|
+
} | null> {
|
|
14113
|
+
const result = await client.query<{
|
|
14114
|
+
heartbeat_at: string | Date | null;
|
|
14115
|
+
heartbeat_identity: string | null;
|
|
14116
|
+
updated_at: string | Date | null;
|
|
14117
|
+
status: string;
|
|
14118
|
+
}>(
|
|
14119
|
+
`
|
|
14120
|
+
SELECT heartbeat_at,
|
|
14121
|
+
to_char(heartbeat_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS heartbeat_identity,
|
|
14122
|
+
updated_at, status
|
|
14123
|
+
FROM ${tableName('attempts', options)}
|
|
14124
|
+
WHERE run_id = $1 AND attempt = $2
|
|
14125
|
+
`,
|
|
14126
|
+
[input.runId, input.attempt],
|
|
14127
|
+
);
|
|
14128
|
+
const row = result.rows[0];
|
|
14129
|
+
if (!row) return null;
|
|
14130
|
+
const normalizeTimestamp = (value: string | Date | null): string | null =>
|
|
14131
|
+
value instanceof Date ? value.toISOString() : value;
|
|
14132
|
+
return {
|
|
14133
|
+
heartbeatAt: normalizeTimestamp(row.heartbeat_at),
|
|
14134
|
+
heartbeatIdentity: row.heartbeat_identity,
|
|
14135
|
+
updatedAt: normalizeTimestamp(row.updated_at),
|
|
14136
|
+
status: row.status,
|
|
14137
|
+
};
|
|
14138
|
+
}
|
|
14139
|
+
|
|
14140
|
+
/**
|
|
14141
|
+
* Durable authority for detached-runner progress projection.
|
|
14142
|
+
*
|
|
14143
|
+
* An older runner can keep posting after a timeout starts its replacement.
|
|
14144
|
+
* Require this exact attempt to remain active, the run to remain nonterminal,
|
|
14145
|
+
* and no newer attempt to exist. Gateway replicas must never infer this fence
|
|
14146
|
+
* from process memory.
|
|
14147
|
+
*/
|
|
14148
|
+
function runnerProgressAttemptCurrentCondition(
|
|
14149
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
13425
14150
|
): string {
|
|
13426
14151
|
return `EXISTS (
|
|
13427
14152
|
SELECT 1
|
|
@@ -13482,6 +14207,92 @@ export async function readPostgresSchedulerRunnerAttemptAuthority(
|
|
|
13482
14207
|
};
|
|
13483
14208
|
}
|
|
13484
14209
|
|
|
14210
|
+
export type PostgresSchedulerExecutorAttemptStanding = {
|
|
14211
|
+
runStatus: string;
|
|
14212
|
+
/** Null when the token names no attempt, or no row exists for it. */
|
|
14213
|
+
attemptStatus: string | null;
|
|
14214
|
+
/** A newer attempt exists. Assigning one retires every older attempt. */
|
|
14215
|
+
superseded: boolean;
|
|
14216
|
+
};
|
|
14217
|
+
|
|
14218
|
+
/**
|
|
14219
|
+
* Where an executor token's run and attempt stand, for the gateway's attempt
|
|
14220
|
+
* fence. It reads the same facts as `runnerProgressAttemptCurrentCondition`
|
|
14221
|
+
* but keeps them apart, because only some of them are permanent. Returns null
|
|
14222
|
+
* when the run is missing.
|
|
14223
|
+
*/
|
|
14224
|
+
export async function readPostgresSchedulerExecutorAttemptStanding(
|
|
14225
|
+
client: PostgresSchedulerQueryClient,
|
|
14226
|
+
input: { runId: string; attempt: number | null },
|
|
14227
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
14228
|
+
): Promise<PostgresSchedulerExecutorAttemptStanding | null> {
|
|
14229
|
+
const result = await client.query<{
|
|
14230
|
+
run_status: string;
|
|
14231
|
+
attempt_status: string | null;
|
|
14232
|
+
superseded: boolean;
|
|
14233
|
+
}>(
|
|
14234
|
+
`
|
|
14235
|
+
SELECT run.status AS run_status,
|
|
14236
|
+
attempt.status AS attempt_status,
|
|
14237
|
+
EXISTS (
|
|
14238
|
+
SELECT 1
|
|
14239
|
+
FROM ${tableName('attempts', options)} AS newer_attempt
|
|
14240
|
+
WHERE newer_attempt.run_id = run.run_id
|
|
14241
|
+
AND newer_attempt.attempt > $2::integer
|
|
14242
|
+
) AS superseded
|
|
14243
|
+
FROM ${tableName('runs', options)} AS run
|
|
14244
|
+
LEFT JOIN ${tableName('attempts', options)} AS attempt
|
|
14245
|
+
ON attempt.run_id = run.run_id
|
|
14246
|
+
AND attempt.attempt = $2::integer
|
|
14247
|
+
WHERE run.run_id = $1
|
|
14248
|
+
`,
|
|
14249
|
+
[input.runId, input.attempt],
|
|
14250
|
+
);
|
|
14251
|
+
const row = result.rows[0];
|
|
14252
|
+
if (!row) return null;
|
|
14253
|
+
return {
|
|
14254
|
+
runStatus: row.run_status,
|
|
14255
|
+
attemptStatus: row.attempt_status ?? null,
|
|
14256
|
+
superseded: row.superseded === true,
|
|
14257
|
+
};
|
|
14258
|
+
}
|
|
14259
|
+
|
|
14260
|
+
/**
|
|
14261
|
+
* Register the terminal cleanup obligation before the first external tenant
|
|
14262
|
+
* array-input write. Updating the run row serializes this intent with every
|
|
14263
|
+
* terminal transition, so a terminal that wins first rejects the write, and a
|
|
14264
|
+
* registered write is visible to terminal outbox delivery after any crash.
|
|
14265
|
+
*/
|
|
14266
|
+
export async function registerPostgresSchedulerArrayInputIntent(
|
|
14267
|
+
client: PostgresSchedulerQueryClient,
|
|
14268
|
+
input: { orgId: string; runId: string; attempt: number },
|
|
14269
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
14270
|
+
): Promise<boolean> {
|
|
14271
|
+
const registered = await client.query<{ run_id: string }>(
|
|
14272
|
+
`
|
|
14273
|
+
UPDATE ${tableName('runs', options)} AS run
|
|
14274
|
+
SET array_input_started = true
|
|
14275
|
+
WHERE run.run_id = $1
|
|
14276
|
+
AND run.org_id = $2
|
|
14277
|
+
AND run.status IN ('queued', 'running', 'waiting')
|
|
14278
|
+
AND EXISTS (
|
|
14279
|
+
SELECT 1 FROM ${tableName('attempts', options)} AS current_attempt
|
|
14280
|
+
WHERE current_attempt.run_id = run.run_id
|
|
14281
|
+
AND current_attempt.attempt = $3
|
|
14282
|
+
AND current_attempt.status IN ('claimed', 'running', 'waiting')
|
|
14283
|
+
AND NOT EXISTS (
|
|
14284
|
+
SELECT 1 FROM ${tableName('attempts', options)} AS newer_attempt
|
|
14285
|
+
WHERE newer_attempt.run_id = current_attempt.run_id
|
|
14286
|
+
AND newer_attempt.attempt > current_attempt.attempt
|
|
14287
|
+
)
|
|
14288
|
+
)
|
|
14289
|
+
RETURNING run.run_id
|
|
14290
|
+
`,
|
|
14291
|
+
[input.runId, input.orgId, input.attempt],
|
|
14292
|
+
);
|
|
14293
|
+
return registered.rows.length === 1;
|
|
14294
|
+
}
|
|
14295
|
+
|
|
13485
14296
|
/**
|
|
13486
14297
|
* Push execution (B2-final): the detached runner's liveness/cancellation poll.
|
|
13487
14298
|
* Bumps `heartbeat_at` on the runner's attempt row (the same liveness column
|
|
@@ -13650,10 +14461,16 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13650
14461
|
terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
|
|
13651
14462
|
terminalLogIds?: readonly (string | null)[] | null;
|
|
13652
14463
|
terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
|
|
14464
|
+
/** Canonical runner stats, never Play output metadata. */
|
|
14465
|
+
terminalRuntimeStats?: unknown;
|
|
13653
14466
|
terminalProgressEvents?: readonly unknown[];
|
|
13654
14467
|
},
|
|
13655
14468
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
13656
14469
|
): Promise<boolean> {
|
|
14470
|
+
const terminalPerformanceFacts = terminalRuntimePerformanceFacts(
|
|
14471
|
+
{ stats: input.terminalRuntimeStats },
|
|
14472
|
+
input.attempt,
|
|
14473
|
+
);
|
|
13657
14474
|
const terminalResult = assertTerminalRunResultWithinLimit(input.output);
|
|
13658
14475
|
const ledgerTerminalResult = terminalRunResultForLedger(terminalResult, {
|
|
13659
14476
|
content: 'full',
|
|
@@ -13838,7 +14655,8 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13838
14655
|
updated_at = now()
|
|
13839
14656
|
WHERE run_id IN (SELECT run_id FROM completed)
|
|
13840
14657
|
AND status IN ('queued', 'running', 'waiting')
|
|
13841
|
-
RETURNING run_id
|
|
14658
|
+
RETURNING run_id, launch_json#>>'{contractSnapshot,compatibility,terminalRuntimePerformanceSupported}' = 'true' AS terminal_runtime_performance_supported,
|
|
14659
|
+
launch_json#>>'{contractSnapshot,compatibility,terminalRuntimeBodyAdmissionSupported}' = 'true' AS terminal_runtime_body_admission_supported
|
|
13842
14660
|
),
|
|
13843
14661
|
compute_billing_job AS (
|
|
13844
14662
|
INSERT INTO ${tableName('compute_billing_jobs_v2', options)} (run_id)
|
|
@@ -13877,7 +14695,18 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13877
14695
|
'terminalLogIds', $15::jsonb,
|
|
13878
14696
|
'resultSummary', $10::jsonb,
|
|
13879
14697
|
'terminalWorkCounterSegment', $12::jsonb
|
|
13880
|
-
) || CASE
|
|
14698
|
+
) || CASE WHEN run_terminal.terminal_runtime_performance_supported THEN jsonb_build_object(
|
|
14699
|
+
'terminalRuntimePerformance', CASE
|
|
14700
|
+
WHEN run_terminal.terminal_runtime_body_admission_supported THEN $16::jsonb
|
|
14701
|
+
WHEN jsonb_typeof($16::jsonb->'memoryAdmission') = 'object' THEN
|
|
14702
|
+
CASE WHEN ($16::jsonb->'memoryAdmission') - 'toolResponseBody' = '{}'::jsonb
|
|
14703
|
+
THEN $16::jsonb - 'memoryAdmission'
|
|
14704
|
+
ELSE jsonb_set($16::jsonb, '{memoryAdmission}', ($16::jsonb->'memoryAdmission') - 'toolResponseBody')
|
|
14705
|
+
END
|
|
14706
|
+
ELSE $16::jsonb
|
|
14707
|
+
END,
|
|
14708
|
+
'terminalRuntimePerformanceUnavailableReason', $17::text
|
|
14709
|
+
) ELSE '{}'::jsonb END || CASE
|
|
13881
14710
|
WHEN jsonb_array_length($14::jsonb) > 0
|
|
13882
14711
|
THEN jsonb_build_object('terminalProgressEvents', $14::jsonb)
|
|
13883
14712
|
ELSE '{}'::jsonb
|
|
@@ -13913,6 +14742,8 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13913
14742
|
? terminalLogEntries.map((entry) => entry.entryId)
|
|
13914
14743
|
: null,
|
|
13915
14744
|
),
|
|
14745
|
+
stringifyPostgresJson(terminalPerformanceFacts.performance ?? null),
|
|
14746
|
+
terminalPerformanceFacts.unavailableReason ?? null,
|
|
13916
14747
|
],
|
|
13917
14748
|
);
|
|
13918
14749
|
if (completed.rows.length === 0) {
|
|
@@ -13971,6 +14802,10 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13971
14802
|
input.result,
|
|
13972
14803
|
input.attempt,
|
|
13973
14804
|
);
|
|
14805
|
+
const terminalPerformanceFacts = terminalRuntimePerformanceFacts(
|
|
14806
|
+
input.result,
|
|
14807
|
+
input.attempt,
|
|
14808
|
+
);
|
|
13974
14809
|
// Keep the recovery tail separate from result: large failures are represented
|
|
13975
14810
|
// by a result reference, but their customer guidance must still be logged.
|
|
13976
14811
|
if (
|
|
@@ -14233,7 +15068,8 @@ export async function failPostgresSchedulerAttempt(
|
|
|
14233
15068
|
updated_at = now()
|
|
14234
15069
|
WHERE run_id IN (SELECT run_id FROM failed)
|
|
14235
15070
|
AND status IN ('queued', 'running', 'waiting')
|
|
14236
|
-
RETURNING run_id
|
|
15071
|
+
RETURNING run_id, launch_json#>>'{contractSnapshot,compatibility,terminalRuntimePerformanceSupported}' = 'true' AS terminal_runtime_performance_supported,
|
|
15072
|
+
launch_json#>>'{contractSnapshot,compatibility,terminalRuntimeBodyAdmissionSupported}' = 'true' AS terminal_runtime_body_admission_supported
|
|
14237
15073
|
),
|
|
14238
15074
|
compute_billing_job AS (
|
|
14239
15075
|
INSERT INTO ${tableName('compute_billing_jobs_v2', options)} (run_id)
|
|
@@ -14273,7 +15109,18 @@ export async function failPostgresSchedulerAttempt(
|
|
|
14273
15109
|
'terminalLogIds', $18::jsonb,
|
|
14274
15110
|
'resultSummary', $13::jsonb,
|
|
14275
15111
|
'terminalWorkCounterSegment', $15::jsonb
|
|
14276
|
-
) || CASE
|
|
15112
|
+
) || CASE WHEN run_terminal.terminal_runtime_performance_supported THEN jsonb_build_object(
|
|
15113
|
+
'terminalRuntimePerformance', CASE
|
|
15114
|
+
WHEN run_terminal.terminal_runtime_body_admission_supported THEN $19::jsonb
|
|
15115
|
+
WHEN jsonb_typeof($19::jsonb->'memoryAdmission') = 'object' THEN
|
|
15116
|
+
CASE WHEN ($19::jsonb->'memoryAdmission') - 'toolResponseBody' = '{}'::jsonb
|
|
15117
|
+
THEN $19::jsonb - 'memoryAdmission'
|
|
15118
|
+
ELSE jsonb_set($19::jsonb, '{memoryAdmission}', ($19::jsonb->'memoryAdmission') - 'toolResponseBody')
|
|
15119
|
+
END
|
|
15120
|
+
ELSE $19::jsonb
|
|
15121
|
+
END,
|
|
15122
|
+
'terminalRuntimePerformanceUnavailableReason', $20::text
|
|
15123
|
+
) ELSE '{}'::jsonb END || CASE
|
|
14277
15124
|
WHEN jsonb_array_length($16::jsonb) > 0
|
|
14278
15125
|
THEN jsonb_build_object('terminalProgressEvents', $16::jsonb)
|
|
14279
15126
|
ELSE '{}'::jsonb
|
|
@@ -14312,6 +15159,8 @@ export async function failPostgresSchedulerAttempt(
|
|
|
14312
15159
|
? terminalLogEntries.map((entry) => entry.entryId)
|
|
14313
15160
|
: null,
|
|
14314
15161
|
),
|
|
15162
|
+
stringifyPostgresJson(terminalPerformanceFacts.performance ?? null),
|
|
15163
|
+
terminalPerformanceFacts.unavailableReason ?? null,
|
|
14315
15164
|
],
|
|
14316
15165
|
);
|
|
14317
15166
|
const failed = input.protectWaitingSuspensionFailure
|
|
@@ -14878,10 +15727,16 @@ export async function finalizePostgresSchedulerComputeBilling(
|
|
|
14878
15727
|
'Late runtime resource billing is waiting for provider-confirmed cleanup or its bounded billing ceiling.',
|
|
14879
15728
|
);
|
|
14880
15729
|
}
|
|
14881
|
-
const lateItems =
|
|
14882
|
-
|
|
14883
|
-
|
|
14884
|
-
|
|
15730
|
+
const lateItems =
|
|
15731
|
+
lateResources.length === 0
|
|
15732
|
+
? []
|
|
15733
|
+
: resolveLateRuntimeSandboxComputeItems({
|
|
15734
|
+
resources: lateResources,
|
|
15735
|
+
endedAt: Date.now(),
|
|
15736
|
+
maxBillingDurationSeconds:
|
|
15737
|
+
sandboxRuntimeLimitsFromContractSnapshot(launch.contractSnapshot)
|
|
15738
|
+
.timeoutSeconds + PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
15739
|
+
});
|
|
14885
15740
|
for (const item of lateItems) {
|
|
14886
15741
|
const adjustmentSessionId = `${launch.runId}:late-resource:${createHash(
|
|
14887
15742
|
'sha256',
|
|
@@ -15500,6 +16355,7 @@ export async function executePostgresSchedulerClaim(
|
|
|
15500
16355
|
attempt: claim.attempt,
|
|
15501
16356
|
leaseToken: claim.leaseToken,
|
|
15502
16357
|
output: terminalOutput,
|
|
16358
|
+
terminalRuntimeStats: (output as { stats?: unknown } | null)?.stats,
|
|
15503
16359
|
rowOutcomes: (output as { rowOutcomes?: unknown } | null)?.rowOutcomes,
|
|
15504
16360
|
runtimeTiming: extractRunnerRuntimeTiming(output),
|
|
15505
16361
|
terminalLogLines: terminalLogProjection.lines,
|
|
@@ -16268,6 +17124,7 @@ export async function claimPostgresSchedulerReceipt(
|
|
|
16268
17124
|
input: ClaimRuntimeStepReceiptInput,
|
|
16269
17125
|
options: PostgresSchedulerReceiptOptions,
|
|
16270
17126
|
): Promise<RuntimeStepReceipt | null> {
|
|
17127
|
+
assertWorkReceiptKeyMutable(input.key);
|
|
16271
17128
|
const runAttempt = normalizePostgresSchedulerReceiptRunAttempt(
|
|
16272
17129
|
input.runAttempt,
|
|
16273
17130
|
);
|
|
@@ -16427,6 +17284,7 @@ export async function claimPostgresSchedulerReceipts(
|
|
|
16427
17284
|
input: ClaimRuntimeStepReceiptsInput,
|
|
16428
17285
|
options: PostgresSchedulerReceiptOptions,
|
|
16429
17286
|
): Promise<Array<RuntimeStepReceipt | null>> {
|
|
17287
|
+
for (const key of input.keys) assertWorkReceiptKeyMutable(key);
|
|
16430
17288
|
if (input.keys.length === 0) return [];
|
|
16431
17289
|
const runAttempt = normalizePostgresSchedulerReceiptRunAttempt(
|
|
16432
17290
|
input.runAttempt,
|
|
@@ -16771,6 +17629,7 @@ export async function completePostgresSchedulerReceipt(
|
|
|
16771
17629
|
input: RuntimeStepReceiptCompletionInput & { returnOutput?: boolean },
|
|
16772
17630
|
options: PostgresSchedulerReceiptOptions,
|
|
16773
17631
|
): Promise<RuntimeStepReceipt | null> {
|
|
17632
|
+
assertWorkReceiptKeyMutable(input.key);
|
|
16774
17633
|
if (input.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID) {
|
|
16775
17634
|
const inserted = await client.query<ReceiptRow>(
|
|
16776
17635
|
`
|
|
@@ -17038,6 +17897,8 @@ export async function completePostgresSchedulerReceipts(
|
|
|
17038
17897
|
},
|
|
17039
17898
|
options: PostgresSchedulerReceiptOptions,
|
|
17040
17899
|
): Promise<Array<RuntimeStepReceipt | null>> {
|
|
17900
|
+
for (const receipt of input.receipts)
|
|
17901
|
+
assertWorkReceiptKeyMutable(receipt.key);
|
|
17041
17902
|
if (input.receipts.length === 0) return [];
|
|
17042
17903
|
const cacheInsertIndexes: number[] = [];
|
|
17043
17904
|
const leasedIndexes: number[] = [];
|
|
@@ -17193,6 +18054,7 @@ export async function failPostgresSchedulerReceipt(
|
|
|
17193
18054
|
input: FailRuntimeStepReceiptInput,
|
|
17194
18055
|
options: PostgresSchedulerReceiptOptions,
|
|
17195
18056
|
): Promise<RuntimeStepReceipt | null> {
|
|
18057
|
+
assertWorkReceiptKeyMutable(input.key);
|
|
17196
18058
|
const runAttempt = normalizePostgresSchedulerReceiptRunAttempt(
|
|
17197
18059
|
input.runAttempt,
|
|
17198
18060
|
);
|
|
@@ -17243,6 +18105,8 @@ export async function failPostgresSchedulerReceipts(
|
|
|
17243
18105
|
input: FailRuntimeStepReceiptsInput,
|
|
17244
18106
|
options: PostgresSchedulerReceiptOptions,
|
|
17245
18107
|
): Promise<Array<RuntimeStepReceipt | null>> {
|
|
18108
|
+
for (const receipt of input.receipts)
|
|
18109
|
+
assertWorkReceiptKeyMutable(receipt.key);
|
|
17246
18110
|
if (input.receipts.length === 0) return [];
|
|
17247
18111
|
const normalized = input.receipts.map((receipt) => ({
|
|
17248
18112
|
...receipt,
|
|
@@ -17351,6 +18215,7 @@ export async function releasePostgresSchedulerReceipt(
|
|
|
17351
18215
|
input: ReleaseRuntimeStepReceiptInput,
|
|
17352
18216
|
options: PostgresSchedulerReceiptOptions,
|
|
17353
18217
|
): Promise<RuntimeStepReceipt | null> {
|
|
18218
|
+
assertWorkReceiptKeyMutable(input.key);
|
|
17354
18219
|
const runAttempt = normalizePostgresSchedulerReceiptRunAttempt(
|
|
17355
18220
|
input.runAttempt,
|
|
17356
18221
|
);
|
|
@@ -17437,6 +18302,8 @@ async function markPostgresSchedulerReceiptsStatus(
|
|
|
17437
18302
|
status: 'running' | 'queued',
|
|
17438
18303
|
options: PostgresSchedulerReceiptOptions,
|
|
17439
18304
|
): Promise<Array<RuntimeStepReceipt | null>> {
|
|
18305
|
+
for (const receipt of input.receipts)
|
|
18306
|
+
assertWorkReceiptKeyMutable(receipt.key);
|
|
17440
18307
|
if (input.receipts.length === 0) return [];
|
|
17441
18308
|
const normalized = input.receipts.map((receipt) => ({
|
|
17442
18309
|
...receipt,
|
|
@@ -17543,6 +18410,7 @@ export async function heartbeatPostgresSchedulerReceipts(
|
|
|
17543
18410
|
input: HeartbeatRuntimeStepReceiptsInput,
|
|
17544
18411
|
options: PostgresSchedulerReceiptOptions,
|
|
17545
18412
|
): Promise<Array<RuntimeStepReceipt | null>> {
|
|
18413
|
+
for (const key of input.keys) assertWorkReceiptKeyMutable(key);
|
|
17546
18414
|
if (input.keys.length === 0) return [];
|
|
17547
18415
|
if (
|
|
17548
18416
|
input.leaseIds !== undefined &&
|
|
@@ -17640,6 +18508,7 @@ export async function skipPostgresSchedulerReceipt(
|
|
|
17640
18508
|
input: SkipRuntimeStepReceiptInput,
|
|
17641
18509
|
options: PostgresSchedulerReceiptOptions,
|
|
17642
18510
|
): Promise<RuntimeStepReceipt | null> {
|
|
18511
|
+
assertWorkReceiptKeyMutable(input.key);
|
|
17643
18512
|
const runAttempt = normalizePostgresSchedulerReceiptRunAttempt(
|
|
17644
18513
|
input.runAttempt,
|
|
17645
18514
|
);
|
|
@@ -18113,6 +18982,7 @@ export async function readPostgresSchedulerRunStatus(
|
|
|
18113
18982
|
| 'run_id'
|
|
18114
18983
|
| 'org_id'
|
|
18115
18984
|
| 'status'
|
|
18985
|
+
| 'run_log_store'
|
|
18116
18986
|
| 'play_name'
|
|
18117
18987
|
| 'graph_hash'
|
|
18118
18988
|
| 'runtime_profile'
|
|
@@ -18126,6 +18996,7 @@ export async function readPostgresSchedulerRunStatus(
|
|
|
18126
18996
|
>(
|
|
18127
18997
|
`
|
|
18128
18998
|
SELECT scheduler_run.run_id, scheduler_run.org_id, scheduler_run.status,
|
|
18999
|
+
scheduler_run.run_log_store,
|
|
18129
19000
|
scheduler_run.play_name, scheduler_run.graph_hash,
|
|
18130
19001
|
scheduler_run.runtime_profile, scheduler_run.created_at,
|
|
18131
19002
|
scheduler_run.updated_at, scheduler_run.started_at,
|
|
@@ -18146,6 +19017,10 @@ export async function readPostgresSchedulerRunStatus(
|
|
|
18146
19017
|
? { definitionScope: row.definition_scope }
|
|
18147
19018
|
: {}),
|
|
18148
19019
|
status: row.status,
|
|
19020
|
+
...(row.run_log_store === 'planet_scale'
|
|
19021
|
+
? { runLogStore: 'planet_scale' as const }
|
|
19022
|
+
: {}),
|
|
19023
|
+
schedulerSchema: schemaName(options),
|
|
18149
19024
|
playName: row.play_name,
|
|
18150
19025
|
graphHash: row.graph_hash,
|
|
18151
19026
|
runtimeBackend: null,
|
|
@@ -18172,6 +19047,7 @@ export async function readPostgresSchedulerRunStatus(
|
|
|
18172
19047
|
const result = await client.query<RunStatusRow>(
|
|
18173
19048
|
`
|
|
18174
19049
|
SELECT scheduler_run.run_id, scheduler_run.org_id, scheduler_run.status,
|
|
19050
|
+
scheduler_run.run_log_store,
|
|
18175
19051
|
scheduler_run.play_name, scheduler_run.graph_hash,
|
|
18176
19052
|
scheduler_run.runtime_profile, scheduler_run.launch_json,
|
|
18177
19053
|
scheduler_run.suspension_json, scheduler_run.terminal_result_json,
|
|
@@ -18200,6 +19076,10 @@ export async function readPostgresSchedulerRunStatus(
|
|
|
18200
19076
|
? { definitionScope: row.definition_scope }
|
|
18201
19077
|
: {}),
|
|
18202
19078
|
status: row.status,
|
|
19079
|
+
...(row.run_log_store === 'planet_scale'
|
|
19080
|
+
? { runLogStore: 'planet_scale' as const }
|
|
19081
|
+
: {}),
|
|
19082
|
+
schedulerSchema: schemaName(options),
|
|
18203
19083
|
playName: row.play_name,
|
|
18204
19084
|
graphHash: row.graph_hash,
|
|
18205
19085
|
runtimeBackend: launch.runtimeBackend || null,
|
|
@@ -18468,7 +19348,52 @@ function outboxNotFailedSql(alias: string): string {
|
|
|
18468
19348
|
// Share that existing boundary between its SQL snapshot and prefix validation.
|
|
18469
19349
|
const RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS = 8;
|
|
18470
19350
|
|
|
18471
|
-
/**
|
|
19351
|
+
/** Read only after the caller proved its terminal outbox delivery lease. */
|
|
19352
|
+
export async function readPostgresSchedulerTerminalArrayInputIntent(
|
|
19353
|
+
client: PostgresSchedulerQueryClient,
|
|
19354
|
+
input: { orgId: string; runId: string },
|
|
19355
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19356
|
+
): Promise<{ playName: string | null; started: boolean }> {
|
|
19357
|
+
return readTerminalArrayInputIntentForScope(client, input, options);
|
|
19358
|
+
}
|
|
19359
|
+
|
|
19360
|
+
/** Internal dispatch already owns the run's outbox lease; receiver reads also bind its org. */
|
|
19361
|
+
async function readTerminalArrayInputIntentForScope(
|
|
19362
|
+
client: PostgresSchedulerQueryClient,
|
|
19363
|
+
input: { orgId?: string; runId: string },
|
|
19364
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19365
|
+
): Promise<{ playName: string | null; started: boolean }> {
|
|
19366
|
+
// Historical scheduler schemas have no marker. Inspect only a terminal
|
|
19367
|
+
// delivery, then read the scalar column directly when it exists. Converting
|
|
19368
|
+
// the whole wide run tuple to JSON for every outbox fact would copy launch,
|
|
19369
|
+
// input, and checkpoint payloads just to access this one bit.
|
|
19370
|
+
const column = await client.query<{ present: boolean }>(
|
|
19371
|
+
`SELECT EXISTS (
|
|
19372
|
+
SELECT 1 FROM pg_attribute
|
|
19373
|
+
WHERE attrelid = to_regclass($1)
|
|
19374
|
+
AND attname = 'array_input_started'
|
|
19375
|
+
AND NOT attisdropped
|
|
19376
|
+
) AS present`,
|
|
19377
|
+
[tableName('runs', options)],
|
|
19378
|
+
);
|
|
19379
|
+
if (column.rows[0]?.present !== true) {
|
|
19380
|
+
return { playName: null, started: false };
|
|
19381
|
+
}
|
|
19382
|
+
const run = await client.query<{
|
|
19383
|
+
array_input_started: boolean;
|
|
19384
|
+
play_name: string;
|
|
19385
|
+
}>(
|
|
19386
|
+
`SELECT array_input_started, play_name FROM ${tableName('runs', options)}
|
|
19387
|
+
WHERE run_id = $1 ${input.orgId === undefined ? '' : 'AND org_id = $2'}
|
|
19388
|
+
AND status IN ('completed', 'failed', 'cancelled')`,
|
|
19389
|
+
input.orgId === undefined ? [input.runId] : [input.runId, input.orgId],
|
|
19390
|
+
);
|
|
19391
|
+
return {
|
|
19392
|
+
playName: run.rows[0]?.play_name ?? null,
|
|
19393
|
+
started: run.rows[0]?.array_input_started === true,
|
|
19394
|
+
};
|
|
19395
|
+
}
|
|
19396
|
+
|
|
18472
19397
|
export async function assertPostgresRunLedgerDelivery(
|
|
18473
19398
|
client: PostgresSchedulerQueryClient,
|
|
18474
19399
|
input: {
|
|
@@ -18630,6 +19555,15 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
18630
19555
|
runEventsForAppRuntimeProjectionCandidates(
|
|
18631
19556
|
prefixEvents,
|
|
18632
19557
|
input.projectionChunk,
|
|
19558
|
+
input.projectedEvents.every(
|
|
19559
|
+
(event) =>
|
|
19560
|
+
!Object.hasOwn(event, 'terminalRuntimePerformance') &&
|
|
19561
|
+
!Object.hasOwn(
|
|
19562
|
+
event,
|
|
19563
|
+
'terminalRuntimePerformanceUnavailableReason',
|
|
19564
|
+
),
|
|
19565
|
+
),
|
|
19566
|
+
input.projectedEvents,
|
|
18633
19567
|
).map((candidate) =>
|
|
18634
19568
|
candidate === null
|
|
18635
19569
|
? null
|
|
@@ -18722,6 +19656,105 @@ function runLedgerPendingRunsSql(
|
|
|
18722
19656
|
)`;
|
|
18723
19657
|
}
|
|
18724
19658
|
|
|
19659
|
+
/** The causal first head is chosen before eligibility filtering. */
|
|
19660
|
+
function runLedgerDiscoveryHeadsSql(
|
|
19661
|
+
options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
|
|
19662
|
+
searchOnly: boolean,
|
|
19663
|
+
includePayload = false,
|
|
19664
|
+
): string {
|
|
19665
|
+
return `WITH RECURSIVE ${runLedgerPendingRunsSql(options, searchOnly)}, heads AS MATERIALIZED (
|
|
19666
|
+
SELECT pending_run.run_id, head.*
|
|
19667
|
+
FROM pending_runs AS pending_run
|
|
19668
|
+
CROSS JOIN LATERAL (
|
|
19669
|
+
SELECT pending.event_id,
|
|
19670
|
+
pending.event_type,
|
|
19671
|
+
${includePayload ? 'pending.payload_json,' : ''}
|
|
19672
|
+
pending.created_at,
|
|
19673
|
+
pending.next_attempt_at,
|
|
19674
|
+
pending.failed_at
|
|
19675
|
+
FROM ${tableName('outbox', options)} AS pending
|
|
19676
|
+
WHERE pending.run_id = pending_run.run_id
|
|
19677
|
+
AND pending.delivered_at IS NULL
|
|
19678
|
+
AND ${searchLaneSql('pending')} = ${searchOnly ? 'true' : 'false'}
|
|
19679
|
+
ORDER BY ${runLedgerOrderSql('pending')}
|
|
19680
|
+
LIMIT 1
|
|
19681
|
+
) AS head
|
|
19682
|
+
)`;
|
|
19683
|
+
}
|
|
19684
|
+
|
|
19685
|
+
function runLedgerReadyHeadSql(
|
|
19686
|
+
options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
|
|
19687
|
+
searchOnly: boolean,
|
|
19688
|
+
): string {
|
|
19689
|
+
return `head.failed_at IS NULL
|
|
19690
|
+
AND coalesce(head.next_attempt_at, head.created_at) <= now()
|
|
19691
|
+
AND NOT EXISTS (
|
|
19692
|
+
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
19693
|
+
WHERE active.run_id = runs.run_id
|
|
19694
|
+
AND ${searchLaneSql('active')} = ${String(searchOnly)}
|
|
19695
|
+
AND active.delivered_at IS NULL
|
|
19696
|
+
AND active.lease_owner IS NOT NULL
|
|
19697
|
+
AND active.lease_expires_at > now()
|
|
19698
|
+
)`;
|
|
19699
|
+
}
|
|
19700
|
+
|
|
19701
|
+
function runLedgerAdmissionOrderSql(
|
|
19702
|
+
options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
|
|
19703
|
+
searchOnly: boolean,
|
|
19704
|
+
): string {
|
|
19705
|
+
return `CASE
|
|
19706
|
+
WHEN head.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
|
|
19707
|
+
WHEN ${runLedgerTerminalPendingSql('head', options)} THEN 1
|
|
19708
|
+
${searchOnly ? '' : `WHEN ${runLedgerWaitingPendingSql('head', options)} THEN 2`}
|
|
19709
|
+
ELSE 3
|
|
19710
|
+
END,`;
|
|
19711
|
+
}
|
|
19712
|
+
|
|
19713
|
+
export type RunLedgerOutboxAdmissionHint = {
|
|
19714
|
+
origin: 'admission_snapshot';
|
|
19715
|
+
runId: string;
|
|
19716
|
+
};
|
|
19717
|
+
|
|
19718
|
+
/**
|
|
19719
|
+
* Read a small ordered, discardable window; it reserves no run or delivery.
|
|
19720
|
+
* Every hinted claim still uses the canonical run-scoped locking/lease path.
|
|
19721
|
+
* This is only used after productive delivery, leaving idle probes unchanged.
|
|
19722
|
+
*/
|
|
19723
|
+
export async function readPostgresSchedulerOutboxAdmissionHints(
|
|
19724
|
+
client: PostgresSchedulerQueryClient,
|
|
19725
|
+
input: { limit: number; excludeRunIds: readonly string[] },
|
|
19726
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19727
|
+
): Promise<RunLedgerOutboxAdmissionHint[]> {
|
|
19728
|
+
if (
|
|
19729
|
+
!Number.isInteger(input.limit) ||
|
|
19730
|
+
input.limit < 1 ||
|
|
19731
|
+
input.limit > 3 ||
|
|
19732
|
+
input.excludeRunIds.length > 32 ||
|
|
19733
|
+
input.excludeRunIds.some((runId) => !runId || runId.trim() !== runId) ||
|
|
19734
|
+
new Set(input.excludeRunIds).size !== input.excludeRunIds.length
|
|
19735
|
+
)
|
|
19736
|
+
throw new Error(
|
|
19737
|
+
'Outbox admission discovery requires1–3 hints and at most32 distinct in-flight run ids.',
|
|
19738
|
+
);
|
|
19739
|
+
const result = await client.query<{ run_id: string }>(
|
|
19740
|
+
`
|
|
19741
|
+
${runLedgerDiscoveryHeadsSql(options, false)}
|
|
19742
|
+
SELECT runs.run_id
|
|
19743
|
+
FROM heads AS head
|
|
19744
|
+
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
19745
|
+
WHERE ${runLedgerReadyHeadSql(options, false)}
|
|
19746
|
+
AND NOT (runs.run_id = ANY($2::text[]))
|
|
19747
|
+
ORDER BY ${runLedgerAdmissionOrderSql(options, false)} head.created_at, runs.run_id
|
|
19748
|
+
LIMIT $1
|
|
19749
|
+
`,
|
|
19750
|
+
[input.limit, input.excludeRunIds],
|
|
19751
|
+
);
|
|
19752
|
+
return result.rows.map<RunLedgerOutboxAdmissionHint>(({ run_id }) => ({
|
|
19753
|
+
origin: 'admission_snapshot',
|
|
19754
|
+
runId: run_id,
|
|
19755
|
+
}));
|
|
19756
|
+
}
|
|
19757
|
+
|
|
18725
19758
|
/** Prioritize a terminal's causal head without skipping its predecessors. */
|
|
18726
19759
|
function runLedgerTerminalPendingSql(
|
|
18727
19760
|
alias: string,
|
|
@@ -18832,6 +19865,7 @@ type RunLedgerOutboxPrefixClaimRow = {
|
|
|
18832
19865
|
| 'delivered'
|
|
18833
19866
|
| 'transport_delivered'
|
|
18834
19867
|
| 'transport_unconfirmed';
|
|
19868
|
+
array_input_started?: boolean;
|
|
18835
19869
|
event_id: string;
|
|
18836
19870
|
run_id: string;
|
|
18837
19871
|
event_type: string;
|
|
@@ -19369,6 +20403,7 @@ function runLedgerOutboxPrefixClaimBatchSql(input: {
|
|
|
19369
20403
|
waitForRunLock: boolean;
|
|
19370
20404
|
lockStrength?: 'no_key_update';
|
|
19371
20405
|
transportHeadSql?: string;
|
|
20406
|
+
readArrayInputIntent?: boolean;
|
|
19372
20407
|
}): string {
|
|
19373
20408
|
// Ordinary claims have four bounded primitive arguments. Give PostgreSQL
|
|
19374
20409
|
// constants instead of a scalar InitPlan for every repeated argument. Keep
|
|
@@ -19424,10 +20459,19 @@ function runLedgerOutboxPrefixClaimBatchSql(input: {
|
|
|
19424
20459
|
return `BEGIN;
|
|
19425
20460
|
SELECT set_config('lock_timeout', '${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms', true),
|
|
19426
20461
|
set_config('statement_timeout', '${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms', true);
|
|
19427
|
-
|
|
20462
|
+
${
|
|
20463
|
+
input.readArrayInputIntent
|
|
20464
|
+
? `WITH locked_run AS MATERIALIZED (
|
|
20465
|
+
SELECT run_id, array_input_started FROM ${tableName('runs', { schema: input.schema })}
|
|
20466
|
+
WHERE run_id = ${escapeLiteral(input.runId)} FOR ${input.lockStrength === 'no_key_update' ? 'NO KEY UPDATE' : 'UPDATE'}${input.waitForRunLock ? '' : ' SKIP LOCKED'}
|
|
20467
|
+
)
|
|
20468
|
+
SELECT set_config('deepline.run_ledger_outbox_lock', coalesce((SELECT run_id FROM locked_run), ''), true),
|
|
20469
|
+
(SELECT array_input_started FROM locked_run) AS array_input_started;`
|
|
20470
|
+
: `SELECT set_config('deepline.run_ledger_outbox_lock', coalesce((
|
|
19428
20471
|
SELECT run_id FROM ${tableName('runs', { schema: input.schema })}
|
|
19429
20472
|
WHERE run_id = ${escapeLiteral(input.runId)} FOR ${input.lockStrength === 'no_key_update' ? 'NO KEY UPDATE' : 'UPDATE'}${input.waitForRunLock ? '' : ' SKIP LOCKED'}
|
|
19430
|
-
), ''), true)
|
|
20473
|
+
), ''), true);`
|
|
20474
|
+
}
|
|
19431
20475
|
${input.transportHeadSql ? `SELECT set_config('deepline.run_ledger_transport_ack_count', '0', true);\n ${input.transportHeadSql};` : ''}
|
|
19432
20476
|
${claimSql};
|
|
19433
20477
|
COMMIT;`;
|
|
@@ -19451,6 +20495,7 @@ async function claimRunLedgerOutboxPrefixInBatch(
|
|
|
19451
20495
|
/** Discover the zero-effect transport head under this transaction's run lock. */
|
|
19452
20496
|
discoverTransportHead?: boolean;
|
|
19453
20497
|
recoverTransportDelivery?: boolean;
|
|
20498
|
+
readArrayInputIntent?: boolean;
|
|
19454
20499
|
},
|
|
19455
20500
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19456
20501
|
acknowledgement?: {
|
|
@@ -19464,6 +20509,7 @@ async function claimRunLedgerOutboxPrefixInBatch(
|
|
|
19464
20509
|
freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
|
|
19465
20510
|
transportDelivered?: RunLedgerTransportDelivery;
|
|
19466
20511
|
transportOwnershipUnconfirmed?: boolean;
|
|
20512
|
+
arrayInputStarted?: boolean;
|
|
19467
20513
|
} | null> {
|
|
19468
20514
|
if (
|
|
19469
20515
|
runLedgerOutboxBatchUnsupportedClients.has(client) ||
|
|
@@ -19653,6 +20699,7 @@ async function claimRunLedgerOutboxPrefixInBatch(
|
|
|
19653
20699
|
lockStrength: input.lockStrength,
|
|
19654
20700
|
schema: options?.schema,
|
|
19655
20701
|
transportHeadSql,
|
|
20702
|
+
readArrayInputIntent: input.readArrayInputIntent !== false,
|
|
19656
20703
|
}),
|
|
19657
20704
|
);
|
|
19658
20705
|
} catch (error) {
|
|
@@ -19671,6 +20718,26 @@ async function claimRunLedgerOutboxPrefixInBatch(
|
|
|
19671
20718
|
});
|
|
19672
20719
|
throw error;
|
|
19673
20720
|
}
|
|
20721
|
+
if (
|
|
20722
|
+
input.readArrayInputIntent !== false &&
|
|
20723
|
+
error !== null &&
|
|
20724
|
+
typeof error === 'object' &&
|
|
20725
|
+
'code' in error &&
|
|
20726
|
+
error.code === '42703' &&
|
|
20727
|
+
'message' in error &&
|
|
20728
|
+
typeof error.message === 'string' &&
|
|
20729
|
+
/\barray_input_started\b/.test(error.message)
|
|
20730
|
+
) {
|
|
20731
|
+
// Retry only this absent optional column after the owned transaction
|
|
20732
|
+
// rolled back. Do not cache absence across a concurrent schema expansion;
|
|
20733
|
+
// terminal delivery will use the historical read path for this claim.
|
|
20734
|
+
return claimRunLedgerOutboxPrefixInBatch(
|
|
20735
|
+
client,
|
|
20736
|
+
{ ...input, readArrayInputIntent: false },
|
|
20737
|
+
options,
|
|
20738
|
+
acknowledgement,
|
|
20739
|
+
);
|
|
20740
|
+
}
|
|
19674
20741
|
if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
|
|
19675
20742
|
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
19676
20743
|
return null;
|
|
@@ -19682,18 +20749,22 @@ async function claimRunLedgerOutboxPrefixInBatch(
|
|
|
19682
20749
|
)
|
|
19683
20750
|
? result
|
|
19684
20751
|
: [result];
|
|
19685
|
-
|
|
19686
|
-
|
|
19687
|
-
|
|
19688
|
-
.filter(
|
|
20752
|
+
const rows = results.flatMap((entry) => entry.rows);
|
|
20753
|
+
return {
|
|
20754
|
+
...projectRunLedgerOutboxPrefixRows(
|
|
20755
|
+
rows.filter(
|
|
19689
20756
|
(row) =>
|
|
19690
20757
|
row.row_kind === 'claimed' ||
|
|
19691
20758
|
row.row_kind === 'delivered' ||
|
|
19692
20759
|
row.row_kind === 'transport_delivered' ||
|
|
19693
20760
|
row.row_kind === 'transport_unconfirmed',
|
|
19694
20761
|
),
|
|
19695
|
-
|
|
19696
|
-
|
|
20762
|
+
leaseOwner,
|
|
20763
|
+
),
|
|
20764
|
+
arrayInputStarted: rows.find(
|
|
20765
|
+
(row) => typeof row.array_input_started === 'boolean',
|
|
20766
|
+
)?.array_input_started,
|
|
20767
|
+
};
|
|
19697
20768
|
}
|
|
19698
20769
|
|
|
19699
20770
|
async function claimPostgresSchedulerOutboxPrefix(
|
|
@@ -20677,6 +21748,30 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
20677
21748
|
normalizePlayRunLedgerTerminalWorkCounterSegment(
|
|
20678
21749
|
payloadField(event.payload, 'terminalWorkCounterSegment'),
|
|
20679
21750
|
);
|
|
21751
|
+
const rawPerformance = payloadField(
|
|
21752
|
+
event.payload,
|
|
21753
|
+
'terminalRuntimePerformance',
|
|
21754
|
+
);
|
|
21755
|
+
const terminalRuntimePerformance =
|
|
21756
|
+
payloadField(event.payload, 'terminalWorkCounterSegment') == null ||
|
|
21757
|
+
terminalWorkCounterSegment
|
|
21758
|
+
? normalizeTerminalRuntimePerformance(
|
|
21759
|
+
rawPerformance,
|
|
21760
|
+
terminalWorkCounterSegment,
|
|
21761
|
+
)
|
|
21762
|
+
: undefined;
|
|
21763
|
+
const terminalRuntimePerformanceUnavailableReason =
|
|
21764
|
+
!terminalRuntimePerformance
|
|
21765
|
+
? (normalizeTerminalRuntimePerformanceUnavailableReason(
|
|
21766
|
+
payloadField(
|
|
21767
|
+
event.payload,
|
|
21768
|
+
'terminalRuntimePerformanceUnavailableReason',
|
|
21769
|
+
),
|
|
21770
|
+
) ??
|
|
21771
|
+
(rawPerformance != null
|
|
21772
|
+
? ('invalid_terminal_performance' as const)
|
|
21773
|
+
: undefined))
|
|
21774
|
+
: undefined;
|
|
20680
21775
|
const progressEvents = ledgerEventsPayloadField(
|
|
20681
21776
|
event.payload,
|
|
20682
21777
|
'terminalProgressEvents',
|
|
@@ -20707,6 +21802,10 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
20707
21802
|
occurredAt,
|
|
20708
21803
|
source: 'system',
|
|
20709
21804
|
...(terminalWorkCounterSegment ? { terminalWorkCounterSegment } : {}),
|
|
21805
|
+
...(terminalRuntimePerformance ? { terminalRuntimePerformance } : {}),
|
|
21806
|
+
...(terminalRuntimePerformanceUnavailableReason
|
|
21807
|
+
? { terminalRuntimePerformanceUnavailableReason }
|
|
21808
|
+
: {}),
|
|
20710
21809
|
result: terminalRunResultForLedger(
|
|
20711
21810
|
payloadField(event.payload, 'output'),
|
|
20712
21811
|
),
|
|
@@ -20721,6 +21820,30 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
20721
21820
|
normalizePlayRunLedgerTerminalWorkCounterSegment(
|
|
20722
21821
|
payloadField(event.payload, 'terminalWorkCounterSegment'),
|
|
20723
21822
|
);
|
|
21823
|
+
const rawPerformance = payloadField(
|
|
21824
|
+
event.payload,
|
|
21825
|
+
'terminalRuntimePerformance',
|
|
21826
|
+
);
|
|
21827
|
+
const terminalRuntimePerformance =
|
|
21828
|
+
payloadField(event.payload, 'terminalWorkCounterSegment') == null ||
|
|
21829
|
+
terminalWorkCounterSegment
|
|
21830
|
+
? normalizeTerminalRuntimePerformance(
|
|
21831
|
+
rawPerformance,
|
|
21832
|
+
terminalWorkCounterSegment,
|
|
21833
|
+
)
|
|
21834
|
+
: undefined;
|
|
21835
|
+
const terminalRuntimePerformanceUnavailableReason =
|
|
21836
|
+
!terminalRuntimePerformance
|
|
21837
|
+
? (normalizeTerminalRuntimePerformanceUnavailableReason(
|
|
21838
|
+
payloadField(
|
|
21839
|
+
event.payload,
|
|
21840
|
+
'terminalRuntimePerformanceUnavailableReason',
|
|
21841
|
+
),
|
|
21842
|
+
) ??
|
|
21843
|
+
(rawPerformance != null
|
|
21844
|
+
? ('invalid_terminal_performance' as const)
|
|
21845
|
+
: undefined))
|
|
21846
|
+
: undefined;
|
|
20724
21847
|
const progressEvents = ledgerEventsPayloadField(
|
|
20725
21848
|
event.payload,
|
|
20726
21849
|
'terminalProgressEvents',
|
|
@@ -20754,6 +21877,10 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
20754
21877
|
occurredAt,
|
|
20755
21878
|
source: 'system',
|
|
20756
21879
|
...(terminalWorkCounterSegment ? { terminalWorkCounterSegment } : {}),
|
|
21880
|
+
...(terminalRuntimePerformance ? { terminalRuntimePerformance } : {}),
|
|
21881
|
+
...(terminalRuntimePerformanceUnavailableReason
|
|
21882
|
+
? { terminalRuntimePerformanceUnavailableReason }
|
|
21883
|
+
: {}),
|
|
20757
21884
|
error: stringPayloadField(event.payload, 'error'),
|
|
20758
21885
|
...(result === null || result === undefined ? {} : { result }),
|
|
20759
21886
|
...(resultSummary === undefined || resultSummary === null
|
|
@@ -21230,6 +22357,7 @@ type RunLedgerOutboxDrainScope = {
|
|
|
21230
22357
|
leaseOwner: string;
|
|
21231
22358
|
targetEventId: string;
|
|
21232
22359
|
nextClaimed: PostgresSchedulerOutboxEvent[];
|
|
22360
|
+
nextArrayInputStarted?: boolean;
|
|
21233
22361
|
};
|
|
21234
22362
|
|
|
21235
22363
|
type RunLedgerOutboxAuthorityScope = {
|
|
@@ -21563,37 +22691,11 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21563
22691
|
dispatch_launch_json: unknown;
|
|
21564
22692
|
transport_head?: RunLedgerTransportCandidate | null;
|
|
21565
22693
|
}>(`
|
|
21566
|
-
|
|
21567
|
-
SELECT pending_run.run_id, head.*
|
|
21568
|
-
FROM pending_runs AS pending_run
|
|
21569
|
-
CROSS JOIN LATERAL (
|
|
21570
|
-
SELECT pending.event_id,
|
|
21571
|
-
pending.event_type,
|
|
21572
|
-
${needsLaunch ? 'pending.payload_json,' : ''}
|
|
21573
|
-
pending.created_at,
|
|
21574
|
-
pending.next_attempt_at,
|
|
21575
|
-
pending.failed_at
|
|
21576
|
-
FROM ${tableName('outbox', options)} AS pending
|
|
21577
|
-
WHERE pending.run_id = pending_run.run_id
|
|
21578
|
-
AND pending.delivered_at IS NULL
|
|
21579
|
-
AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
21580
|
-
ORDER BY ${runLedgerOrderSql('pending')}
|
|
21581
|
-
LIMIT 1
|
|
21582
|
-
) AS head
|
|
21583
|
-
)
|
|
22694
|
+
${runLedgerDiscoveryHeadsSql(options, input.searchOnly === true, needsLaunch)}
|
|
21584
22695
|
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json, ${transportHeadSelection} AS transport_head
|
|
21585
22696
|
FROM heads AS head
|
|
21586
22697
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
21587
|
-
WHERE
|
|
21588
|
-
AND coalesce(head.next_attempt_at, head.created_at) <= now()
|
|
21589
|
-
AND NOT EXISTS (
|
|
21590
|
-
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
21591
|
-
WHERE active.run_id = runs.run_id
|
|
21592
|
-
AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
21593
|
-
AND active.delivered_at IS NULL
|
|
21594
|
-
AND active.lease_owner IS NOT NULL
|
|
21595
|
-
AND active.lease_expires_at > now()
|
|
21596
|
-
)
|
|
22698
|
+
WHERE ${runLedgerReadyHeadSql(options, input.searchOnly === true)}
|
|
21597
22699
|
-- admission_first: admission, then the causal head of a run with
|
|
21598
22700
|
-- a pending terminal or public wait, then FIFO progress. oldest_ready keeps its
|
|
21599
22701
|
-- pure age order so continuous admissions/terminals cannot starve
|
|
@@ -21601,12 +22703,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21601
22703
|
ORDER BY ${
|
|
21602
22704
|
input.headPriority === 'oldest_ready'
|
|
21603
22705
|
? ''
|
|
21604
|
-
:
|
|
21605
|
-
WHEN head.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
|
|
21606
|
-
WHEN ${runLedgerTerminalPendingSql('head', options)} THEN 1
|
|
21607
|
-
${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('head', options)} THEN 2`}
|
|
21608
|
-
ELSE 3
|
|
21609
|
-
END,`
|
|
22706
|
+
: runLedgerAdmissionOrderSql(options, input.searchOnly === true)
|
|
21610
22707
|
}
|
|
21611
22708
|
head.created_at, runs.run_id
|
|
21612
22709
|
FOR UPDATE OF runs SKIP LOCKED LIMIT 1
|
|
@@ -21732,6 +22829,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21732
22829
|
freshClaimLease: result.freshClaimLease,
|
|
21733
22830
|
transportDelivered: result.transportDelivered,
|
|
21734
22831
|
transportOwnershipUnconfirmed: result.transportOwnershipUnconfirmed,
|
|
22832
|
+
arrayInputStarted: result.arrayInputStarted,
|
|
21735
22833
|
};
|
|
21736
22834
|
};
|
|
21737
22835
|
const claimEvents = async (client: PostgresSchedulerQueryClient) => {
|
|
@@ -21765,6 +22863,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21765
22863
|
freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
|
|
21766
22864
|
transportDelivered?: RunLedgerTransportDelivery;
|
|
21767
22865
|
transportOwnershipUnconfirmed?: boolean;
|
|
22866
|
+
arrayInputStarted?: boolean;
|
|
21768
22867
|
} = { events: [], launchJson: undefined, ingressPrefixClaimed: false };
|
|
21769
22868
|
return unclaimed;
|
|
21770
22869
|
}
|
|
@@ -21842,6 +22941,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21842
22941
|
freshClaimLease: claimed.freshClaimLease,
|
|
21843
22942
|
transportDelivered: undefined,
|
|
21844
22943
|
transportOwnershipUnconfirmed: undefined,
|
|
22944
|
+
arrayInputStarted: undefined,
|
|
21845
22945
|
};
|
|
21846
22946
|
} catch (error) {
|
|
21847
22947
|
await client.query('ROLLBACK');
|
|
@@ -21851,6 +22951,9 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21851
22951
|
const preclaimed = scope?.nextClaimed.length
|
|
21852
22952
|
? scope.nextClaimed.splice(0)
|
|
21853
22953
|
: null;
|
|
22954
|
+
let terminalArrayProjectionOrigin: string | null | undefined;
|
|
22955
|
+
const terminalArrayProjectionOriginFailure: { current?: { error: unknown } } =
|
|
22956
|
+
{};
|
|
21854
22957
|
const claimStartedAtMonotonicMs = performance.now();
|
|
21855
22958
|
const claimed = preclaimed
|
|
21856
22959
|
? {
|
|
@@ -21860,18 +22963,46 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21860
22963
|
freshClaimLease: undefined,
|
|
21861
22964
|
transportDelivered: undefined,
|
|
21862
22965
|
transportOwnershipUnconfirmed: undefined,
|
|
22966
|
+
arrayInputStarted: scope?.nextArrayInputStarted,
|
|
21863
22967
|
}
|
|
21864
22968
|
: await withClient(async (client) => {
|
|
21865
22969
|
const fast = await claimFastEvents(client);
|
|
21866
|
-
|
|
21867
|
-
|
|
21868
|
-
|
|
21869
|
-
}
|
|
21870
|
-
|
|
21871
|
-
|
|
21872
|
-
|
|
21873
|
-
|
|
22970
|
+
let result: Awaited<ReturnType<typeof claimEvents>>;
|
|
22971
|
+
if (fast !== null) {
|
|
22972
|
+
result = fast;
|
|
22973
|
+
} else {
|
|
22974
|
+
try {
|
|
22975
|
+
result = await claimEvents(client);
|
|
22976
|
+
} catch (error) {
|
|
22977
|
+
if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
|
|
22978
|
+
// claimEvents rolled back. Reacquire the run fence and its fresh
|
|
22979
|
+
// snapshot using only the named missing-column compatibility path.
|
|
22980
|
+
result = await claimEvents(client);
|
|
22981
|
+
}
|
|
22982
|
+
}
|
|
22983
|
+
if (
|
|
22984
|
+
result.events.some((event) =>
|
|
22985
|
+
RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(event.eventType),
|
|
22986
|
+
)
|
|
22987
|
+
) {
|
|
22988
|
+
// Reuse the claim checkout. Ordinary terminals must not acquire an
|
|
22989
|
+
// extra scarce client merely to prove that no array was staged.
|
|
22990
|
+
try {
|
|
22991
|
+
terminalArrayProjectionOrigin =
|
|
22992
|
+
await readTerminalArrayProjectionOrigin(
|
|
22993
|
+
(operation) => operation(client),
|
|
22994
|
+
result.launchJson,
|
|
22995
|
+
result.events[0]!.runId,
|
|
22996
|
+
options,
|
|
22997
|
+
result.arrayInputStarted,
|
|
22998
|
+
);
|
|
22999
|
+
} catch (error) {
|
|
23000
|
+
// The claim committed. Settle a routing failure through the same
|
|
23001
|
+
// fenced delivery policy below instead of stranding its lease.
|
|
23002
|
+
terminalArrayProjectionOriginFailure.current = { error };
|
|
23003
|
+
}
|
|
21874
23004
|
}
|
|
23005
|
+
return result;
|
|
21875
23006
|
});
|
|
21876
23007
|
const claimReceivedAtMonotonicMs = performance.now();
|
|
21877
23008
|
// Debit the complete admission/claim/COMMIT round trip, even though SQL's
|
|
@@ -22182,6 +23313,9 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
22182
23313
|
// any authority or renewal work. The abort branch below hands the lease
|
|
22183
23314
|
// straight back without charging a projection attempt.
|
|
22184
23315
|
input.signal?.throwIfAborted();
|
|
23316
|
+
if (terminalArrayProjectionOriginFailure.current) {
|
|
23317
|
+
throw terminalArrayProjectionOriginFailure.current.error;
|
|
23318
|
+
}
|
|
22185
23319
|
if (unconfirmedTransportError) throw unconfirmedTransportError;
|
|
22186
23320
|
// The ledger converter rejects raw deferred facts. Inline callers hand
|
|
22187
23321
|
// them off; the background owner projects them before inspecting the
|
|
@@ -22251,6 +23385,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
22251
23385
|
)
|
|
22252
23386
|
? null
|
|
22253
23387
|
: events.flatMap(buildRunLedgerEventsFromPostgresSchedulerOutbox);
|
|
23388
|
+
const terminalProjection =
|
|
23389
|
+
events.some((event) =>
|
|
23390
|
+
RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(event.eventType),
|
|
23391
|
+
) ||
|
|
23392
|
+
ledgerEvents?.some(
|
|
23393
|
+
(event) =>
|
|
23394
|
+
event.type === RUN_EVENT_KIND.runCompleted ||
|
|
23395
|
+
event.type === RUN_EVENT_KIND.runFailed ||
|
|
23396
|
+
event.type === RUN_EVENT_KIND.runCancelled,
|
|
23397
|
+
) === true;
|
|
22254
23398
|
// Search-only facts are already durable and have no ledger projection.
|
|
22255
23399
|
// Validate and acknowledge them without acquiring runtime authority or
|
|
22256
23400
|
// renewing a network-delivery lease. A following terminal still uses the
|
|
@@ -22269,9 +23413,23 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
22269
23413
|
) => {
|
|
22270
23414
|
authorityObserved.token = observation;
|
|
22271
23415
|
};
|
|
23416
|
+
const readTerminalProjectionOrigin = async () => {
|
|
23417
|
+
if (terminalArrayProjectionOrigin === undefined) {
|
|
23418
|
+
terminalArrayProjectionOrigin = await readTerminalArrayProjectionOrigin(
|
|
23419
|
+
withClient,
|
|
23420
|
+
launchJson,
|
|
23421
|
+
first.runId,
|
|
23422
|
+
options,
|
|
23423
|
+
claimed.arrayInputStarted,
|
|
23424
|
+
);
|
|
23425
|
+
}
|
|
23426
|
+
return terminalArrayProjectionOrigin;
|
|
23427
|
+
};
|
|
23428
|
+
if (terminalProjection) await readTerminalProjectionOrigin();
|
|
22272
23429
|
const resolveRunRuntimeApi = () =>
|
|
22273
23430
|
launchJson !== null
|
|
22274
23431
|
? resolvePostgresSchedulerRuntimeApiForRun(
|
|
23432
|
+
withClient,
|
|
22275
23433
|
launchJson,
|
|
22276
23434
|
first.runId,
|
|
22277
23435
|
first.leaseOwner
|
|
@@ -22282,6 +23440,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
22282
23440
|
input.runtimeApi?.signal,
|
|
22283
23441
|
reuseMarginMs,
|
|
22284
23442
|
observeDispatchTokenResponse,
|
|
23443
|
+
terminalProjection,
|
|
23444
|
+
terminalArrayProjectionOrigin,
|
|
22285
23445
|
)
|
|
22286
23446
|
: loadPostgresSchedulerRuntimeApiForRun(
|
|
22287
23447
|
withClient,
|
|
@@ -22294,10 +23454,13 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
22294
23454
|
input.runtimeApi?.signal,
|
|
22295
23455
|
reuseMarginMs,
|
|
22296
23456
|
observeDispatchTokenResponse,
|
|
23457
|
+
terminalProjection,
|
|
23458
|
+
terminalArrayProjectionOrigin,
|
|
22297
23459
|
);
|
|
22298
23460
|
const gatewayRuntimeApi =
|
|
22299
23461
|
needsRuntimeApi && tokenAuthority === 'gateway'
|
|
22300
|
-
?
|
|
23462
|
+
? (!terminalProjection || terminalArrayProjectionOrigin === null) &&
|
|
23463
|
+
cachedAuthority?.runId === first.runId &&
|
|
22301
23464
|
cachedAuthority.leaseOwner === leaseOwner &&
|
|
22302
23465
|
cachedAuthority.expiresAtMs !== null &&
|
|
22303
23466
|
cachedAuthority.expiresAtMs > Date.now() + reuseMarginMs
|
|
@@ -22315,13 +23478,30 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
22315
23478
|
cachedAuthority.leaseOwner === leaseOwner &&
|
|
22316
23479
|
cachedAuthority.runtimeApi.baseUrl !== gatewayRuntimeApi.baseUrl
|
|
22317
23480
|
) {
|
|
22318
|
-
|
|
22319
|
-
|
|
22320
|
-
|
|
23481
|
+
const terminalOrigin = terminalProjection
|
|
23482
|
+
? await readTerminalProjectionOrigin()
|
|
23483
|
+
: null;
|
|
23484
|
+
if (gatewayRuntimeApi.baseUrl !== terminalOrigin) {
|
|
23485
|
+
throw new Error(
|
|
23486
|
+
'Run Ledger dispatch origin changed within an immutable launch.',
|
|
23487
|
+
);
|
|
23488
|
+
}
|
|
22321
23489
|
}
|
|
22322
23490
|
if (needsRuntimeApi) {
|
|
23491
|
+
let suppliedRuntimeApi = input.runtimeApi;
|
|
23492
|
+
if (terminalProjection && suppliedRuntimeApi && !gatewayRuntimeApi) {
|
|
23493
|
+
const terminalOrigin = await readTerminalProjectionOrigin();
|
|
23494
|
+
if (terminalOrigin) {
|
|
23495
|
+
suppliedRuntimeApi = {
|
|
23496
|
+
...suppliedRuntimeApi,
|
|
23497
|
+
baseUrl: terminalOrigin,
|
|
23498
|
+
};
|
|
23499
|
+
}
|
|
23500
|
+
}
|
|
22323
23501
|
const resolvedRuntimeApi =
|
|
22324
|
-
gatewayRuntimeApi ??
|
|
23502
|
+
gatewayRuntimeApi ??
|
|
23503
|
+
suppliedRuntimeApi ??
|
|
23504
|
+
(await resolveRunRuntimeApi());
|
|
22325
23505
|
runtimeApi = {
|
|
22326
23506
|
...resolvedRuntimeApi,
|
|
22327
23507
|
...(tokenAuthority === 'local_mint' && input.appRuntimeFetch
|
|
@@ -23119,6 +24299,7 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
23119
24299
|
nextClaimed: [],
|
|
23120
24300
|
};
|
|
23121
24301
|
continuation.nextClaimed = fast.events;
|
|
24302
|
+
continuation.nextArrayInputStarted = fast.arrayInputStarted;
|
|
23122
24303
|
input.continuation.current = continuation;
|
|
23123
24304
|
}
|
|
23124
24305
|
return { deliveredRows: fast.deliveredRows, continuation };
|
|
@@ -23195,6 +24376,7 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
23195
24376
|
nextClaimed: [],
|
|
23196
24377
|
};
|
|
23197
24378
|
continuation.nextClaimed = nextClaimed;
|
|
24379
|
+
continuation.nextArrayInputStarted = undefined;
|
|
23198
24380
|
input.continuation.current = continuation;
|
|
23199
24381
|
}
|
|
23200
24382
|
if (retainContinuation) await client.query('COMMIT');
|
|
@@ -23370,6 +24552,48 @@ async function projectPostgresSchedulerStartEvent(
|
|
|
23370
24552
|
|
|
23371
24553
|
export type { RunLedgerDispatchLaunch } from './run-ledger-dispatch-authority';
|
|
23372
24554
|
|
|
24555
|
+
async function readTerminalArrayProjectionOrigin(
|
|
24556
|
+
withClient: RunLedgerOutboxWithClient,
|
|
24557
|
+
launchJson: unknown,
|
|
24558
|
+
runId: string,
|
|
24559
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
24560
|
+
arrayInputStarted?: boolean,
|
|
24561
|
+
): Promise<string | null> {
|
|
24562
|
+
const started =
|
|
24563
|
+
arrayInputStarted ??
|
|
24564
|
+
(
|
|
24565
|
+
await withClient((client) =>
|
|
24566
|
+
readTerminalArrayInputIntentForScope(client, { runId }, options),
|
|
24567
|
+
)
|
|
24568
|
+
).started;
|
|
24569
|
+
// Legacy deliveries may supply their runtime authority directly. They need
|
|
24570
|
+
// no launch parsing or routing lookup when there is no durable array intent.
|
|
24571
|
+
if (!started) return null;
|
|
24572
|
+
const launch = parseRunLedgerDispatchLaunch(
|
|
24573
|
+
launchJson ??
|
|
24574
|
+
(
|
|
24575
|
+
await withClient((client) =>
|
|
24576
|
+
client.query<{ launch_json: unknown }>(
|
|
24577
|
+
`SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
|
|
24578
|
+
FROM ${tableName('runs', options)} AS runs WHERE run_id = $1`,
|
|
24579
|
+
[runId],
|
|
24580
|
+
),
|
|
24581
|
+
)
|
|
24582
|
+
).rows[0]?.launch_json,
|
|
24583
|
+
);
|
|
24584
|
+
const origin = launch.arrayInputTerminalProjectionOrigin;
|
|
24585
|
+
if (
|
|
24586
|
+
launch.arrayInputTerminalRetirementSupported !== true ||
|
|
24587
|
+
typeof origin !== 'string' ||
|
|
24588
|
+
!isImmutableRuntimeExecutionRecoveryOrigin(origin)
|
|
24589
|
+
) {
|
|
24590
|
+
throw new Error(
|
|
24591
|
+
'Staged array input has no immutable terminal projection receiver.',
|
|
24592
|
+
);
|
|
24593
|
+
}
|
|
24594
|
+
return origin;
|
|
24595
|
+
}
|
|
24596
|
+
|
|
23373
24597
|
/** Search-only `run.events` facts settle without consulting launch credentials. */
|
|
23374
24598
|
function runLedgerSearchOnlyOutboxFactSql(alias: string): string {
|
|
23375
24599
|
const events = `CASE
|
|
@@ -23385,6 +24609,7 @@ function runLedgerSearchOnlyOutboxFactSql(alias: string): string {
|
|
|
23385
24609
|
}
|
|
23386
24610
|
|
|
23387
24611
|
async function resolvePostgresSchedulerRuntimeApiForRun(
|
|
24612
|
+
withClient: RunLedgerOutboxWithClient,
|
|
23388
24613
|
launchJson: unknown,
|
|
23389
24614
|
runId: string,
|
|
23390
24615
|
dispatchLease: { eventId: string; leaseOwner: string } | null,
|
|
@@ -23395,6 +24620,8 @@ async function resolvePostgresSchedulerRuntimeApiForRun(
|
|
|
23395
24620
|
onDispatchTokenResponse?: (
|
|
23396
24621
|
observation: RunLedgerDispatchTokenObservation,
|
|
23397
24622
|
) => void,
|
|
24623
|
+
terminalProjection = false,
|
|
24624
|
+
terminalArrayProjectionOrigin?: string | null,
|
|
23398
24625
|
): Promise<
|
|
23399
24626
|
WorkerRuntimeApiContext & {
|
|
23400
24627
|
refreshExecutorToken?: () => Promise<string>;
|
|
@@ -23407,6 +24634,18 @@ async function resolvePostgresSchedulerRuntimeApiForRun(
|
|
|
23407
24634
|
`Run-ledger outbox dispatcher cannot resolve runtime API context for ${runId}.`,
|
|
23408
24635
|
);
|
|
23409
24636
|
}
|
|
24637
|
+
let receiverBaseUrl = runtimeApiBaseUrlFromLaunch(launch);
|
|
24638
|
+
if (terminalProjection) {
|
|
24639
|
+
receiverBaseUrl =
|
|
24640
|
+
(terminalArrayProjectionOrigin === undefined
|
|
24641
|
+
? await readTerminalArrayProjectionOrigin(
|
|
24642
|
+
withClient,
|
|
24643
|
+
launch,
|
|
24644
|
+
runId,
|
|
24645
|
+
options,
|
|
24646
|
+
)
|
|
24647
|
+
: terminalArrayProjectionOrigin) ?? receiverBaseUrl;
|
|
24648
|
+
}
|
|
23410
24649
|
let executorToken: string;
|
|
23411
24650
|
let runLedgerIngressPrefixBatching: true | undefined;
|
|
23412
24651
|
let refreshExecutorToken: (() => Promise<string>) | undefined;
|
|
@@ -23435,12 +24674,12 @@ async function resolvePostgresSchedulerRuntimeApiForRun(
|
|
|
23435
24674
|
leaseOwner: dispatchLease.leaseOwner,
|
|
23436
24675
|
schedulerSchema: options?.schema ?? null,
|
|
23437
24676
|
signal,
|
|
24677
|
+
receiverBaseUrl,
|
|
23438
24678
|
};
|
|
23439
|
-
const preMintedToken =
|
|
23440
|
-
launch
|
|
23441
|
-
|
|
23442
|
-
|
|
23443
|
-
);
|
|
24679
|
+
const preMintedToken =
|
|
24680
|
+
receiverBaseUrl === runtimeApiBaseUrlFromLaunch(launch)
|
|
24681
|
+
? usableRunLedgerExecutorToken(launch, tokenReuseMarginMs, options)
|
|
24682
|
+
: null;
|
|
23444
24683
|
if (preMintedToken) {
|
|
23445
24684
|
// Admission minted this token with only `run.ledger.project`.
|
|
23446
24685
|
// Each projection request still proves its exact outbox lease at the
|
|
@@ -23476,8 +24715,18 @@ async function resolvePostgresSchedulerRuntimeApiForRun(
|
|
|
23476
24715
|
dispatchAuthority.runLedgerIngressPrefixBatching;
|
|
23477
24716
|
}
|
|
23478
24717
|
}
|
|
24718
|
+
// A mutable source can seal array pages after this prefix was claimed.
|
|
24719
|
+
// Never widen a nonterminal receipt into a terminal on an older receiver;
|
|
24720
|
+
// the next claim will see the terminal and select its immutable owner.
|
|
24721
|
+
if (
|
|
24722
|
+
launch.arrayInputTerminalRetirementSupported === true &&
|
|
24723
|
+
typeof launch.arrayInputTerminalProjectionOrigin === 'string' &&
|
|
24724
|
+
launch.arrayInputTerminalProjectionOrigin !== receiverBaseUrl
|
|
24725
|
+
) {
|
|
24726
|
+
runLedgerIngressPrefixBatching = undefined;
|
|
24727
|
+
}
|
|
23479
24728
|
return {
|
|
23480
|
-
baseUrl:
|
|
24729
|
+
baseUrl: receiverBaseUrl,
|
|
23481
24730
|
executorToken,
|
|
23482
24731
|
...(runLedgerIngressPrefixBatching
|
|
23483
24732
|
? { runLedgerIngressPrefixBatching }
|
|
@@ -23501,6 +24750,8 @@ async function loadPostgresSchedulerRuntimeApiForRun(
|
|
|
23501
24750
|
onDispatchTokenResponse?: (
|
|
23502
24751
|
observation: RunLedgerDispatchTokenObservation,
|
|
23503
24752
|
) => void,
|
|
24753
|
+
terminalProjection = false,
|
|
24754
|
+
terminalArrayProjectionOrigin?: string | null,
|
|
23504
24755
|
): ReturnType<typeof resolvePostgresSchedulerRuntimeApiForRun> {
|
|
23505
24756
|
// A continuation already owns its next event lease, so its launch context
|
|
23506
24757
|
// is not part of the preclaim result. Load only the authority fields here.
|
|
@@ -23514,6 +24765,7 @@ async function loadPostgresSchedulerRuntimeApiForRun(
|
|
|
23514
24765
|
),
|
|
23515
24766
|
);
|
|
23516
24767
|
return resolvePostgresSchedulerRuntimeApiForRun(
|
|
24768
|
+
withClient,
|
|
23517
24769
|
result.rows[0]?.launch_json,
|
|
23518
24770
|
runId,
|
|
23519
24771
|
dispatchLease,
|
|
@@ -23522,6 +24774,8 @@ async function loadPostgresSchedulerRuntimeApiForRun(
|
|
|
23522
24774
|
signal,
|
|
23523
24775
|
tokenReuseMarginMs,
|
|
23524
24776
|
onDispatchTokenResponse,
|
|
24777
|
+
terminalProjection,
|
|
24778
|
+
terminalArrayProjectionOrigin,
|
|
23525
24779
|
);
|
|
23526
24780
|
}
|
|
23527
24781
|
|
|
@@ -23702,6 +24956,7 @@ async function requestRunLedgerDispatchToken(
|
|
|
23702
24956
|
leaseOwner: string;
|
|
23703
24957
|
schedulerSchema: string | null;
|
|
23704
24958
|
signal?: AbortSignal;
|
|
24959
|
+
receiverBaseUrl: string;
|
|
23705
24960
|
},
|
|
23706
24961
|
forceFresh = false,
|
|
23707
24962
|
): Promise<{
|
|
@@ -23721,7 +24976,14 @@ async function requestRunLedgerDispatchToken(
|
|
|
23721
24976
|
input.signal?.throwIfAborted();
|
|
23722
24977
|
const { executorToken: _storedToken, ...authority } = input.launch;
|
|
23723
24978
|
const cacheKey = createHash('sha256')
|
|
23724
|
-
.update(
|
|
24979
|
+
.update(
|
|
24980
|
+
JSON.stringify([
|
|
24981
|
+
input.schedulerSchema,
|
|
24982
|
+
authority,
|
|
24983
|
+
input.receiverBaseUrl,
|
|
24984
|
+
internalToken,
|
|
24985
|
+
]),
|
|
24986
|
+
)
|
|
23725
24987
|
.digest('hex');
|
|
23726
24988
|
const now = Date.now();
|
|
23727
24989
|
for (const [key, entry] of runLedgerDispatchTokens) {
|
|
@@ -23738,7 +25000,7 @@ async function requestRunLedgerDispatchToken(
|
|
|
23738
25000
|
: {}),
|
|
23739
25001
|
};
|
|
23740
25002
|
runLedgerDispatchTokens.delete(cacheKey);
|
|
23741
|
-
const url = `${
|
|
25003
|
+
const url = `${input.receiverBaseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime/run-ledger-dispatch-token`;
|
|
23742
25004
|
const requestSignal = input.signal
|
|
23743
25005
|
? AbortSignal.any([
|
|
23744
25006
|
input.signal,
|
|
@@ -23752,7 +25014,7 @@ async function requestRunLedgerDispatchToken(
|
|
|
23752
25014
|
schedulerSchema: input.schedulerSchema,
|
|
23753
25015
|
});
|
|
23754
25016
|
const response = await fetchWithVercelProtectionBypass({
|
|
23755
|
-
baseUrl:
|
|
25017
|
+
baseUrl: input.receiverBaseUrl,
|
|
23756
25018
|
token: input.launch.vercelProtectionBypassToken,
|
|
23757
25019
|
signal: requestSignal,
|
|
23758
25020
|
send: (protectionHeaders) =>
|
|
@@ -23809,9 +25071,7 @@ async function requestRunLedgerDispatchToken(
|
|
|
23809
25071
|
}
|
|
23810
25072
|
const reusablePrefix =
|
|
23811
25073
|
body.runLedgerIngressPrefixBatching === true &&
|
|
23812
|
-
isImmutableRuntimeExecutionRecoveryOrigin(
|
|
23813
|
-
runtimeApiBaseUrlFromLaunch(input.launch),
|
|
23814
|
-
);
|
|
25074
|
+
isImmutableRuntimeExecutionRecoveryOrigin(input.receiverBaseUrl);
|
|
23815
25075
|
// Decode only the lifetime, never an authentication decision: the app
|
|
23816
25076
|
// validates the signature, run scope, and live delivery lease on append.
|
|
23817
25077
|
// Opaque/older token formats continue to work without this optimization.
|