deepline 0.3.408 → 0.3.410
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/agent-runtime.ts +18 -1
- package/dist/bundling-sources/sdk/src/client.ts +212 -334
- package/dist/bundling-sources/sdk/src/compat.ts +45 -2
- package/dist/bundling-sources/sdk/src/db/query.ts +9 -10
- package/dist/bundling-sources/sdk/src/http.ts +38 -20
- package/dist/bundling-sources/sdk/src/index.ts +4 -5
- package/dist/bundling-sources/sdk/src/play.ts +10 -10
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +6 -4
- package/dist/bundling-sources/sdk/src/plays/check-local-play-graph.ts +114 -0
- package/dist/bundling-sources/sdk/src/plays/client.ts +431 -0
- package/dist/bundling-sources/sdk/src/plays/local-play-execution.ts +2 -2
- package/dist/bundling-sources/sdk/src/plays/run-request.ts +106 -16
- package/dist/bundling-sources/sdk/src/release.ts +13 -7
- package/dist/bundling-sources/sdk/src/runs/read-errors.ts +8 -0
- package/dist/bundling-sources/sdk/src/runs/watch-polling.ts +124 -13
- package/dist/bundling-sources/sdk/src/tool-calls.ts +108 -19
- package/dist/bundling-sources/sdk/src/types.ts +18 -0
- package/dist/bundling-sources/shared_libs/integration-contracts/execute-retry-contract.ts +37 -0
- package/dist/bundling-sources/shared_libs/integration-contracts/package.json +3 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +277 -76
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/dataset-row-input.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +6 -62
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-tool-identity.ts +67 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +64 -6
- package/dist/bundling-sources/shared_libs/play-runtime/freshness-receipt-execution.ts +1 -5
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-artifact-runtime-contract.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-http-request.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/query-result-dataset.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +26 -16
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-heartbeat.ts +36 -11
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +35 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +21 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +11 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +85 -20
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +71 -13
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-execution-queries.ts +100 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts +9 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +337 -146
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +26 -12
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +425 -102
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +181 -124
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-worker-wake.ts +152 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +5 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-cache-policy.ts +31 -0
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +10 -0
- package/dist/bundling-sources/shared_libs/plays/ambiguous-write-error.ts +25 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +4 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +63 -68
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +129 -63
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +14 -0
- package/dist/bundling-sources/shared_libs/plays/enrich-play-compiler.ts +58 -16
- package/dist/bundling-sources/shared_libs/plays/fetch-http-error.ts +50 -0
- package/dist/bundling-sources/shared_libs/plays/source-authoring-edition.ts +351 -0
- package/dist/bundling-sources/shared_libs/plays/tool-list-getter-mode.ts +9 -0
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +1 -1
- package/dist/cli/{auth-UVD3CUBB.mjs → auth-LLXLIILR.mjs} +6 -6
- package/dist/cli/auth-UG4WOYGL.js +40 -0
- package/dist/cli/{billing-BZVTXXLM.mjs → billing-5FVXZYAC.mjs} +8 -8
- package/dist/cli/{billing-BZVYAF6T.js → billing-NFPK7GYP.js} +84 -84
- package/dist/cli/{chunk-ZAAKSJAS.js → chunk-27GJKC5Q.js} +17 -10
- package/dist/cli/{chunk-UHNDVD3P.js → chunk-2H4EY7KH.js} +100 -41
- package/dist/cli/{chunk-WTT6CLED.mjs → chunk-2YR7BIIC.mjs} +5 -2
- package/dist/cli/{chunk-3ZB6IIA7.js → chunk-36AHOWVS.js} +26 -26
- package/dist/cli/{chunk-OSDXE27V.mjs → chunk-4PG4ARL2.mjs} +17 -10
- package/dist/cli/{chunk-LOYISF7S.mjs → chunk-4XQLVUC7.mjs} +738 -512
- package/dist/cli/{chunk-J5PTX66Y.js → chunk-5MLCBXXU.js} +8 -1
- package/dist/cli/chunk-5TUGWDEB.js +313 -0
- package/dist/cli/{chunk-D6FFBSM5.js → chunk-5UUQAGJS.js} +5 -2
- package/dist/cli/{chunk-4CFULMRB.mjs → chunk-5V5Y37BC.mjs} +2 -2
- package/dist/cli/{chunk-PMVFAYL7.mjs → chunk-63ILLN5H.mjs} +2 -2
- package/dist/cli/{chunk-AUMD2VXP.js → chunk-63S7X326.js} +1032 -806
- package/dist/cli/{chunk-4QFT5LGV.mjs → chunk-6PGATQ6L.mjs} +6 -3
- package/dist/cli/{chunk-6BJK4EJ3.mjs → chunk-AARWVK4C.mjs} +2132 -65
- package/dist/cli/{chunk-FERRDNMO.js → chunk-AD47BXN2.js} +5 -5
- package/dist/cli/{chunk-4RNDKFNQ.js → chunk-D37FEBES.js} +2 -2
- package/dist/cli/{chunk-S6R3OX7B.js → chunk-E4CABWXL.js} +25 -2
- package/dist/cli/{chunk-NXGCO4JB.mjs → chunk-E4IXDAZI.mjs} +5 -0
- package/dist/cli/chunk-EQ4Z7RJW.mjs +35 -0
- package/dist/cli/chunk-FR2ILQ4G.mjs +8 -0
- package/dist/cli/{chunk-QUAQX3FQ.js → chunk-FVLNYDI4.js} +128 -376
- package/dist/cli/{chunk-BRJUESQV.js → chunk-FZJAM576.js} +50 -50
- package/dist/cli/{chunk-BZAMOCPA.mjs → chunk-IMZ56EOH.mjs} +3 -3
- package/dist/cli/{chunk-IOHRWLQE.mjs → chunk-IPG42SD2.mjs} +452 -138
- package/dist/cli/chunk-J33TSWO5.js +8 -0
- package/dist/cli/{chunk-XXLXOUUU.mjs → chunk-JNXJZ577.mjs} +1 -1
- package/dist/cli/{chunk-JI2B4RZP.mjs → chunk-KCC3FKIW.mjs} +71 -12
- package/dist/cli/{chunk-6IKARAYD.mjs → chunk-L2VKAASU.mjs} +23 -0
- package/dist/cli/{chunk-PYZZMAJQ.mjs → chunk-LFSHGNT2.mjs} +8 -1
- package/dist/cli/{chunk-2XXXFPTD.js → chunk-LTFL4JRN.js} +426 -721
- package/dist/cli/{chunk-YHPYUZZX.mjs → chunk-MAMVGOSC.mjs} +1 -1
- package/dist/cli/{chunk-URACNNLO.mjs → chunk-MN2OWLFA.mjs} +2 -2
- package/dist/cli/{chunk-Q3PUHXOY.js → chunk-NF4V5DTY.js} +12 -12
- package/dist/cli/{chunk-Q2E4E4FJ.js → chunk-NPBJI7RE.js} +8 -8
- package/dist/cli/chunk-NTVLGHFM.js +501 -0
- package/dist/cli/{chunk-HQXL5526.mjs → chunk-NUX2QHNC.mjs} +1 -1
- package/dist/cli/{chunk-E3PJF6YP.js → chunk-OBB4QXHA.js} +19 -4
- package/dist/cli/{chunk-SEFMGO3Z.js → chunk-OQQOXSYA.js} +2184 -117
- package/dist/cli/chunk-OZ6E3IRZ.js +6084 -0
- package/dist/cli/{chunk-7P5AA3SN.js → chunk-PPDBRJRW.js} +6 -1
- package/dist/cli/{chunk-IWBXJDZA.mjs → chunk-PZSLPBDA.mjs} +2 -2
- package/dist/cli/{chunk-ILHRA7AD.js → chunk-QADCFSGH.js} +7 -7
- package/dist/cli/{chunk-ECVLPS67.js → chunk-QJI5FIOW.js} +170 -49
- package/dist/cli/{chunk-I3DHIXVJ.mjs → chunk-QL22Y56M.mjs} +149 -28
- package/dist/cli/{chunk-SD7MXAJA.mjs → chunk-RFB25DSQ.mjs} +36 -284
- package/dist/cli/chunk-T44IIOT6.mjs +501 -0
- package/dist/cli/chunk-VK2LZ5IP.js +35 -0
- package/dist/cli/chunk-VKKVSFRD.js +62 -0
- package/dist/cli/{chunk-RWCY6YI2.js → chunk-VYKV5XHL.js} +644 -330
- package/dist/cli/{chunk-GSK6NHVM.js → chunk-WU4U6VSB.js} +5 -5
- package/dist/cli/chunk-XHLKJOTN.mjs +313 -0
- package/dist/cli/{chunk-MJELBQBU.mjs → chunk-XIHWSXSV.mjs} +330 -625
- package/dist/cli/chunk-XNZJKRJA.mjs +6084 -0
- package/dist/cli/chunk-ZCALLYF6.mjs +62 -0
- package/dist/cli/{chunk-ZBABPK6Y.js → chunk-ZFZ427HN.js} +11 -8
- package/dist/cli/{chunk-SJVVBWRL.mjs → chunk-ZKYEW55U.mjs} +16 -1
- package/dist/cli/command-docs.js +2 -2
- package/dist/cli/command-docs.mjs +1 -1
- package/dist/cli/{csv-XLGLHJ6P.js → csv-7U3FJYKZ.js} +14 -14
- package/dist/cli/{csv-WPSXG6ZZ.mjs → csv-ZWLDMM4A.mjs} +5 -5
- package/dist/cli/{db-5PTNJG4X.js → db-DNCGV5PC.js} +42 -47
- package/dist/cli/{db-ABS57ABJ.mjs → db-IMYXVMGI.mjs} +15 -20
- package/dist/cli/{describe-PVWQJ2PB.mjs → describe-Q23DWJUK.mjs} +20 -15
- package/dist/cli/{describe-EZS6PQXA.js → describe-S34JUX5W.js} +27 -22
- package/dist/cli/describe-core.d.mts +2 -2
- package/dist/cli/describe-core.d.ts +2 -2
- package/dist/cli/describe-core.js +6 -6
- package/dist/cli/describe-core.mjs +5 -5
- package/dist/cli/{executions-OKJGATQZ.mjs → executions-A73EUGSI.mjs} +8 -8
- package/dist/cli/{executions-FNTAFQ7Y.js → executions-XOMPX2YJ.js} +10 -10
- package/dist/cli/{feedback-W27HDMRR.mjs → feedback-N4GJ2POI.mjs} +11 -7
- package/dist/cli/{feedback-6YBOMSUA.js → feedback-QTCHNJJH.js} +13 -9
- package/dist/cli/{git-YOOUEL6M.mjs → git-4Q3JIQYZ.mjs} +6 -6
- package/dist/cli/git-I6CPBEVG.js +24 -0
- package/dist/cli/index.js +240 -352
- package/dist/cli/index.mjs +163 -275
- package/dist/cli/{legacy-noops-V6OKW7VR.mjs → legacy-noops-3SOVGFF7.mjs} +6 -6
- package/dist/cli/{legacy-noops-2VZ7XT6C.js → legacy-noops-4OSHZM34.js} +9 -9
- package/dist/cli/{monitors-FBWGVIUS.js → monitors-LVE2KZNG.js} +94 -94
- package/dist/cli/{monitors-P2YYWTWT.mjs → monitors-QP2XID2B.mjs} +8 -8
- package/dist/cli/{org-DVMG6H3C.mjs → org-IVRETVQA.mjs} +8 -8
- package/dist/cli/{org-3XUSVX5T.js → org-XFLK5IAP.js} +68 -68
- package/dist/cli/{play-GNC5CQAS.mjs → play-DE7VH6NU.mjs} +533 -98
- package/dist/cli/{play-YG5L4BWQ.js → play-XXTPPP23.js} +1067 -632
- package/dist/cli/{providers-7PSWG2OV.js → providers-DLD3I4JF.js} +11 -11
- package/dist/cli/{providers-JR33SLWL.mjs → providers-GX2NIZQQ.mjs} +8 -8
- package/dist/cli/{quickstart-VUOBQYIQ.mjs → quickstart-72B4L4PK.mjs} +5 -5
- package/dist/cli/{quickstart-FDVZ5BL3.js → quickstart-XNS3HSWC.js} +7 -7
- package/dist/cli/search-6NCH4HB5.js +30 -0
- package/dist/cli/{search-JZYLO33K.mjs → search-SUTRANJI.mjs} +9 -9
- package/dist/cli/{secrets-VS266FVA.js → secrets-6OBLOVEV.js} +17 -17
- package/dist/cli/{secrets-3MCYBT4I.mjs → secrets-FPFXXNUH.mjs} +8 -8
- package/dist/cli/sessions-235532DR.js +16 -0
- package/dist/cli/{sessions-ELLLT23E.mjs → sessions-R25I4QKN.mjs} +6 -6
- package/dist/cli/{settings-PZCKAFII.mjs → settings-AOP6F5PZ.mjs} +12 -12
- package/dist/cli/{settings-7364NMMA.js → settings-VL5FOFPN.js} +79 -79
- package/dist/cli/{setup-PMWYLCT6.js → setup-BSC67YT3.js} +45 -44
- package/dist/cli/{setup-A7GQ4KIA.mjs → setup-ZMPHBHVJ.mjs} +11 -10
- package/dist/cli/skills-3IWLYYRL.js +29 -0
- package/dist/cli/{skills-WS6BIFPQ.mjs → skills-KF65TVFO.mjs} +7 -7
- package/dist/cli/tools-LR66SPYU.js +44 -0
- package/dist/cli/{tools-BR4DOZA2.mjs → tools-SX4XIPM7.mjs} +19 -14
- package/dist/cli/{update-D2B7UNFT.mjs → update-QRMFRAQQ.mjs} +11 -9
- package/dist/cli/update-SP7UEO53.js +31 -0
- package/dist/cli/{workflow-PQXC2R47.mjs → workflow-FOEEEB7G.mjs} +44 -20
- package/dist/cli/{workflow-6ODTCYXU.js → workflow-KOCK2FLW.js} +54 -30
- package/dist/{client-Z9Pye1Wm.d.mts → client-74k4IPh6.d.mts} +137 -54
- package/dist/{client-BcvhDfJJ.d.ts → client-C1IK1f8c.d.ts} +137 -54
- package/dist/{compiler-manifest-Ycx-hoAr.d.mts → compiler-manifest-lzWpbV3w.d.mts} +83 -66
- package/dist/{compiler-manifest-Ycx-hoAr.d.ts → compiler-manifest-lzWpbV3w.d.ts} +83 -66
- package/dist/{index-CZXgiN3E.d.mts → index-C8Cy-YTm.d.ts} +5 -1
- package/dist/{index-NPG5pqxQ.d.ts → index-CtNK-0nI.d.mts} +5 -1
- package/dist/index.d.mts +47 -9
- package/dist/index.d.ts +47 -9
- package/dist/index.js +17 -14
- package/dist/index.mjs +13 -10
- package/dist/install-integrity.json +120 -104
- package/dist/plays/bundle-play-file.d.mts +3 -3
- package/dist/plays/bundle-play-file.d.ts +3 -3
- package/dist/plays/bundle-play-file.mjs +5 -4
- package/dist/plays/csv-data-input.js +3 -3
- package/dist/plays/csv-data-input.mjs +2 -2
- package/dist/plays/play-description.d.mts +2 -2
- package/dist/plays/play-description.d.ts +2 -2
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/plays/run-request.js +11 -3
- package/dist/plays/run-request.mjs +10 -2
- package/dist/release.d.mts +15 -6
- package/dist/release.d.ts +15 -6
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +2 -2
- package/dist/cli/auth-3GB2K636.js +0 -40
- package/dist/cli/chunk-67RWR34T.mjs +0 -155
- package/dist/cli/chunk-CLJGSOVM.mjs +0 -1716
- package/dist/cli/chunk-HQY2SG3K.js +0 -83
- package/dist/cli/chunk-OSFWFV72.js +0 -155
- package/dist/cli/chunk-Q3LFI3VH.js +0 -392
- package/dist/cli/chunk-WPAB4DCU.mjs +0 -83
- package/dist/cli/chunk-YMOFWHGO.mjs +0 -392
- package/dist/cli/chunk-Z4VNJOF7.js +0 -1716
- package/dist/cli/git-YKUXXJOR.js +0 -24
- package/dist/cli/search-PPPYC5OZ.js +0 -30
- package/dist/cli/sessions-TEHEWW7O.js +0 -16
- package/dist/cli/skills-SYS7VRRK.js +0 -29
- package/dist/cli/tools-KK4BKVQR.js +0 -39
- package/dist/cli/update-2IV5JTX2.js +0 -29
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* seam (Daytona sandbox / local subprocess), never in this worker process.
|
|
16
16
|
*
|
|
17
17
|
* ADR-0011 attempt fencing: each handler invocation requests
|
|
18
|
-
* `
|
|
18
|
+
* `max(attempt) + 1` from its preparation snapshot and a fresh lease
|
|
19
19
|
* token. The store's assign/mark/complete guards ensure only the highest
|
|
20
20
|
* generation with a matching lease token writes terminals; a reclaimed or
|
|
21
21
|
* lease-lost handler observes `idle` and abandons WITHOUT writing a terminal.
|
|
@@ -45,8 +45,11 @@ import {
|
|
|
45
45
|
} from '../app-runtime-api';
|
|
46
46
|
import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
|
|
47
47
|
import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
|
|
48
|
+
import {
|
|
49
|
+
isRuntimeTrafficPolicy,
|
|
50
|
+
type RuntimeTrafficPolicy,
|
|
51
|
+
} from '../runtime-traffic-policy';
|
|
48
52
|
import { RUNTIME_CLAIM_HEARTBEAT_DIVISOR } from '../runtime-constants';
|
|
49
|
-
import type { RuntimeTrafficPolicy } from '../runtime-traffic-policy';
|
|
50
53
|
import { runtimeLimitExceededCause } from '../run-failure';
|
|
51
54
|
import {
|
|
52
55
|
playRunRecoveryLogDetail,
|
|
@@ -74,7 +77,10 @@ import {
|
|
|
74
77
|
releaseRuntimeWorkReceipts,
|
|
75
78
|
} from '../runtime-api';
|
|
76
79
|
import { PLAY_RUNTIME_ATTEMPT_EXECUTOR_TOKEN_PATH } from '../runtime-api-paths';
|
|
77
|
-
import {
|
|
80
|
+
import {
|
|
81
|
+
fetchWithVercelProtectionBypass,
|
|
82
|
+
vercelProtectionBypassHeaders,
|
|
83
|
+
} from '../vercel-protection';
|
|
78
84
|
import {
|
|
79
85
|
isManagedSandboxRuntimeBackend,
|
|
80
86
|
normalizePlayRuntimeBackend,
|
|
@@ -122,7 +128,6 @@ import {
|
|
|
122
128
|
} from '../runner-backends/runtime-sandbox-reconciliation';
|
|
123
129
|
import { readPersistedRuntimeSandboxRef } from '../runtime-sandbox-contract';
|
|
124
130
|
import {
|
|
125
|
-
allocatePostgresSchedulerNextAttempt,
|
|
126
131
|
assignPostgresSchedulerRunAttempt,
|
|
127
132
|
blockPostgresSchedulerComputeBillingJob,
|
|
128
133
|
buildPostgresSchedulerRunnerConfig,
|
|
@@ -168,11 +173,11 @@ import {
|
|
|
168
173
|
recordPostgresSchedulerRunnerTerminal,
|
|
169
174
|
resumePostgresSchedulerDetachedRunnerAttempt,
|
|
170
175
|
retryPostgresSchedulerComputeBillingJob,
|
|
176
|
+
readPostgresSchedulerAttemptPreparation,
|
|
177
|
+
readPostgresSchedulerAttemptAuthorityForLease,
|
|
171
178
|
readPostgresSchedulerLaunch,
|
|
172
|
-
readPostgresSchedulerOutboxDeliveryStatus,
|
|
173
179
|
readComputeBillingDispatchLaunchForLease,
|
|
174
180
|
readPostgresSchedulerResult,
|
|
175
|
-
readPostgresSchedulerRunLaneState,
|
|
176
181
|
listPostgresSchedulerAdoptableReleaseLanes,
|
|
177
182
|
readPostgresSchedulerRunnerTerminal,
|
|
178
183
|
readPostgresSchedulerRunnerLiveness,
|
|
@@ -249,6 +254,7 @@ import {
|
|
|
249
254
|
ABSURD_PLAY_RUN_TASK,
|
|
250
255
|
ABSURD_RUNS_QUEUE,
|
|
251
256
|
absurdRunnerTerminalEventName,
|
|
257
|
+
absurdRunCreatedProjectionEventName,
|
|
252
258
|
absurdCheckpointBase,
|
|
253
259
|
absurdCheckpointWithIntegrationEvent,
|
|
254
260
|
absurdCheckpointWithSleep,
|
|
@@ -781,6 +787,9 @@ export type AbsurdRuntimeWorkerDeps = {
|
|
|
781
787
|
attempt: number;
|
|
782
788
|
leaseToken: string;
|
|
783
789
|
schedulerSchema?: string | null;
|
|
790
|
+
initializeComputeSession?: boolean;
|
|
791
|
+
onComputeSessionInitialized?: () => void;
|
|
792
|
+
onRuntimeTrafficPolicy?: (policy: RuntimeTrafficPolicy) => void;
|
|
784
793
|
}) => Promise<string>;
|
|
785
794
|
/** This worker's release lane id. Seam so tests can drive the belt-and-
|
|
786
795
|
* suspenders lane-mismatch guard without mutating process env. Defaults to
|
|
@@ -1006,6 +1015,9 @@ export async function requestAttemptExecutorToken(input: {
|
|
|
1006
1015
|
attempt: number;
|
|
1007
1016
|
leaseToken: string;
|
|
1008
1017
|
schedulerSchema?: string | null;
|
|
1018
|
+
initializeComputeSession?: boolean;
|
|
1019
|
+
onComputeSessionInitialized?: () => void;
|
|
1020
|
+
onRuntimeTrafficPolicy?: (policy: RuntimeTrafficPolicy) => void;
|
|
1009
1021
|
}): Promise<string> {
|
|
1010
1022
|
const internalToken = process.env.DEEPLINE_INTERNAL_TOKEN?.trim();
|
|
1011
1023
|
if (!internalToken) {
|
|
@@ -1014,13 +1026,6 @@ export async function requestAttemptExecutorToken(input: {
|
|
|
1014
1026
|
);
|
|
1015
1027
|
}
|
|
1016
1028
|
const url = `${runtimeApiBaseUrlFromLaunch(input.launch).replace(/\/$/, '')}${PLAY_RUNTIME_ATTEMPT_EXECUTOR_TOKEN_PATH}`;
|
|
1017
|
-
const vercelProtectionHeaders = await vercelProtectionBypassHeaders({
|
|
1018
|
-
baseUrl: runtimeApiBaseUrlFromLaunch(input.launch),
|
|
1019
|
-
token:
|
|
1020
|
-
input.launch.vercelProtectionBypassToken ??
|
|
1021
|
-
process.env.VERCEL_PROTECTION_BYPASS_TOKEN ??
|
|
1022
|
-
null,
|
|
1023
|
-
});
|
|
1024
1029
|
const traceId = randomUUID();
|
|
1025
1030
|
const startedAt = performance.now();
|
|
1026
1031
|
console.info('[runtime-attempt-token.request_started]', {
|
|
@@ -1033,26 +1038,38 @@ export async function requestAttemptExecutorToken(input: {
|
|
|
1033
1038
|
);
|
|
1034
1039
|
let response: Response;
|
|
1035
1040
|
try {
|
|
1036
|
-
response = await
|
|
1041
|
+
response = await fetchWithVercelProtectionBypass({
|
|
1042
|
+
baseUrl: runtimeApiBaseUrlFromLaunch(input.launch),
|
|
1043
|
+
token:
|
|
1044
|
+
input.launch.vercelProtectionBypassToken ??
|
|
1045
|
+
process.env.VERCEL_PROTECTION_BYPASS_TOKEN ??
|
|
1046
|
+
null,
|
|
1037
1047
|
signal: requestSignal,
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1048
|
+
send: (protectionHeaders) =>
|
|
1049
|
+
fetch(url, {
|
|
1050
|
+
signal: requestSignal,
|
|
1051
|
+
method: 'POST',
|
|
1052
|
+
headers: {
|
|
1053
|
+
'content-type': 'application/json',
|
|
1054
|
+
'x-deepline-internal-token': internalToken,
|
|
1055
|
+
'x-deepline-runtime-trace-id': traceId,
|
|
1056
|
+
...protectionHeaders,
|
|
1057
|
+
},
|
|
1058
|
+
body: JSON.stringify({
|
|
1059
|
+
runId: input.launch.runId,
|
|
1060
|
+
attempt: input.attempt,
|
|
1061
|
+
leaseToken: input.leaseToken,
|
|
1062
|
+
schedulerSchema: input.schedulerSchema ?? null,
|
|
1063
|
+
...(input.initializeComputeSession
|
|
1064
|
+
? { initializeComputeSession: true }
|
|
1065
|
+
: {}),
|
|
1066
|
+
// Expand-before-enable rollout fence. An old worker never sends this,
|
|
1067
|
+
// so a newly deployed app cannot activate direct terminal reduction
|
|
1068
|
+
// before the owning Fly lane has the cleanup controller.
|
|
1069
|
+
terminalReducerContract: DIRECT_COMPLETED_TERMINAL_CONTRACT,
|
|
1070
|
+
}),
|
|
1071
|
+
redirect: 'manual',
|
|
1072
|
+
}),
|
|
1056
1073
|
});
|
|
1057
1074
|
} catch (error) {
|
|
1058
1075
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1076,6 +1093,12 @@ export async function requestAttemptExecutorToken(input: {
|
|
|
1076
1093
|
error?: unknown;
|
|
1077
1094
|
code?: unknown;
|
|
1078
1095
|
capacitySource?: unknown;
|
|
1096
|
+
computeSessionInitialized?: { sessionId?: unknown; attempt?: unknown };
|
|
1097
|
+
runtimeTrafficPolicy?: {
|
|
1098
|
+
runId?: unknown;
|
|
1099
|
+
attempt?: unknown;
|
|
1100
|
+
policy?: unknown;
|
|
1101
|
+
};
|
|
1079
1102
|
} | null;
|
|
1080
1103
|
if (!response.ok || typeof body?.executorToken !== 'string') {
|
|
1081
1104
|
const frontDoorTitle = body
|
|
@@ -1129,6 +1152,21 @@ export async function requestAttemptExecutorToken(input: {
|
|
|
1129
1152
|
response.headers.get('x-request-id') ??
|
|
1130
1153
|
null,
|
|
1131
1154
|
});
|
|
1155
|
+
if (
|
|
1156
|
+
input.initializeComputeSession === true &&
|
|
1157
|
+
body.computeSessionInitialized?.sessionId === input.launch.runId &&
|
|
1158
|
+
body.computeSessionInitialized.attempt === input.attempt
|
|
1159
|
+
) {
|
|
1160
|
+
input.onComputeSessionInitialized?.();
|
|
1161
|
+
const policyAcknowledgement = body.runtimeTrafficPolicy;
|
|
1162
|
+
if (
|
|
1163
|
+
policyAcknowledgement?.runId === input.launch.runId &&
|
|
1164
|
+
policyAcknowledgement.attempt === input.attempt &&
|
|
1165
|
+
isRuntimeTrafficPolicy(policyAcknowledgement.policy)
|
|
1166
|
+
) {
|
|
1167
|
+
input.onRuntimeTrafficPolicy?.(policyAcknowledgement.policy);
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1132
1170
|
return body.executorToken;
|
|
1133
1171
|
}
|
|
1134
1172
|
|
|
@@ -2433,13 +2471,13 @@ async function prepareAbsurdAttempt(input: {
|
|
|
2433
2471
|
}) {
|
|
2434
2472
|
return input.deps.runWithClient(
|
|
2435
2473
|
async (client) => {
|
|
2436
|
-
const
|
|
2474
|
+
const preparation = await readPostgresSchedulerAttemptPreparation(
|
|
2437
2475
|
client,
|
|
2438
2476
|
input.runId,
|
|
2439
2477
|
input.options,
|
|
2440
|
-
{ caller: 'runtime_worker_execute' },
|
|
2441
2478
|
);
|
|
2442
|
-
if (!
|
|
2479
|
+
if (!preparation) return { kind: 'missing' as const };
|
|
2480
|
+
const { launch, existing, laneState } = preparation;
|
|
2443
2481
|
|
|
2444
2482
|
const configuredBackend = input.deps.configuredRuntimeBackend;
|
|
2445
2483
|
const launchBackend = normalizePlayRuntimeBackend(
|
|
@@ -2464,11 +2502,6 @@ async function prepareAbsurdAttempt(input: {
|
|
|
2464
2502
|
);
|
|
2465
2503
|
}
|
|
2466
2504
|
|
|
2467
|
-
const existing = await readPostgresSchedulerResult(
|
|
2468
|
-
client,
|
|
2469
|
-
input.runId,
|
|
2470
|
-
input.options,
|
|
2471
|
-
);
|
|
2472
2505
|
if (
|
|
2473
2506
|
existing &&
|
|
2474
2507
|
(existing.status === 'completed' ||
|
|
@@ -2483,17 +2516,12 @@ async function prepareAbsurdAttempt(input: {
|
|
|
2483
2516
|
// One durable read decides own/adopt/defer/refuse (lane handoff). A run on a
|
|
2484
2517
|
// retired lane resumes here on the newest release; anything else foreign
|
|
2485
2518
|
// either parks for its owning lane or fails loudly.
|
|
2486
|
-
const laneState = await readPostgresSchedulerRunLaneState(
|
|
2487
|
-
client,
|
|
2488
|
-
{ runId: input.runId },
|
|
2489
|
-
input.options,
|
|
2490
|
-
);
|
|
2491
2519
|
const disposition = resolveAbsurdReleaseLaneDisposition({
|
|
2492
2520
|
runId: input.runId,
|
|
2493
2521
|
runReleaseId,
|
|
2494
2522
|
workerReleaseId,
|
|
2495
|
-
laneClaiming: laneState
|
|
2496
|
-
parkedInAdoptableWait: laneState
|
|
2523
|
+
laneClaiming: laneState.laneClaiming,
|
|
2524
|
+
parkedInAdoptableWait: laneState.parkedInAdoptableWait,
|
|
2497
2525
|
});
|
|
2498
2526
|
if (disposition.kind === 'refuse') {
|
|
2499
2527
|
// Reuse the one operator-actionable message and remediation.
|
|
@@ -2512,17 +2540,13 @@ async function prepareAbsurdAttempt(input: {
|
|
|
2512
2540
|
runId: input.runId,
|
|
2513
2541
|
adoptedReleaseId: disposition.adoptedReleaseId,
|
|
2514
2542
|
workerReleaseId,
|
|
2515
|
-
waitKind: laneState
|
|
2543
|
+
waitKind: laneState.waitKind,
|
|
2516
2544
|
}
|
|
2517
2545
|
: null;
|
|
2518
2546
|
|
|
2519
2547
|
if (launch.requiresRunCreatedProjection) {
|
|
2520
2548
|
const eventId = `run-created:${input.runId}`;
|
|
2521
|
-
const projection =
|
|
2522
|
-
client,
|
|
2523
|
-
{ runId: input.runId, eventId },
|
|
2524
|
-
input.options,
|
|
2525
|
-
);
|
|
2549
|
+
const projection = preparation.createdProjection;
|
|
2526
2550
|
if (!projection.delivered) {
|
|
2527
2551
|
// This is intentionally before allocating or assigning an attempt:
|
|
2528
2552
|
// app-runtime routes authorize artifacts against the projected run,
|
|
@@ -2547,55 +2571,69 @@ async function prepareAbsurdAttempt(input: {
|
|
|
2547
2571
|
delaySeconds: projection.blocked ? 5 : 1,
|
|
2548
2572
|
attemptAllocated: false,
|
|
2549
2573
|
});
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2574
|
+
return {
|
|
2575
|
+
kind: 'created_projection_pending' as const,
|
|
2576
|
+
launch,
|
|
2577
|
+
delaySeconds: projection.blocked ? 5 : 1,
|
|
2578
|
+
};
|
|
2554
2579
|
}
|
|
2555
2580
|
}
|
|
2556
2581
|
|
|
2557
|
-
// A
|
|
2558
|
-
//
|
|
2582
|
+
// A queued run with no persisted attempt cannot have a detached runner.
|
|
2583
|
+
// Its assignment still locks the run and rejects a racing first writer.
|
|
2584
|
+
// Every later generation keeps detached recovery's original fence.
|
|
2559
2585
|
const resumedDetached =
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
(await allocatePostgresSchedulerNextAttempt(
|
|
2568
|
-
client,
|
|
2569
|
-
input.runId,
|
|
2570
|
-
input.options,
|
|
2571
|
-
));
|
|
2572
|
-
const leaseToken = resumedDetached?.leaseToken ?? randomUUID();
|
|
2573
|
-
const assigned =
|
|
2574
|
-
resumedDetached ??
|
|
2575
|
-
(await assignPostgresSchedulerRunAttempt(
|
|
2576
|
-
client,
|
|
2577
|
-
{ runId: input.runId, attempt: generation, taskToken: leaseToken },
|
|
2578
|
-
input.options,
|
|
2579
|
-
));
|
|
2580
|
-
if (!assigned) return { kind: 'claim_rejected' as const, launch };
|
|
2586
|
+
preparation.nextAttempt === 1 && laneState.status === 'queued'
|
|
2587
|
+
? null
|
|
2588
|
+
: await resumePostgresSchedulerDetachedRunnerAttempt(
|
|
2589
|
+
client,
|
|
2590
|
+
{ runId: input.runId },
|
|
2591
|
+
input.options,
|
|
2592
|
+
);
|
|
2581
2593
|
|
|
2582
|
-
|
|
2594
|
+
if (resumedDetached) {
|
|
2595
|
+
return {
|
|
2596
|
+
kind: 'detached_recovery_claimed' as const,
|
|
2597
|
+
launch,
|
|
2598
|
+
generation: resumedDetached.attempt,
|
|
2599
|
+
leaseToken: resumedDetached.leaseToken,
|
|
2600
|
+
assigned: resumedDetached,
|
|
2601
|
+
adoptedParkedWake,
|
|
2602
|
+
};
|
|
2603
|
+
}
|
|
2604
|
+
|
|
2605
|
+
const generation = preparation.nextAttempt;
|
|
2606
|
+
const leaseToken = randomUUID();
|
|
2607
|
+
const initialRuntimeResource = isManagedSandboxRuntimeBackend(
|
|
2608
|
+
launch.runtimeBackend,
|
|
2609
|
+
)
|
|
2583
2610
|
? {
|
|
2584
|
-
kind: '
|
|
2585
|
-
launch,
|
|
2586
|
-
|
|
2587
|
-
leaseToken,
|
|
2588
|
-
assigned,
|
|
2589
|
-
adoptedParkedWake,
|
|
2611
|
+
kind: 'compute_session',
|
|
2612
|
+
runtimeBackend: launch.runtimeBackend,
|
|
2613
|
+
billingStartedAt: Date.now(),
|
|
2590
2614
|
}
|
|
2591
|
-
:
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
}
|
|
2615
|
+
: undefined;
|
|
2616
|
+
const assigned = await assignPostgresSchedulerRunAttempt(
|
|
2617
|
+
client,
|
|
2618
|
+
{
|
|
2619
|
+
runId: input.runId,
|
|
2620
|
+
attempt: generation,
|
|
2621
|
+
taskToken: leaseToken,
|
|
2622
|
+
...(initialRuntimeResource ? { initialRuntimeResource } : {}),
|
|
2623
|
+
},
|
|
2624
|
+
input.options,
|
|
2625
|
+
);
|
|
2626
|
+
if (!assigned) return { kind: 'claim_rejected' as const, launch };
|
|
2627
|
+
|
|
2628
|
+
return {
|
|
2629
|
+
kind: 'fresh_claimed' as const,
|
|
2630
|
+
launch,
|
|
2631
|
+
generation,
|
|
2632
|
+
leaseToken,
|
|
2633
|
+
assigned,
|
|
2634
|
+
computeSessionIntentPersisted: assigned.initialRuntimeResourcePersisted,
|
|
2635
|
+
adoptedParkedWake,
|
|
2636
|
+
};
|
|
2599
2637
|
},
|
|
2600
2638
|
{ runId: input.runId, phase: 'attempt_prepare' },
|
|
2601
2639
|
);
|
|
@@ -2612,7 +2650,30 @@ async function executeAbsurdAttemptAdapter(
|
|
|
2612
2650
|
schedulerOptions.schema ? { schema: schedulerOptions.schema } : {};
|
|
2613
2651
|
const runId = params.runId;
|
|
2614
2652
|
|
|
2615
|
-
|
|
2653
|
+
let prepared = await prepareAbsurdAttempt({ deps, options, runId });
|
|
2654
|
+
while (prepared.kind === 'created_projection_pending') {
|
|
2655
|
+
const delaySeconds = prepared.delaySeconds;
|
|
2656
|
+
// The preparation checkout is already released. Absurd retains a winning
|
|
2657
|
+
// ACK before park, and re-arming this same native wait closes the race at a
|
|
2658
|
+
// timeout; a fixed DeferTask would miss a signal arriving after the recheck.
|
|
2659
|
+
// Older projectors still wake at the bounded deadline and re-read their ACK.
|
|
2660
|
+
let signalled = false;
|
|
2661
|
+
try {
|
|
2662
|
+
signalled =
|
|
2663
|
+
(await ctx.awaitEvent(absurdRunCreatedProjectionEventName(runId), {
|
|
2664
|
+
timeout: delaySeconds,
|
|
2665
|
+
stepName: 'run-created-projection',
|
|
2666
|
+
})) === true;
|
|
2667
|
+
} catch (error) {
|
|
2668
|
+
if (!(error instanceof TimeoutError)) throw error;
|
|
2669
|
+
}
|
|
2670
|
+
prepared = await prepareAbsurdAttempt({ deps, options, runId });
|
|
2671
|
+
if (signalled && prepared.kind === 'created_projection_pending') {
|
|
2672
|
+
throw new Error(
|
|
2673
|
+
`Run-created wake has no durable projection acknowledgement for ${runId}.`,
|
|
2674
|
+
);
|
|
2675
|
+
}
|
|
2676
|
+
}
|
|
2616
2677
|
if (prepared.kind === 'missing') {
|
|
2617
2678
|
throw new Error(
|
|
2618
2679
|
`absurd runtime worker: no launch row for run ${runId} (schema=${
|
|
@@ -2696,18 +2757,48 @@ async function executeAbsurdAttemptAdapter(
|
|
|
2696
2757
|
// durable start projection be the single writer here.
|
|
2697
2758
|
|
|
2698
2759
|
let executionLaunch = launch;
|
|
2760
|
+
const computeSessionIntentPersisted =
|
|
2761
|
+
prepared.kind === 'fresh_claimed' && prepared.computeSessionIntentPersisted;
|
|
2762
|
+
let computeSessionInitialized = false;
|
|
2763
|
+
let runtimeTrafficPolicy: RuntimeTrafficPolicy | null = null;
|
|
2699
2764
|
try {
|
|
2700
2765
|
const tokenStartedAt = performance.now();
|
|
2701
2766
|
const executorToken = await runWithAbsurdExecutionLeaseHeartbeat({
|
|
2702
2767
|
ctx,
|
|
2703
2768
|
runId,
|
|
2704
|
-
operation: () =>
|
|
2705
|
-
|
|
2769
|
+
operation: async () => {
|
|
2770
|
+
// Own finalization before the app can open the session. A lost token
|
|
2771
|
+
// response or a later configuration failure still has a durable owner.
|
|
2772
|
+
const initializeComputeSession =
|
|
2773
|
+
prepared.kind === 'fresh_claimed' &&
|
|
2774
|
+
assigned.checkpoint == null &&
|
|
2775
|
+
assigned.suspension == null &&
|
|
2776
|
+
isManagedSandboxRuntimeBackend(launch.runtimeBackend);
|
|
2777
|
+
if (initializeComputeSession) {
|
|
2778
|
+
if (!computeSessionIntentPersisted) {
|
|
2779
|
+
throw new Error(
|
|
2780
|
+
`Postgres scheduler failed to atomically record compute session for ${runId}/${generation}.`,
|
|
2781
|
+
);
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
return deps.mintAttemptExecutorToken({
|
|
2706
2785
|
launch,
|
|
2707
2786
|
attempt: generation,
|
|
2708
2787
|
leaseToken,
|
|
2709
2788
|
schedulerSchema: options?.schema ?? null,
|
|
2710
|
-
|
|
2789
|
+
...(initializeComputeSession
|
|
2790
|
+
? {
|
|
2791
|
+
initializeComputeSession: true,
|
|
2792
|
+
onComputeSessionInitialized: () => {
|
|
2793
|
+
computeSessionInitialized = true;
|
|
2794
|
+
},
|
|
2795
|
+
onRuntimeTrafficPolicy: (policy: RuntimeTrafficPolicy) => {
|
|
2796
|
+
runtimeTrafficPolicy = policy;
|
|
2797
|
+
},
|
|
2798
|
+
}
|
|
2799
|
+
: {}),
|
|
2800
|
+
});
|
|
2801
|
+
},
|
|
2711
2802
|
}).finally(() => {
|
|
2712
2803
|
console.info(
|
|
2713
2804
|
'[absurd-runtime.worker.phase] ' +
|
|
@@ -2735,6 +2826,9 @@ async function executeAbsurdAttemptAdapter(
|
|
|
2735
2826
|
initialCheckpoint: assigned.checkpoint ?? null,
|
|
2736
2827
|
initialSuspension: assigned.suspension ?? null,
|
|
2737
2828
|
initialAttemptAlreadyWaiting: isDetachedRecovery,
|
|
2829
|
+
computeSessionIntentPersisted,
|
|
2830
|
+
computeSessionInitialized,
|
|
2831
|
+
runtimeTrafficPolicy,
|
|
2738
2832
|
runtimeApi,
|
|
2739
2833
|
});
|
|
2740
2834
|
} catch (error) {
|
|
@@ -2979,6 +3073,9 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
2979
3073
|
initialSuspension: unknown;
|
|
2980
3074
|
/** A live detached runner wake keeps its attempt parked until resolution. */
|
|
2981
3075
|
initialAttemptAlreadyWaiting: boolean;
|
|
3076
|
+
computeSessionIntentPersisted: boolean;
|
|
3077
|
+
computeSessionInitialized: boolean;
|
|
3078
|
+
runtimeTrafficPolicy: RuntimeTrafficPolicy | null;
|
|
2982
3079
|
runtimeApi: WorkerRuntimeApiContext;
|
|
2983
3080
|
}): Promise<AbsurdRuntimeResult> {
|
|
2984
3081
|
const {
|
|
@@ -2992,6 +3089,29 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
2992
3089
|
leaseToken,
|
|
2993
3090
|
runtimeApi,
|
|
2994
3091
|
} = input;
|
|
3092
|
+
const measureLaunchPhase = async <T>(
|
|
3093
|
+
phase: string,
|
|
3094
|
+
operation: () => Promise<T>,
|
|
3095
|
+
): Promise<T> => {
|
|
3096
|
+
const startedAt = performance.now();
|
|
3097
|
+
let ok = false;
|
|
3098
|
+
try {
|
|
3099
|
+
const result = await operation();
|
|
3100
|
+
ok = true;
|
|
3101
|
+
return result;
|
|
3102
|
+
} finally {
|
|
3103
|
+
console.info(
|
|
3104
|
+
'[absurd-runtime.worker.phase] ' +
|
|
3105
|
+
JSON.stringify({
|
|
3106
|
+
runId,
|
|
3107
|
+
attempt: generation,
|
|
3108
|
+
phase,
|
|
3109
|
+
totalMs: performance.now() - startedAt,
|
|
3110
|
+
ok,
|
|
3111
|
+
}),
|
|
3112
|
+
);
|
|
3113
|
+
}
|
|
3114
|
+
};
|
|
2995
3115
|
const runtimeBackend = normalizePlayRuntimeBackend(
|
|
2996
3116
|
launch.runtimeBackend || deps.configuredRuntimeBackend,
|
|
2997
3117
|
);
|
|
@@ -3809,11 +3929,15 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
3809
3929
|
}
|
|
3810
3930
|
|
|
3811
3931
|
const progressReporter = deps.createProgressReporter(launch, generation);
|
|
3812
|
-
const preparedRunnerConfig = await
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3932
|
+
const preparedRunnerConfig = await measureLaunchPhase(
|
|
3933
|
+
'runner_config_prepare',
|
|
3934
|
+
() =>
|
|
3935
|
+
runWithAbsurdExecutionLeaseHeartbeat({
|
|
3936
|
+
ctx,
|
|
3937
|
+
runId,
|
|
3938
|
+
operation: () => prepareRunnerConfig(claim),
|
|
3939
|
+
}),
|
|
3940
|
+
);
|
|
3817
3941
|
const runnerConfig = preparedRunnerConfig.runnerConfig;
|
|
3818
3942
|
|
|
3819
3943
|
let runnerResult: PlayRunnerResult;
|
|
@@ -3868,41 +3992,74 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
3868
3992
|
);
|
|
3869
3993
|
}
|
|
3870
3994
|
backendExecutions += 1;
|
|
3995
|
+
// Read the policy while preparing durable billing state, then enforce
|
|
3996
|
+
// it at the existing pre-create barrier. Observe both outcomes now so
|
|
3997
|
+
// a failed read cannot become an unhandled rejection during setup.
|
|
3998
|
+
const trafficPolicyRead =
|
|
3999
|
+
isManagedSandboxRuntimeBackend(launch.runtimeBackend) &&
|
|
4000
|
+
checkpoint === null &&
|
|
4001
|
+
pendingDetachedRunner === null
|
|
4002
|
+
? input.runtimeTrafficPolicy
|
|
4003
|
+
? Promise.resolve({
|
|
4004
|
+
ok: true as const,
|
|
4005
|
+
policy: input.runtimeTrafficPolicy,
|
|
4006
|
+
})
|
|
4007
|
+
: measureLaunchPhase('traffic_policy_read', () =>
|
|
4008
|
+
getRuntimeTrafficPolicyViaAppRuntime(runtimeApi),
|
|
4009
|
+
).then(
|
|
4010
|
+
(policy) => ({ ok: true as const, policy }),
|
|
4011
|
+
(error: unknown) => ({ ok: false as const, error }),
|
|
4012
|
+
)
|
|
4013
|
+
: null;
|
|
3871
4014
|
// Every backend profile enters through this runner-start seam. Open the
|
|
3872
4015
|
// zero-credit compute session once before any sandbox/provider work so
|
|
3873
4016
|
// later item writes can be a session-only patch and fail loudly when
|
|
3874
4017
|
// the lifecycle contract is violated.
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
// The session exists before any provider work, whereas a sandbox
|
|
3887
|
-
// resource exists only after acquisition. Persist that fact so every
|
|
3888
|
-
// opened session — including local-process and a provider-create
|
|
3889
|
-
// failure — receives durable finalization. A queued run never passes
|
|
3890
|
-
// this seam, so it still creates no zero-cost finalization job.
|
|
3891
|
-
const recordedComputeSession = await deps.runWithClient((client) =>
|
|
3892
|
-
recordPostgresSchedulerAttemptRuntimeResource(
|
|
3893
|
-
client,
|
|
3894
|
-
{
|
|
3895
|
-
runId,
|
|
3896
|
-
attempt: generation,
|
|
3897
|
-
leaseToken,
|
|
3898
|
-
resource: {
|
|
3899
|
-
kind: 'compute_session',
|
|
3900
|
-
runtimeBackend,
|
|
3901
|
-
billingStartedAt: Date.now(),
|
|
4018
|
+
if (!input.computeSessionInitialized) {
|
|
4019
|
+
await measureLaunchPhase('compute_session_open', () =>
|
|
4020
|
+
upsertComputeBillingSessionViaAppRuntime(
|
|
4021
|
+
deps.runtimeApiFromLaunch(launch),
|
|
4022
|
+
{
|
|
4023
|
+
sessionId: launch.runId,
|
|
4024
|
+
orgId: launch.orgId,
|
|
4025
|
+
...(launch.userId ? { userId: launch.userId } : {}),
|
|
4026
|
+
operation: 'workflow_run',
|
|
4027
|
+
workflowId: launch.runId,
|
|
4028
|
+
runId: launch.runId,
|
|
3902
4029
|
},
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
4030
|
+
),
|
|
4031
|
+
);
|
|
4032
|
+
}
|
|
4033
|
+
// Retain the pre-provider lease fence even when intent was recorded
|
|
4034
|
+
// before token issuance: cancellation can win during that request.
|
|
4035
|
+
// Legacy setup still records its cleanup owner here.
|
|
4036
|
+
const recordedComputeSession = await measureLaunchPhase(
|
|
4037
|
+
'compute_session_record',
|
|
4038
|
+
() =>
|
|
4039
|
+
deps.runWithClient(
|
|
4040
|
+
(client) =>
|
|
4041
|
+
input.computeSessionIntentPersisted
|
|
4042
|
+
? readPostgresSchedulerAttemptAuthorityForLease(
|
|
4043
|
+
client,
|
|
4044
|
+
{ runId, attempt: generation, leaseToken },
|
|
4045
|
+
options,
|
|
4046
|
+
).then(Boolean)
|
|
4047
|
+
: recordPostgresSchedulerAttemptRuntimeResource(
|
|
4048
|
+
client,
|
|
4049
|
+
{
|
|
4050
|
+
runId,
|
|
4051
|
+
attempt: generation,
|
|
4052
|
+
leaseToken,
|
|
4053
|
+
resource: {
|
|
4054
|
+
kind: 'compute_session',
|
|
4055
|
+
runtimeBackend,
|
|
4056
|
+
billingStartedAt: Date.now(),
|
|
4057
|
+
},
|
|
4058
|
+
},
|
|
4059
|
+
options,
|
|
4060
|
+
),
|
|
4061
|
+
{ runId, phase: 'compute_session_record' },
|
|
4062
|
+
),
|
|
3906
4063
|
);
|
|
3907
4064
|
if (!recordedComputeSession) {
|
|
3908
4065
|
throw new Error(
|
|
@@ -3933,15 +4090,12 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
3933
4090
|
// sandbox can exist. A resumed/checkpointed run may already have
|
|
3934
4091
|
// external-effect evidence, so holding it here could turn a safe
|
|
3935
4092
|
// resume into an unsafe fresh-retry decision.
|
|
3936
|
-
if (
|
|
3937
|
-
isManagedSandboxRuntimeBackend(launch.runtimeBackend) &&
|
|
3938
|
-
checkpoint === null &&
|
|
3939
|
-
pendingDetachedRunner === null
|
|
3940
|
-
) {
|
|
4093
|
+
if (trafficPolicyRead) {
|
|
3941
4094
|
let trafficPolicy: RuntimeTrafficPolicy;
|
|
3942
4095
|
try {
|
|
3943
|
-
|
|
3944
|
-
|
|
4096
|
+
const policyRead = await trafficPolicyRead;
|
|
4097
|
+
if (!policyRead.ok) throw policyRead.error;
|
|
4098
|
+
trafficPolicy = policyRead.policy;
|
|
3945
4099
|
} catch (error) {
|
|
3946
4100
|
const fallback =
|
|
3947
4101
|
fallbackRuntimeTrafficPolicyForUnavailableControlPlane(error);
|
|
@@ -7990,6 +8144,8 @@ export type AbsurdWorkerWakeHook = ((
|
|
|
7990
8144
|
wake: () => void,
|
|
7991
8145
|
) => (() => void) | void) & {
|
|
7992
8146
|
isListening: () => boolean;
|
|
8147
|
+
wake: () => boolean;
|
|
8148
|
+
idlePollFallbackMs: number;
|
|
7993
8149
|
close: () => void;
|
|
7994
8150
|
};
|
|
7995
8151
|
|
|
@@ -7997,9 +8153,7 @@ export function resolveAbsurdIdlePollFallbackMs(
|
|
|
7997
8153
|
onWake: AbsurdWorkerWakeHook | undefined,
|
|
7998
8154
|
): number | null {
|
|
7999
8155
|
if (onWake?.isListening() === true) return null;
|
|
8000
|
-
return onWake
|
|
8001
|
-
? ABSURD_ACTIVE_STANDBY_POLL_INTERVAL_MS
|
|
8002
|
-
: ABSURD_WAKE_UNAVAILABLE_POLL_INTERVAL_MS;
|
|
8156
|
+
return onWake?.idlePollFallbackMs ?? ABSURD_WAKE_UNAVAILABLE_POLL_INTERVAL_MS;
|
|
8003
8157
|
}
|
|
8004
8158
|
|
|
8005
8159
|
export function createAbsurdWorkerWakeHook(input: {
|
|
@@ -8008,6 +8162,8 @@ export function createAbsurdWorkerWakeHook(input: {
|
|
|
8008
8162
|
draining: boolean;
|
|
8009
8163
|
/** Isolated preview schemas use durable polling instead of a direct LISTEN session. */
|
|
8010
8164
|
useDirectNotify?: boolean;
|
|
8165
|
+
/** Accept local wake hints without allocating a direct database session. */
|
|
8166
|
+
enableWakeHints?: boolean;
|
|
8011
8167
|
acquireNotifyHub?: (input: {
|
|
8012
8168
|
url: string;
|
|
8013
8169
|
channel: string;
|
|
@@ -8015,12 +8171,26 @@ export function createAbsurdWorkerWakeHook(input: {
|
|
|
8015
8171
|
leaderLockKey?: string;
|
|
8016
8172
|
}) => PgNotifyHubLease | null;
|
|
8017
8173
|
}): AbsurdWorkerWakeHook | undefined {
|
|
8018
|
-
if (
|
|
8174
|
+
if (
|
|
8175
|
+
input.draining ||
|
|
8176
|
+
(input.useDirectNotify === false && !input.enableWakeHints)
|
|
8177
|
+
)
|
|
8178
|
+
return undefined;
|
|
8019
8179
|
let listening = false;
|
|
8020
8180
|
let closed = false;
|
|
8021
8181
|
const subscriptions = new Set<() => void>();
|
|
8182
|
+
const wakeHandlers = new Set<() => void>();
|
|
8022
8183
|
const hook: AbsurdWorkerWakeHook = (wake) => {
|
|
8023
8184
|
if (closed) return;
|
|
8185
|
+
wakeHandlers.add(wake);
|
|
8186
|
+
if (input.useDirectNotify === false) {
|
|
8187
|
+
const release = () => {
|
|
8188
|
+
wakeHandlers.delete(wake);
|
|
8189
|
+
subscriptions.delete(release);
|
|
8190
|
+
};
|
|
8191
|
+
subscriptions.add(release);
|
|
8192
|
+
return release;
|
|
8193
|
+
}
|
|
8024
8194
|
const acquireNotifyHub =
|
|
8025
8195
|
input.acquireNotifyHub ??
|
|
8026
8196
|
((notifyInput: {
|
|
@@ -8035,7 +8205,7 @@ export function createAbsurdWorkerWakeHook(input: {
|
|
|
8035
8205
|
applicationName: 'deepline-absurd-worker-wake',
|
|
8036
8206
|
leaderLockKey: `deepline:absurd-wake:${input.queue}`,
|
|
8037
8207
|
});
|
|
8038
|
-
if (!lease) return;
|
|
8208
|
+
if (!lease) return () => wakeHandlers.delete(wake);
|
|
8039
8209
|
const unsubscribeNotify = lease.hub.subscribe((payload) => {
|
|
8040
8210
|
if (payload === input.queue) wake();
|
|
8041
8211
|
});
|
|
@@ -8052,6 +8222,7 @@ export function createAbsurdWorkerWakeHook(input: {
|
|
|
8052
8222
|
released = true;
|
|
8053
8223
|
unsubscribeNotify();
|
|
8054
8224
|
unsubscribeState();
|
|
8225
|
+
wakeHandlers.delete(wake);
|
|
8055
8226
|
lease.release();
|
|
8056
8227
|
subscriptions.delete(release);
|
|
8057
8228
|
};
|
|
@@ -8059,10 +8230,22 @@ export function createAbsurdWorkerWakeHook(input: {
|
|
|
8059
8230
|
return release;
|
|
8060
8231
|
};
|
|
8061
8232
|
hook.isListening = () => listening;
|
|
8233
|
+
hook.wake = () => {
|
|
8234
|
+
if (closed || wakeHandlers.size === 0) return false;
|
|
8235
|
+
for (const wake of wakeHandlers) wake();
|
|
8236
|
+
return true;
|
|
8237
|
+
};
|
|
8238
|
+
// A local hint does not establish LISTEN health. Preserve the fast durable
|
|
8239
|
+
// fallback on isolated workers, including when a hint is lost or unavailable.
|
|
8240
|
+
hook.idlePollFallbackMs =
|
|
8241
|
+
input.useDirectNotify === false
|
|
8242
|
+
? ABSURD_WAKE_UNAVAILABLE_POLL_INTERVAL_MS
|
|
8243
|
+
: ABSURD_ACTIVE_STANDBY_POLL_INTERVAL_MS;
|
|
8062
8244
|
hook.close = () => {
|
|
8063
8245
|
closed = true;
|
|
8064
8246
|
listening = false;
|
|
8065
8247
|
for (const release of subscriptions) release();
|
|
8248
|
+
wakeHandlers.clear();
|
|
8066
8249
|
};
|
|
8067
8250
|
return hook;
|
|
8068
8251
|
}
|
|
@@ -8091,6 +8274,8 @@ export type RuntimeWorkerHostConfig = {
|
|
|
8091
8274
|
|
|
8092
8275
|
export type RuntimeWorkerHost = {
|
|
8093
8276
|
close(): Promise<void>;
|
|
8277
|
+
/** A lossy hint to re-read this host's durable queue, never task admission. */
|
|
8278
|
+
wake?(): boolean;
|
|
8094
8279
|
};
|
|
8095
8280
|
|
|
8096
8281
|
/** Own shutdown ordering across the primary queue, listener and maintenance.
|
|
@@ -8099,13 +8284,17 @@ export type RuntimeWorkerHost = {
|
|
|
8099
8284
|
export function createAbsurdRuntimeWorkerHostLifecycle(input: {
|
|
8100
8285
|
worker: RuntimeWorkerHost;
|
|
8101
8286
|
closeWake: () => void;
|
|
8287
|
+
wake?: () => boolean;
|
|
8102
8288
|
stopControllers: () => void;
|
|
8103
8289
|
controllers: readonly (Promise<unknown> | null)[];
|
|
8104
8290
|
}): RuntimeWorkerHost {
|
|
8105
8291
|
let closing: Promise<void> | null = null;
|
|
8292
|
+
let wake = input.wake;
|
|
8106
8293
|
return {
|
|
8294
|
+
...(wake ? { wake: () => wake?.() ?? false } : {}),
|
|
8107
8295
|
close: () => {
|
|
8108
8296
|
if (closing) return closing;
|
|
8297
|
+
wake = undefined;
|
|
8109
8298
|
const start = (operation: () => unknown): Promise<unknown> => {
|
|
8110
8299
|
try {
|
|
8111
8300
|
return Promise.resolve(operation());
|
|
@@ -8242,6 +8431,7 @@ export async function startRuntimeWorkerHost(
|
|
|
8242
8431
|
useDirectNotify: shouldUseRuntimeSchedulerDirectNotify(
|
|
8243
8432
|
schedulerOptions.schema,
|
|
8244
8433
|
),
|
|
8434
|
+
enableWakeHints: true,
|
|
8245
8435
|
});
|
|
8246
8436
|
const reportCancellationProgress = createRateLimitedRuntimeLogger({
|
|
8247
8437
|
log: (...args) => console.info(...args),
|
|
@@ -8508,6 +8698,7 @@ export async function startRuntimeWorkerHost(
|
|
|
8508
8698
|
|
|
8509
8699
|
return createAbsurdRuntimeWorkerHostLifecycle({
|
|
8510
8700
|
worker,
|
|
8701
|
+
...(onWake ? { wake: () => onWake.wake() } : {}),
|
|
8511
8702
|
// Release LISTEN before joining accepted work, so a replacement can elect
|
|
8512
8703
|
// its listener without exhausting primary scheduler sessions.
|
|
8513
8704
|
closeWake: () => onWake?.close(),
|