deepline 0.3.473 → 0.3.475
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client-core.ts +4 -1
- package/dist/bundling-sources/sdk/src/play.ts +7 -2
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/integration-contracts/billing-entitlements.ts +3 -1
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +101 -2
- package/dist/bundling-sources/shared_libs/play-runtime/modal-runtime-config.ts +62 -20
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +24 -8
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +19 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +196 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +40 -17
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal-resource-usage.ts +222 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +46 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts +9 -5
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +39 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +18 -8
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +64 -35
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +36 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +18 -8
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/sandbox-runtime-limits.ts +64 -16
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/product-notifications/billing-sales.ts +14 -1
- package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +40 -0
- package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +23 -1
- package/dist/bundling-sources/shared_libs/product-notifications/events.ts +18 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-delivery.ts +2 -1
- package/dist/cli/auth-7GVCW24N.js +41 -0
- package/dist/cli/{auth-KLJOQ4DZ.mjs → auth-CFDYJ7FZ.mjs} +5 -5
- package/dist/cli/{billing-DNDYNBQ7.mjs → billing-PEXWWLRT.mjs} +12 -10
- package/dist/cli/{billing-IQNJXKH5.js → billing-XLIL3FLK.js} +85 -83
- package/dist/cli/{chunk-4U3RNA3J.js → chunk-3MQJJ6YX.js} +51 -51
- package/dist/cli/{chunk-EUHTZV2E.mjs → chunk-3QSFSHCJ.mjs} +1 -1
- package/dist/cli/{chunk-CNCK2D6B.mjs → chunk-3WVA7E7O.mjs} +2 -2
- package/dist/cli/{chunk-NTFM7JQW.mjs → chunk-42F6LTBJ.mjs} +3 -3
- package/dist/cli/{chunk-V25CXRQ7.mjs → chunk-4QHAP3MR.mjs} +5 -5
- package/dist/cli/{chunk-3LFSIUFI.js → chunk-4QHQ3WYW.js} +21 -21
- package/dist/cli/{chunk-6ZTSHKJF.js → chunk-5QS4T4KR.js} +10 -3
- package/dist/cli/{chunk-VS7JKS22.mjs → chunk-6PRYHABU.mjs} +1 -1
- package/dist/cli/{chunk-63OOFT6B.js → chunk-7SWRXZ4A.js} +93 -93
- package/dist/cli/{chunk-6YOGOXOJ.mjs → chunk-7U3L2S7X.mjs} +10 -10
- package/dist/cli/{chunk-D4B3EV55.js → chunk-7XTBEYLS.js} +36 -36
- package/dist/cli/{chunk-QD6IW2HP.mjs → chunk-AD7KL6AI.mjs} +4 -4
- package/dist/cli/{chunk-YH5SFKHJ.js → chunk-B7NJ5JQ2.js} +4 -4
- package/dist/cli/{chunk-F4PUDYBY.js → chunk-BLW2K6YH.js} +3 -3
- package/dist/cli/{chunk-N4ZCMQ54.js → chunk-BSK2TGB4.js} +8 -8
- package/dist/cli/{chunk-QG3THSTN.mjs → chunk-BVAYLXMH.mjs} +2 -2
- package/dist/cli/{chunk-IC6AHOYS.js → chunk-CKSRTD45.js} +1 -1
- package/dist/cli/{chunk-RVRG57OB.js → chunk-D3Q4FUFS.js} +8 -8
- package/dist/cli/{chunk-ABLEU3QI.mjs → chunk-D6SXN34W.mjs} +4 -4
- package/dist/cli/{chunk-GTH7JORC.mjs → chunk-DIMKSDF6.mjs} +2 -2
- package/dist/cli/{chunk-Z52WEIBR.js → chunk-DKR7QPEO.js} +11 -7
- package/dist/cli/{chunk-YUNPYX5T.mjs → chunk-DUGZKPRK.mjs} +10 -3
- package/dist/cli/{chunk-6QS7522K.mjs → chunk-EA45HSEC.mjs} +2 -2
- package/dist/cli/{chunk-TUFQFTOC.js → chunk-F4FNMXI3.js} +53 -53
- package/dist/cli/{chunk-3OPQLFUK.mjs → chunk-FC4C7MZM.mjs} +4 -4
- package/dist/cli/{chunk-WGQKOLXH.js → chunk-G4SGYWKV.js} +6 -6
- package/dist/cli/{chunk-GISBRE2H.mjs → chunk-HALAYT3Y.mjs} +3 -3
- package/dist/cli/{chunk-GQBAJTQU.mjs → chunk-HMHH7JQD.mjs} +2 -2
- package/dist/cli/{chunk-PMJAGTSQ.js → chunk-KTWXY73Q.js} +5 -5
- package/dist/cli/{chunk-GAWP2YF5.mjs → chunk-M27SSAQ7.mjs} +1 -1
- package/dist/cli/{chunk-UON3WCBD.js → chunk-M4QO3GRD.js} +3 -3
- package/dist/cli/{chunk-36KEQFWZ.js → chunk-M7K5BCWP.js} +10 -10
- package/dist/cli/{chunk-SWAYZST7.mjs → chunk-MUDOASYL.mjs} +2 -2
- package/dist/cli/{chunk-D22QIUE3.js → chunk-NBVYS6VG.js} +9 -9
- package/dist/cli/{chunk-GE5GPLNM.mjs → chunk-NGOKBSBH.mjs} +1 -1
- package/dist/cli/{chunk-VGVSHQN5.mjs → chunk-NHQP4UPJ.mjs} +5 -5
- package/dist/cli/{chunk-3J2N7YF5.js → chunk-QKMWQTMI.js} +3 -3
- package/dist/cli/{chunk-B3BM3PAI.js → chunk-TLPW2SHS.js} +6 -6
- package/dist/cli/{chunk-QI73EL5C.js → chunk-TZRTT7EI.js} +26 -26
- package/dist/cli/{chunk-DD7E4ZVH.js → chunk-U3F63UWD.js} +7 -7
- package/dist/cli/{chunk-RZHUHL53.mjs → chunk-V4ILQ2DS.mjs} +11 -7
- package/dist/cli/{chunk-YHB6MASP.js → chunk-VBGPRRJ6.js} +2 -2
- package/dist/cli/{chunk-HQCTPAXP.mjs → chunk-VD3E3QMI.mjs} +1 -1
- package/dist/cli/{chunk-X6R5UNDZ.mjs → chunk-VFP3MFLD.mjs} +1 -1
- package/dist/cli/{chunk-EVLUSXQG.mjs → chunk-VTCXSICG.mjs} +2 -2
- package/dist/cli/{chunk-TBLBYIKO.mjs → chunk-YAAQNDP6.mjs} +1 -1
- package/dist/cli/{chunk-IFZ76E5H.mjs → chunk-YMS7VYFA.mjs} +1 -1
- package/dist/cli/{chunk-OHUGXZ7N.js → chunk-Z5VSBCFV.js} +16 -16
- package/dist/cli/{chunk-ZVLYUKJU.js → chunk-ZANUM4XV.js} +5 -5
- package/dist/cli/{chunk-MREO5LZD.js → chunk-ZQYBKDPA.js} +2 -2
- package/dist/cli/command-docs.js +2 -2
- package/dist/cli/command-docs.mjs +1 -1
- package/dist/cli/{csv-OVV3HWN4.js → csv-4YHSYV2C.js} +13 -13
- package/dist/cli/{csv-NCY5MWS2.mjs → csv-WC7ILC7P.mjs} +4 -4
- package/dist/cli/{db-BOELNDS4.js → db-APZNM2GI.js} +27 -27
- package/dist/cli/{db-VPH6JHUD.mjs → db-I5PV6NTC.mjs} +6 -6
- package/dist/cli/describe-OR7FZBCD.js +48 -0
- package/dist/cli/{describe-FFDCPYZH.mjs → describe-XH5F3X5W.mjs} +17 -17
- package/dist/cli/describe-core.d.mts +3 -3
- package/dist/cli/describe-core.d.ts +3 -3
- package/dist/cli/{executions-5GCCDENS.mjs → executions-PIY4XM7T.mjs} +6 -6
- package/dist/cli/{executions-RF44ANF5.js → executions-VIXFKZYU.js} +8 -8
- package/dist/cli/{feedback-OI2A5L6F.mjs → feedback-IGDHBWGT.mjs} +4 -4
- package/dist/cli/{feedback-NJM3V7QV.js → feedback-WCLIFQTR.js} +8 -8
- package/dist/cli/git-CKN2F32Y.js +25 -0
- package/dist/cli/{git-EOGSZ3T2.mjs → git-FRICPPTA.mjs} +5 -5
- package/dist/cli/index.js +139 -139
- package/dist/cli/index.mjs +41 -41
- package/dist/cli/{legacy-noops-5QM7XGGX.js → legacy-noops-DSFOPYDY.js} +8 -8
- package/dist/cli/{legacy-noops-UYKK7AAB.mjs → legacy-noops-ROAXEG6D.mjs} +5 -5
- package/dist/cli/{monitors-N3ITC5MP.mjs → monitors-FZ5PRLW6.mjs} +7 -7
- package/dist/cli/{monitors-52GW2V4V.js → monitors-KUCYQLU5.js} +91 -91
- package/dist/cli/{org-KUZUG33W.mjs → org-DRNND5XU.mjs} +6 -6
- package/dist/cli/{org-ADF47NPV.js → org-SRHMWKQR.js} +72 -72
- package/dist/cli/{pages-H3H7ODBN.js → pages-TEJQIHTU.js} +17 -17
- package/dist/cli/{pages-C3S5WU3S.mjs → pages-XNGA356T.mjs} +6 -6
- package/dist/cli/{play-YWZWBG32.js → play-GCGFQSFP.js} +662 -601
- package/dist/cli/{play-SZETTQPV.mjs → play-NE45ROIZ.mjs} +110 -49
- package/dist/cli/{providers-FSCD35HB.mjs → providers-57TJOSMV.mjs} +6 -6
- package/dist/cli/{providers-V3BP7ZB2.js → providers-VHMUR63R.js} +9 -9
- package/dist/cli/{quickstart-ICULVGAR.js → quickstart-BPXWO4UD.js} +6 -6
- package/dist/cli/{quickstart-SYZCT2WI.mjs → quickstart-K3SH4NXP.mjs} +4 -4
- package/dist/cli/{run-WUJSKGUM.js → run-553AAXDU.js} +36 -36
- package/dist/cli/{run-HA5B3AYQ.mjs → run-ZRWJUEN2.mjs} +22 -22
- package/dist/cli/{search-A27AGYDV.mjs → search-BW6XRS4I.mjs} +7 -7
- package/dist/cli/search-WF26AUDJ.js +32 -0
- package/dist/cli/{secrets-ODX56QBF.js → secrets-AMYQYUL5.js} +15 -15
- package/dist/cli/{secrets-2L4VRJ2F.mjs → secrets-OYYCFGZE.mjs} +6 -6
- package/dist/cli/{sessions-PRF36DHO.mjs → sessions-O4SVVOMO.mjs} +5 -5
- package/dist/cli/{sessions-DVZ4FECP.js → sessions-RYCJUMJ2.js} +6 -6
- package/dist/cli/{settings-RJT5DBWG.js → settings-7R5FQKII.js} +78 -78
- package/dist/cli/{settings-LUATIKJE.mjs → settings-QK7DHDU7.mjs} +11 -11
- package/dist/cli/{setup-FJSBKCLU.js → setup-OTQDQHDN.js} +44 -44
- package/dist/cli/{setup-IQXTYLRH.mjs → setup-VDAAVCKK.mjs} +9 -9
- package/dist/cli/{skills-XJH4ZC5Y.mjs → skills-HRK5O5JO.mjs} +7 -7
- package/dist/cli/skills-VNLMEOXW.js +31 -0
- package/dist/cli/tools-C3MUHL7G.js +55 -0
- package/dist/cli/{tools-IGXLZCKZ.mjs → tools-O7BM62QV.mjs} +16 -16
- package/dist/cli/update-TCKTU6QJ.js +33 -0
- package/dist/cli/{update-AH7G5VSL.mjs → update-UHKHCSTY.mjs} +9 -9
- package/dist/cli/{workflow-IQ2J7LWM.js → workflow-7Z5NCTGG.js} +17 -17
- package/dist/cli/{workflow-WOYSCWVD.mjs → workflow-T2EDR7L4.mjs} +7 -7
- package/dist/{client-C9bJ0QXu.d.mts → client-Bgl0_zXi.d.mts} +1 -1
- package/dist/{client-CuNH54w3.d.ts → client-_ah6IuhK.d.ts} +1 -1
- package/dist/{client-core-CSiXbe2n.d.mts → client-core-Bfsedm0_.d.mts} +44 -3
- package/dist/{client-core-C8Q6DKga.d.ts → client-core-C-ZF5vh9.d.ts} +44 -3
- package/dist/{compiler-manifest-Zwm9yqOA.d.mts → compiler-manifest-CCOqDGjU.d.mts} +12 -10
- package/dist/{compiler-manifest-Zwm9yqOA.d.ts → compiler-manifest-CCOqDGjU.d.ts} +12 -10
- package/dist/{errors-nEsqNLL8.d.mts → errors-BLccD7tG.d.mts} +1 -1
- package/dist/{errors-CsYwlpn4.d.ts → errors-Cy3kPnsi.d.ts} +1 -1
- package/dist/{index-CXF2jtza.d.mts → index-4-j44HuT.d.mts} +1 -1
- package/dist/{index-9pUZqkPH.d.ts → index-D5e4h1-H.d.ts} +1 -1
- package/dist/index.d.mts +14 -9
- package/dist/index.d.ts +14 -9
- package/dist/index.js +9 -9
- package/dist/index.mjs +6 -6
- package/dist/install-integrity.json +111 -110
- package/dist/plays/bundle-play-file.d.mts +3 -3
- package/dist/plays/bundle-play-file.d.ts +3 -3
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/dist/plays/play-description.d.mts +2 -2
- package/dist/plays/play-description.d.ts +2 -2
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/dist/server.d.mts +4 -4
- package/dist/server.d.ts +4 -4
- package/dist/server.js +5 -5
- package/dist/server.mjs +3 -3
- package/package.json +1 -1
- package/dist/cli/auth-IJOWKNSB.js +0 -41
- package/dist/cli/describe-7SFESEFL.js +0 -48
- package/dist/cli/git-4V5ZS5SE.js +0 -25
- package/dist/cli/search-XATDPNEM.js +0 -32
- package/dist/cli/skills-N6CPBF6T.js +0 -31
- package/dist/cli/tools-YMVEVRCE.js +0 -55
- package/dist/cli/update-MUCWGTXH.js +0 -33
|
@@ -18,10 +18,11 @@ import {
|
|
|
18
18
|
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
19
19
|
} from '../../runtime-constants';
|
|
20
20
|
import {
|
|
21
|
-
|
|
22
|
-
MODAL_SANDBOX_MEMORY_MIB,
|
|
21
|
+
modalSandboxResourcesForPlay,
|
|
23
22
|
loadModalClientConfig,
|
|
24
23
|
loadModalRequiredConfig,
|
|
24
|
+
resolveModalSandboxMemoryPolicy,
|
|
25
|
+
isModalPreviewEvaluation,
|
|
25
26
|
withModalCallSignal,
|
|
26
27
|
} from '../../modal-runtime-config';
|
|
27
28
|
import type { PlaySandboxRuntimeLimits } from '../../../plays/sandbox-runtime-limits';
|
|
@@ -371,10 +372,33 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
371
372
|
runtimeSchedulerSchema: config.context.runtimeSchedulerSchema,
|
|
372
373
|
limits: config.context.sandboxRuntimeLimits,
|
|
373
374
|
});
|
|
375
|
+
const resources = modalSandboxResourcesForPlay(modalConfig.limits);
|
|
376
|
+
const memoryPolicy = resolveModalSandboxMemoryPolicy({
|
|
377
|
+
limits: modalConfig.limits,
|
|
378
|
+
runtimeSchedulerSchema: config.context.runtimeSchedulerSchema,
|
|
379
|
+
});
|
|
374
380
|
const app = await modalConfig.client.apps.fromName(modalConfig.appName, {
|
|
375
381
|
createIfMissing: true,
|
|
376
382
|
});
|
|
377
|
-
const
|
|
383
|
+
const baseImage = modalConfig.client.images.fromRegistry(
|
|
384
|
+
modalConfig.image,
|
|
385
|
+
);
|
|
386
|
+
// Modal caches this immutable layer by its build commands. Build before
|
|
387
|
+
// sandbox admission so a cold Python image owns no sandbox-capacity lease
|
|
388
|
+
// or ambiguous SandboxCreate intent. Production keeps its existing image.
|
|
389
|
+
const image = isModalPreviewEvaluation({
|
|
390
|
+
runtimeSchedulerSchema: config.context.runtimeSchedulerSchema,
|
|
391
|
+
})
|
|
392
|
+
? await withModalCallSignal(
|
|
393
|
+
AbortSignal.timeout(MODAL_SANDBOX_CREATE_TIMEOUT_MS),
|
|
394
|
+
() =>
|
|
395
|
+
baseImage
|
|
396
|
+
.dockerfileCommands([
|
|
397
|
+
'RUN apt-get update && apt-get install -y --no-install-recommends python3 python-is-python3 && rm -rf /var/lib/apt/lists/*',
|
|
398
|
+
])
|
|
399
|
+
.build(app),
|
|
400
|
+
)
|
|
401
|
+
: baseImage;
|
|
378
402
|
const capacityReservation =
|
|
379
403
|
await callbacks?.reserveSandboxCapacity?.('modal');
|
|
380
404
|
const createTags = {
|
|
@@ -428,10 +452,10 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
428
452
|
createSent = true;
|
|
429
453
|
sandbox = await awaitModalSandboxCreate({
|
|
430
454
|
create: modalConfig.client.sandboxes.create(app, image, {
|
|
431
|
-
cpu:
|
|
432
|
-
cpuLimit:
|
|
433
|
-
memoryMiB:
|
|
434
|
-
memoryLimitMiB:
|
|
455
|
+
cpu: resources.physicalCpuCores,
|
|
456
|
+
cpuLimit: resources.physicalCpuCores,
|
|
457
|
+
memoryMiB: memoryPolicy.reservationMemoryMiB,
|
|
458
|
+
memoryLimitMiB: memoryPolicy.memoryLimitMiB,
|
|
435
459
|
timeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
|
|
436
460
|
idleTimeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
|
|
437
461
|
workdir: modalConfig.workdir,
|
|
@@ -473,8 +497,12 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
473
497
|
maxBillingDurationSeconds:
|
|
474
498
|
modalConfig.limits.timeoutSeconds +
|
|
475
499
|
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
476
|
-
cpu:
|
|
477
|
-
memoryGiB:
|
|
500
|
+
cpu: resources.physicalCpuCores,
|
|
501
|
+
memoryGiB: modalConfig.limits.memoryGiB,
|
|
502
|
+
...(memoryPolicy.memoryLimitMiB >
|
|
503
|
+
memoryPolicy.reservationMemoryMiB
|
|
504
|
+
? { memoryLimitGiB: memoryPolicy.memoryLimitMiB / 1024 }
|
|
505
|
+
: {}),
|
|
478
506
|
diskGiB: 0,
|
|
479
507
|
...(capacityReservation
|
|
480
508
|
? { sandboxCapacityLeaseId: capacityReservation.leaseId }
|
|
@@ -587,8 +615,11 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
587
615
|
maxBillingDurationSeconds:
|
|
588
616
|
modalConfig.limits.timeoutSeconds +
|
|
589
617
|
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
590
|
-
cpu:
|
|
591
|
-
memoryGiB:
|
|
618
|
+
cpu: resources.physicalCpuCores,
|
|
619
|
+
memoryGiB: modalConfig.limits.memoryGiB,
|
|
620
|
+
...(memoryPolicy.memoryLimitMiB > memoryPolicy.reservationMemoryMiB
|
|
621
|
+
? { memoryLimitGiB: memoryPolicy.memoryLimitMiB / 1024 }
|
|
622
|
+
: {}),
|
|
592
623
|
diskGiB: 0,
|
|
593
624
|
...(capacityReservation
|
|
594
625
|
? { sandboxCapacityLeaseId: capacityReservation.leaseId }
|
|
@@ -654,6 +685,9 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
654
685
|
},
|
|
655
686
|
bundlePromise: buildPlayRunnerBundle(),
|
|
656
687
|
config: withRunnerSandboxProvider(config, 'modal'),
|
|
688
|
+
...(memoryPolicy.memoryLimitMiB > memoryPolicy.reservationMemoryMiB
|
|
689
|
+
? { memoryLimitMiB: memoryPolicy.memoryLimitMiB }
|
|
690
|
+
: {}),
|
|
657
691
|
workDir: modalConfig.workdir,
|
|
658
692
|
startedAt,
|
|
659
693
|
emitStage: (stage, extra) =>
|
|
@@ -972,9 +1006,7 @@ export async function inspectDetachedModalRunner(input: {
|
|
|
972
1006
|
const deadline = new AbortController();
|
|
973
1007
|
const timer = setTimeout(() => {
|
|
974
1008
|
deadline.abort(
|
|
975
|
-
new Error(
|
|
976
|
-
`Modal detached-runner inspection exceeded ${timeoutMs}ms.`,
|
|
977
|
-
),
|
|
1009
|
+
new Error(`Modal detached-runner inspection exceeded ${timeoutMs}ms.`),
|
|
978
1010
|
);
|
|
979
1011
|
}, timeoutMs);
|
|
980
1012
|
try {
|
|
@@ -41,6 +41,8 @@ export type PlayRunnerRuntimeResource = {
|
|
|
41
41
|
terminalAt?: number | null;
|
|
42
42
|
cpu?: number | null;
|
|
43
43
|
memoryGiB?: number | null;
|
|
44
|
+
/** Optional Modal burst ceiling; absence retains the fixed reservation. */
|
|
45
|
+
memoryLimitGiB?: number | null;
|
|
44
46
|
diskGiB?: number | null;
|
|
45
47
|
/** Scheduler-only soft-capacity reservation made before provider create. */
|
|
46
48
|
sandboxCapacityLeaseId?: string;
|
|
@@ -362,16 +362,20 @@ export const RUNTIME_INCIDENT_DRILL_CATALOG: readonly RuntimeIncidentDrill[] = [
|
|
|
362
362
|
topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
|
|
363
363
|
control: {
|
|
364
364
|
kind: 'runtime_test_fault',
|
|
365
|
-
// The 120-second lease plus 80 seconds of confirmation
|
|
366
|
-
// provider read
|
|
367
|
-
// that read
|
|
368
|
-
|
|
365
|
+
// The 120-second lease plus 80 seconds of on-time confirmation is the
|
|
366
|
+
// earliest provider read. Preview woke late enough to exhaust 300
|
|
367
|
+
// seconds of silence before that read. Keep the synthetic runner quiet
|
|
368
|
+
// for seven minutes without changing the worker's liveness policy.
|
|
369
|
+
defaultValue: 420_000,
|
|
369
370
|
valueMeaning: 'milliseconds',
|
|
370
371
|
},
|
|
371
372
|
expected: {
|
|
372
373
|
outcome:
|
|
373
374
|
'The worker asks the provider once the lease and confirmation pass, the sandbox is not proven gone, and the same attempt finishes after heartbeats return.',
|
|
374
|
-
safetyFacts: [
|
|
375
|
+
safetyFacts: [
|
|
376
|
+
'same_logical_run',
|
|
377
|
+
'no_loss_verdict_without_provider_proof',
|
|
378
|
+
],
|
|
375
379
|
},
|
|
376
380
|
verification: {
|
|
377
381
|
kind: 'preview_black_box',
|
|
@@ -23,6 +23,7 @@ export function resolveRuntimeSandboxComputeItems(input: {
|
|
|
23
23
|
endedAt: number | null;
|
|
24
24
|
physicalCpuCores: number | null;
|
|
25
25
|
memoryGiB: number | null;
|
|
26
|
+
memoryLimitGiB: number | null;
|
|
26
27
|
maxBillingDurationSeconds: number | null;
|
|
27
28
|
}
|
|
28
29
|
>();
|
|
@@ -52,6 +53,10 @@ export function resolveRuntimeSandboxComputeItems(input: {
|
|
|
52
53
|
finiteNonNegative(value.cpu) ?? existing?.physicalCpuCores ?? null,
|
|
53
54
|
memoryGiB:
|
|
54
55
|
finiteNonNegative(value.memoryGiB) ?? existing?.memoryGiB ?? null,
|
|
56
|
+
memoryLimitGiB:
|
|
57
|
+
finiteNonNegative(value.memoryLimitGiB) ??
|
|
58
|
+
existing?.memoryLimitGiB ??
|
|
59
|
+
null,
|
|
55
60
|
maxBillingDurationSeconds:
|
|
56
61
|
finiteNonNegative(value.maxBillingDurationSeconds) ??
|
|
57
62
|
existing?.maxBillingDurationSeconds ??
|
|
@@ -77,6 +82,9 @@ export function resolveRuntimeSandboxComputeItems(input: {
|
|
|
77
82
|
wallTimeSeconds: Math.max(0, endedAt - resource.startedAt) / 1_000,
|
|
78
83
|
physicalCpuCores: resource.physicalCpuCores ?? 0.5,
|
|
79
84
|
memoryGiB: resource.memoryGiB ?? 1,
|
|
85
|
+
...(resource.memoryLimitGiB === null
|
|
86
|
+
? {}
|
|
87
|
+
: { memoryLimitGiB: resource.memoryLimitGiB }),
|
|
80
88
|
sandboxId,
|
|
81
89
|
startedAt: resource.startedAt,
|
|
82
90
|
endedAt,
|
|
@@ -204,6 +204,8 @@ export type SandboxExecutionResource = SandboxRunResource & {
|
|
|
204
204
|
billingEndedAtMs?: number | null;
|
|
205
205
|
cpu?: number | null;
|
|
206
206
|
memoryGiB?: number | null;
|
|
207
|
+
/** Optional provider burst ceiling, distinct from the reserved memory. */
|
|
208
|
+
memoryLimitGiB?: number | null;
|
|
207
209
|
diskGiB?: number | null;
|
|
208
210
|
/** A create settled after the launching attempt had already timed out. */
|
|
209
211
|
lateAcquired?: boolean;
|
package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts
CHANGED
|
@@ -254,6 +254,7 @@ function toSandboxResource(
|
|
|
254
254
|
maxBillingDurationSeconds: resource.maxBillingDurationSeconds ?? null,
|
|
255
255
|
cpu: resource.cpu ?? null,
|
|
256
256
|
memoryGiB: resource.memoryGiB ?? null,
|
|
257
|
+
memoryLimitGiB: resource.memoryLimitGiB ?? null,
|
|
257
258
|
diskGiB: resource.diskGiB ?? null,
|
|
258
259
|
// Historical Daytona facts use `daytonaEnvironment`; normalize it at the
|
|
259
260
|
// portable boundary so the cleanup job stays in the resource's actual
|
|
@@ -295,6 +296,7 @@ export function playRunnerRuntimeResourceFromSandboxExecutionResource(
|
|
|
295
296
|
maxBillingDurationSeconds: resource.maxBillingDurationSeconds ?? null,
|
|
296
297
|
cpu: resource.cpu ?? null,
|
|
297
298
|
memoryGiB: resource.memoryGiB ?? null,
|
|
299
|
+
memoryLimitGiB: resource.memoryLimitGiB ?? null,
|
|
298
300
|
diskGiB: resource.diskGiB ?? null,
|
|
299
301
|
...(resource.capacityLeaseId
|
|
300
302
|
? { sandboxCapacityLeaseId: resource.capacityLeaseId }
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts
CHANGED
|
@@ -78,29 +78,51 @@ export function acquireRateScopeSql(
|
|
|
78
78
|
const observationWasIdle = `${successesParameter}::int = 0
|
|
79
79
|
AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
|
|
80
80
|
> GREATEST(1000, 2000.0 / ${refillRate})`;
|
|
81
|
-
// A denied refill may fold a completion before the next
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
const recoverySuccess =
|
|
86
|
-
|
|
87
|
-
? `(${successesParameter}::int > 0 OR (
|
|
81
|
+
// A denied refill may fold a completion before the next recovery interval.
|
|
82
|
+
// Keep it available for one later ramp rather than requiring another call at
|
|
83
|
+
// the old pace. The baseline consumes it once. Unknown-provider upward probes
|
|
84
|
+
// above useful seed capacity retain their separate observation policy.
|
|
85
|
+
const recoverySuccess = probeBaselineSupported
|
|
86
|
+
? `(${successesParameter}::int > 0 OR (
|
|
88
87
|
h.seen_backpressure
|
|
89
|
-
AND ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
88
|
+
AND (${!unknownProvider} OR ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS})
|
|
90
89
|
AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms
|
|
91
90
|
AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms
|
|
92
91
|
AND h.rate_successes > h.rate_probe_successes_at_ramp
|
|
93
92
|
AND NOT (${observationWasIdle})
|
|
94
93
|
))`
|
|
95
|
-
|
|
94
|
+
: `${successesParameter}::int > 0`;
|
|
95
|
+
// Known-provider feedback can arrive before its recovery interval. Wake at
|
|
96
|
+
// that opportunity instead of sleeping an entire old, slow token interval;
|
|
97
|
+
// the baseline below consumes each confirmed success for at most one ramp.
|
|
98
|
+
const pendingRecoveryWaitMs =
|
|
99
|
+
!unknownProvider && probeBaselineSupported
|
|
100
|
+
? `CASE WHEN ${recoverySuccess}
|
|
101
|
+
AND h.seen_backpressure
|
|
102
|
+
AND h.rate_observation_started_at_ms IS NOT NULL
|
|
103
|
+
AND GREATEST(h.retry_at_ms, h.cool_until_ms) <= nv.now_ms
|
|
104
|
+
AND nv.now_ms < h.last_ramp_at_ms + ${backpressuredRecoveryInterval}
|
|
105
|
+
THEN h.last_ramp_at_ms + ${backpressuredRecoveryInterval} - nv.now_ms
|
|
106
|
+
ELSE NULL END`
|
|
107
|
+
: 'NULL::bigint';
|
|
108
|
+
// A provider's elapsed hold authorizes one shared recovery probe. Requiring
|
|
109
|
+
// a whole token at the newly learned rate would add another wait when the
|
|
110
|
+
// measured quota cycle is longer than Retry-After itself. The first grant
|
|
111
|
+
// starts an observation epoch, so competing callers cannot reuse this floor.
|
|
112
|
+
const postHoldProbeDue = `h.seen_backpressure
|
|
113
|
+
AND h.rate_observation_started_at_ms IS NULL
|
|
114
|
+
AND h.retry_at_ms > 0
|
|
115
|
+
AND GREATEST(h.retry_at_ms, h.cool_until_ms) <= nv.now_ms`;
|
|
96
116
|
return `provider_hold_rows AS (
|
|
97
117
|
SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
|
|
98
118
|
), provider_hold AS (
|
|
99
119
|
SELECT h.*,
|
|
100
|
-
|
|
120
|
+
${postHoldProbeDue} AS post_hold_probe_due,
|
|
121
|
+
${pendingRecoveryWaitMs} AS pending_recovery_wait_ms,
|
|
122
|
+
GREATEST(CASE WHEN ${postHoldProbeDue} THEN 1.0 ELSE 0.0 END, LEAST(
|
|
101
123
|
GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
|
|
102
124
|
h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
|
|
103
|
-
) AS tokens_after_refill,
|
|
125
|
+
)) AS tokens_after_refill,
|
|
104
126
|
${observationWasIdle} AS observation_was_idle,
|
|
105
127
|
CASE WHEN ${recoverySuccess}
|
|
106
128
|
AND h.rate_observation_started_at_ms IS NOT NULL
|
|
@@ -140,7 +162,9 @@ export function updateRateScopeSql(
|
|
|
140
162
|
const floorRampedObservation = `${unknownProvider} AND ph.seen_backpressure
|
|
141
163
|
AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
142
164
|
AND ph.next_refill_per_sec > ph.refill_per_sec`;
|
|
143
|
-
const
|
|
165
|
+
const knownRampedObservation = `${!unknownProvider} AND ph.seen_backpressure
|
|
166
|
+
AND ph.next_refill_per_sec > ph.refill_per_sec`;
|
|
167
|
+
const idleFloorObservation = `${unknownProvider} AND ph.observation_was_idle
|
|
144
168
|
AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
|
|
145
169
|
const rebaseProbe =
|
|
146
170
|
'ph.rate_probe_started_at_ms < ph.last_ramp_at_ms OR ph.rate_probe_started_at_ms < COALESCE(ph.rate_observation_started_at_ms, 0) OR ph.rate_probe_successes_at_ramp > ph.rate_successes';
|
|
@@ -163,16 +187,16 @@ export function updateRateScopeSql(
|
|
|
163
187
|
WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
|
|
164
188
|
ELSE ph.rate_successes END,
|
|
165
189
|
${
|
|
166
|
-
|
|
190
|
+
probeBaselineSupported
|
|
167
191
|
? `rate_probe_successes_at_ramp = CASE
|
|
168
192
|
WHEN ${recoveredObservation} THEN 0
|
|
169
193
|
WHEN (SELECT granted FROM grant_calc) > 0
|
|
170
194
|
AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN 0
|
|
171
195
|
WHEN ${idleFloorObservation} THEN ph.rate_successes
|
|
172
|
-
WHEN ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
|
|
196
|
+
WHEN ${knownRampedObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
|
|
173
197
|
ELSE ph.rate_probe_successes_at_ramp END,
|
|
174
198
|
rate_probe_started_at_ms = CASE
|
|
175
|
-
WHEN ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
|
|
199
|
+
WHEN ${knownRampedObservation} OR ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
|
|
176
200
|
WHEN (SELECT granted FROM grant_calc) > 0
|
|
177
201
|
AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN nv.now_ms
|
|
178
202
|
ELSE ph.rate_probe_started_at_ms END,`
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts
CHANGED
|
@@ -1444,7 +1444,12 @@ async function acquirePureRate(
|
|
|
1444
1444
|
LEAST(
|
|
1445
1445
|
${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)},
|
|
1446
1446
|
ph.tokens_after_refill,
|
|
1447
|
-
|
|
1447
|
+
GREATEST(CASE WHEN ph.post_hold_probe_due
|
|
1448
|
+
AND LEAST(b.tokens, sc.capacity)
|
|
1449
|
+
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1450
|
+
* CASE WHEN sc.adaptive_max_configured
|
|
1451
|
+
THEN sc.max_rps ELSE b.max_rps END >= 1
|
|
1452
|
+
THEN 1.0 ELSE 0.0 END, LEAST(b.tokens, sc.capacity)
|
|
1448
1453
|
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1449
1454
|
* LEAST(
|
|
1450
1455
|
CASE
|
|
@@ -1452,7 +1457,7 @@ async function acquirePureRate(
|
|
|
1452
1457
|
ELSE b.max_rps
|
|
1453
1458
|
END,
|
|
1454
1459
|
GREATEST(sc.min_rps, ph.refill_per_sec)
|
|
1455
|
-
)
|
|
1460
|
+
))
|
|
1456
1461
|
) AS tokens_after_refill
|
|
1457
1462
|
FROM ${bucketTableName(input)} b
|
|
1458
1463
|
JOIN seed_config sc ON sc.rule_id = b.rule_id
|
|
@@ -1683,7 +1688,7 @@ async function acquirePureRate(
|
|
|
1683
1688
|
(SELECT cool_until_ms FROM cooling),
|
|
1684
1689
|
(SELECT retry_at_ms FROM cooling)
|
|
1685
1690
|
) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
|
|
1686
|
-
ELSE (
|
|
1691
|
+
ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
|
|
1687
1692
|
SELECT CEIL(
|
|
1688
1693
|
MAX(
|
|
1689
1694
|
(
|
|
@@ -1699,7 +1704,7 @@ async function acquirePureRate(
|
|
|
1699
1704
|
)
|
|
1700
1705
|
)::bigint
|
|
1701
1706
|
FROM updated
|
|
1702
|
-
)
|
|
1707
|
+
))
|
|
1703
1708
|
END AS wait_ms
|
|
1704
1709
|
`,
|
|
1705
1710
|
[
|
|
@@ -1964,7 +1969,12 @@ async function grantWithConcurrency(
|
|
|
1964
1969
|
LEAST(
|
|
1965
1970
|
${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)},
|
|
1966
1971
|
ph.tokens_after_refill,
|
|
1967
|
-
|
|
1972
|
+
GREATEST(CASE WHEN ph.post_hold_probe_due
|
|
1973
|
+
AND LEAST(b.tokens, sc.capacity)
|
|
1974
|
+
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1975
|
+
* CASE WHEN sc.adaptive_max_configured
|
|
1976
|
+
THEN sc.max_rps ELSE b.max_rps END >= 1
|
|
1977
|
+
THEN 1.0 ELSE 0.0 END, LEAST(b.tokens, sc.capacity)
|
|
1968
1978
|
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1969
1979
|
* LEAST(
|
|
1970
1980
|
CASE
|
|
@@ -1972,7 +1982,7 @@ async function grantWithConcurrency(
|
|
|
1972
1982
|
ELSE b.max_rps
|
|
1973
1983
|
END,
|
|
1974
1984
|
GREATEST(sc.min_rps, ph.refill_per_sec)
|
|
1975
|
-
)
|
|
1985
|
+
))
|
|
1976
1986
|
) AS tokens_after_refill
|
|
1977
1987
|
FROM ${bucketTableName(input)} b
|
|
1978
1988
|
JOIN seed_config sc ON sc.rule_id = b.rule_id
|
|
@@ -2306,7 +2316,7 @@ async function grantWithConcurrency(
|
|
|
2306
2316
|
(SELECT retry_at_ms FROM cooling)
|
|
2307
2317
|
) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
|
|
2308
2318
|
WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
|
|
2309
|
-
ELSE (
|
|
2319
|
+
ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
|
|
2310
2320
|
SELECT CEIL(
|
|
2311
2321
|
MAX(
|
|
2312
2322
|
(
|
|
@@ -2323,7 +2333,7 @@ async function grantWithConcurrency(
|
|
|
2323
2333
|
)
|
|
2324
2334
|
)::bigint
|
|
2325
2335
|
FROM updated
|
|
2326
|
-
)
|
|
2336
|
+
))
|
|
2327
2337
|
END AS wait_ms
|
|
2328
2338
|
`,
|
|
2329
2339
|
[
|
|
@@ -16389,7 +16389,6 @@ export async function executePostgresSchedulerClaim(
|
|
|
16389
16389
|
execute: PostgresSchedulerAttemptExecutor,
|
|
16390
16390
|
options: PostgresSchedulerClaimExecutionOptions = {},
|
|
16391
16391
|
): Promise<PostgresSchedulerWorkerOnceResult> {
|
|
16392
|
-
let runtimeResources: unknown[] = [];
|
|
16393
16392
|
let runnerFailureResult: Extract<
|
|
16394
16393
|
PlayRunnerResult,
|
|
16395
16394
|
{ status: 'failed' }
|
|
@@ -16445,11 +16444,6 @@ export async function executePostgresSchedulerClaim(
|
|
|
16445
16444
|
},
|
|
16446
16445
|
options,
|
|
16447
16446
|
);
|
|
16448
|
-
runtimeResources = await readPostgresSchedulerRuntimeResourcesForRun(
|
|
16449
|
-
client,
|
|
16450
|
-
{ runId: claim.runId },
|
|
16451
|
-
options,
|
|
16452
|
-
);
|
|
16453
16447
|
// A PlayRunnerResult envelope ({status, output, logs, steps, ...}) is the
|
|
16454
16448
|
// executor's transport shape; the run's terminal result — what the ledger
|
|
16455
16449
|
// run.completed event and `runs get` expose — is the play's own output.
|
|
@@ -16587,31 +16581,9 @@ export async function executePostgresSchedulerClaim(
|
|
|
16587
16581
|
`Postgres scheduler worker failed after losing lease for ${claim.runId}/${claim.attempt}: ${message}`,
|
|
16588
16582
|
);
|
|
16589
16583
|
}
|
|
16590
|
-
|
|
16591
|
-
|
|
16592
|
-
|
|
16593
|
-
{
|
|
16594
|
-
runId: claim.runId,
|
|
16595
|
-
billingEndedAt: runnerTerminalReceivedAtMs ?? Date.now(),
|
|
16596
|
-
terminalReason,
|
|
16597
|
-
},
|
|
16598
|
-
options,
|
|
16599
|
-
).catch((settlementError) => {
|
|
16600
|
-
console.warn('[postgres-scheduler.resource_settlement_failed]', {
|
|
16601
|
-
runId: claim.runId,
|
|
16602
|
-
terminalReason,
|
|
16603
|
-
error:
|
|
16604
|
-
settlementError instanceof Error
|
|
16605
|
-
? settlementError.message
|
|
16606
|
-
: String(settlementError),
|
|
16607
|
-
});
|
|
16608
|
-
});
|
|
16609
|
-
runtimeResources = await readPostgresSchedulerRuntimeResourcesForRun(
|
|
16610
|
-
client,
|
|
16611
|
-
{ runId: claim.runId },
|
|
16612
|
-
options,
|
|
16613
|
-
).catch(() => []);
|
|
16614
|
-
}
|
|
16584
|
+
// The failure statement already stamps every sandbox and persists its
|
|
16585
|
+
// cleanup/billing obligations. Return to terminal projection without
|
|
16586
|
+
// reacquiring custody locks or reading resources that this caller never uses.
|
|
16615
16587
|
return {
|
|
16616
16588
|
status: 'failed',
|
|
16617
16589
|
runId: claim.runId,
|
|
@@ -19477,6 +19449,27 @@ export async function readPostgresSchedulerTerminalArrayInputIntent(
|
|
|
19477
19449
|
return readTerminalArrayInputIntentForScope(client, input, options);
|
|
19478
19450
|
}
|
|
19479
19451
|
|
|
19452
|
+
/**
|
|
19453
|
+
* Prove delivery before exposing the same run's terminal cleanup intent.
|
|
19454
|
+
* Fold the optional-column probe into that proof, preserving old schemas
|
|
19455
|
+
* without another round trip or copying the wide run tuple to JSON.
|
|
19456
|
+
*/
|
|
19457
|
+
export async function assertPostgresRunLedgerDeliveryAndReadTerminalArrayInputIntent(
|
|
19458
|
+
client: PostgresSchedulerQueryClient,
|
|
19459
|
+
input: Parameters<typeof assertPostgresRunLedgerDelivery>[1],
|
|
19460
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19461
|
+
): Promise<{ playName: string | null; started: boolean }> {
|
|
19462
|
+
const markerPresent = await provePostgresRunLedgerDelivery(
|
|
19463
|
+
client,
|
|
19464
|
+
input,
|
|
19465
|
+
options,
|
|
19466
|
+
true,
|
|
19467
|
+
);
|
|
19468
|
+
return markerPresent
|
|
19469
|
+
? readTerminalArrayInputIntentRow(client, input, options)
|
|
19470
|
+
: { playName: null, started: false };
|
|
19471
|
+
}
|
|
19472
|
+
|
|
19480
19473
|
/** Internal dispatch already owns the run's outbox lease; receiver reads also bind its org. */
|
|
19481
19474
|
async function readTerminalArrayInputIntentForScope(
|
|
19482
19475
|
client: PostgresSchedulerQueryClient,
|
|
@@ -19499,6 +19492,14 @@ async function readTerminalArrayInputIntentForScope(
|
|
|
19499
19492
|
if (column.rows[0]?.present !== true) {
|
|
19500
19493
|
return { playName: null, started: false };
|
|
19501
19494
|
}
|
|
19495
|
+
return readTerminalArrayInputIntentRow(client, input, options);
|
|
19496
|
+
}
|
|
19497
|
+
|
|
19498
|
+
async function readTerminalArrayInputIntentRow(
|
|
19499
|
+
client: PostgresSchedulerQueryClient,
|
|
19500
|
+
input: { orgId?: string; runId: string },
|
|
19501
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19502
|
+
): Promise<{ playName: string | null; started: boolean }> {
|
|
19502
19503
|
const run = await client.query<{
|
|
19503
19504
|
array_input_started: boolean;
|
|
19504
19505
|
play_name: string;
|
|
@@ -19529,6 +19530,15 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
19529
19530
|
},
|
|
19530
19531
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19531
19532
|
): Promise<void> {
|
|
19533
|
+
await provePostgresRunLedgerDelivery(client, input, options);
|
|
19534
|
+
}
|
|
19535
|
+
|
|
19536
|
+
async function provePostgresRunLedgerDelivery(
|
|
19537
|
+
client: PostgresSchedulerQueryClient,
|
|
19538
|
+
input: Parameters<typeof assertPostgresRunLedgerDelivery>[1],
|
|
19539
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19540
|
+
readArrayInputMarker = false,
|
|
19541
|
+
): Promise<boolean> {
|
|
19532
19542
|
const result = await client.query<{
|
|
19533
19543
|
delivery_key: string;
|
|
19534
19544
|
event_type: string;
|
|
@@ -19539,6 +19549,7 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
19539
19549
|
blocked: boolean;
|
|
19540
19550
|
payload_json: unknown;
|
|
19541
19551
|
created_at: Date;
|
|
19552
|
+
array_input_marker_present?: boolean;
|
|
19542
19553
|
}>(
|
|
19543
19554
|
`
|
|
19544
19555
|
SELECT outbox.delivery_key,
|
|
@@ -19551,6 +19562,16 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
19551
19562
|
outbox.failed_at IS NOT NULL AS blocked,
|
|
19552
19563
|
outbox.payload_json,
|
|
19553
19564
|
outbox.created_at
|
|
19565
|
+
${
|
|
19566
|
+
readArrayInputMarker
|
|
19567
|
+
? `, EXISTS (
|
|
19568
|
+
SELECT 1 FROM pg_attribute
|
|
19569
|
+
WHERE attrelid = to_regclass($3)
|
|
19570
|
+
AND attname = 'array_input_started'
|
|
19571
|
+
AND NOT attisdropped
|
|
19572
|
+
) AS array_input_marker_present`
|
|
19573
|
+
: ''
|
|
19574
|
+
}
|
|
19554
19575
|
FROM ${tableName('outbox', options)} AS outbox
|
|
19555
19576
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
|
|
19556
19577
|
WHERE outbox.run_id = $1 AND runs.org_id = $2
|
|
@@ -19562,7 +19583,9 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
19562
19583
|
-- their own delivery without changing the drain's convergence behavior.
|
|
19563
19584
|
LIMIT ${RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS}
|
|
19564
19585
|
`,
|
|
19565
|
-
|
|
19586
|
+
readArrayInputMarker
|
|
19587
|
+
? [input.runId, input.orgId, tableName('runs', options)]
|
|
19588
|
+
: [input.runId, input.orgId],
|
|
19566
19589
|
);
|
|
19567
19590
|
const rows = result.rows;
|
|
19568
19591
|
const leaseOwner = input.leaseOwner?.trim() || rows[0]?.lease_owner;
|
|
@@ -19656,7 +19679,8 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
19656
19679
|
input.idempotencyKey === batchKey ||
|
|
19657
19680
|
legacySingletonKey)
|
|
19658
19681
|
) {
|
|
19659
|
-
if (input.projectedEvents === undefined)
|
|
19682
|
+
if (input.projectedEvents === undefined)
|
|
19683
|
+
return rows[0]?.array_input_marker_present === true;
|
|
19660
19684
|
const prefixEvents: PlayRunLedgerIngressEvent[] = [];
|
|
19661
19685
|
for (const prefixRow of prefix) {
|
|
19662
19686
|
prefixEvents.push(
|
|
@@ -19696,7 +19720,8 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
19696
19720
|
expectedEventCandidates: expectedProjectionCandidates,
|
|
19697
19721
|
...(input.projectionChunk ? { chunk: input.projectionChunk } : {}),
|
|
19698
19722
|
});
|
|
19699
|
-
if (projectionMismatch === null)
|
|
19723
|
+
if (projectionMismatch === null)
|
|
19724
|
+
return rows[0]?.array_input_marker_present === true;
|
|
19700
19725
|
throw new RunLedgerProjectionLeaseLostError(
|
|
19701
19726
|
'projected_events_mismatch',
|
|
19702
19727
|
projectionMismatch,
|
|
@@ -24428,7 +24453,11 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
24428
24453
|
waitForRunLock: true,
|
|
24429
24454
|
},
|
|
24430
24455
|
options,
|
|
24431
|
-
{
|
|
24456
|
+
{
|
|
24457
|
+
eventIds,
|
|
24458
|
+
targetEventId,
|
|
24459
|
+
createdProjectionWake: input.createdProjectionWake,
|
|
24460
|
+
},
|
|
24432
24461
|
);
|
|
24433
24462
|
if (fast !== null) {
|
|
24434
24463
|
let continuation: RunLedgerOutboxDrainScope | null = null;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { PlayDocflowNodeIoState } from './docflow-node-io';
|
|
2
2
|
import type { PlayMapRowActivitySource } from './run-ledger';
|
|
3
3
|
import type { PlayConditionCounts } from './cell-provenance';
|
|
4
|
+
import { PLAY_SANDBOX_SIZE_LIMITS } from '../plays/sandbox-runtime-limits';
|
|
4
5
|
|
|
5
6
|
export type PlayStagedFileRef = {
|
|
6
7
|
storageKind: 'r2';
|
|
@@ -69,6 +70,17 @@ export const MODAL_SANDBOX_COMPUTE_PRICING_USD = {
|
|
|
69
70
|
// Convex billing can use it without importing Modal's Node runtime config.
|
|
70
71
|
export const MODAL_SANDBOX_CPU_CORES = 0.5;
|
|
71
72
|
|
|
73
|
+
/** Scale Modal's physical-core request from Deepline's standard one-vCPU unit. */
|
|
74
|
+
export function modalSandboxResourcesForPlay(input: {
|
|
75
|
+
cpu: number;
|
|
76
|
+
memoryGiB: number;
|
|
77
|
+
}) {
|
|
78
|
+
return {
|
|
79
|
+
physicalCpuCores: MODAL_SANDBOX_CPU_CORES * input.cpu,
|
|
80
|
+
memoryMiB: input.memoryGiB * 1024,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
72
84
|
const NEON_COMPUTE_PRICING_USD = {
|
|
73
85
|
cuHour: 0.106,
|
|
74
86
|
defaultCu: 1,
|
|
@@ -78,6 +90,14 @@ function roundUsd(amount: number): number {
|
|
|
78
90
|
return Number(Math.max(0, amount).toFixed(12));
|
|
79
91
|
}
|
|
80
92
|
|
|
93
|
+
/** Portable billing identity derived from the resources actually acquired. */
|
|
94
|
+
function sandboxSizeMetadata(cpu: number, memoryGiB: number) {
|
|
95
|
+
const entry = Object.entries(PLAY_SANDBOX_SIZE_LIMITS).find(
|
|
96
|
+
([, limits]) => limits.cpu === cpu && limits.memoryGiB === memoryGiB,
|
|
97
|
+
);
|
|
98
|
+
return entry ? { sandboxSize: entry[0] } : {};
|
|
99
|
+
}
|
|
100
|
+
|
|
81
101
|
export function resolveDaytonaSandboxComputeItem(input: {
|
|
82
102
|
itemId: string;
|
|
83
103
|
source?: string;
|
|
@@ -110,6 +130,7 @@ export function resolveDaytonaSandboxComputeItem(input: {
|
|
|
110
130
|
units: billableSeconds,
|
|
111
131
|
providerCostUsd,
|
|
112
132
|
metadata: {
|
|
133
|
+
...sandboxSizeMetadata(cpu, memoryGiB),
|
|
113
134
|
sandboxId: input.sandboxId ?? null,
|
|
114
135
|
cpu,
|
|
115
136
|
memoryGiB,
|
|
@@ -126,6 +147,8 @@ export function resolveModalSandboxComputeItem(input: {
|
|
|
126
147
|
wallTimeSeconds: number;
|
|
127
148
|
physicalCpuCores: number;
|
|
128
149
|
memoryGiB: number;
|
|
150
|
+
/** Bounded burst allocation; absent on fixed-resource historical items. */
|
|
151
|
+
memoryLimitGiB?: number;
|
|
129
152
|
sandboxId?: string | null;
|
|
130
153
|
startedAt?: number;
|
|
131
154
|
endedAt?: number;
|
|
@@ -145,9 +168,22 @@ export function resolveModalSandboxComputeItem(input: {
|
|
|
145
168
|
units: billableSeconds,
|
|
146
169
|
providerCostUsd,
|
|
147
170
|
metadata: {
|
|
171
|
+
...sandboxSizeMetadata(
|
|
172
|
+
physicalCpuCores / MODAL_SANDBOX_CPU_CORES,
|
|
173
|
+
memoryGiB,
|
|
174
|
+
),
|
|
148
175
|
sandboxId: input.sandboxId ?? null,
|
|
149
176
|
physicalCpuCores,
|
|
150
177
|
memoryGiB,
|
|
178
|
+
...(input.memoryLimitGiB === undefined
|
|
179
|
+
? {}
|
|
180
|
+
: {
|
|
181
|
+
memoryLimitGiB: input.memoryLimitGiB,
|
|
182
|
+
// The Preview evaluation preserves the existing compute price.
|
|
183
|
+
// Sampled burst cost is diagnostic until provider billing can be
|
|
184
|
+
// reconciled; a hard ceiling is never presented as actual usage.
|
|
185
|
+
costBasis: 'reserved_resources',
|
|
186
|
+
}),
|
|
151
187
|
startedAt: input.startedAt,
|
|
152
188
|
endedAt: input.endedAt,
|
|
153
189
|
},
|