deepline 0.3.89 → 0.3.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/contracts/install-commands.json +29 -0
- package/dist/bundling-sources/sdk/src/client.ts +120 -37
- package/dist/bundling-sources/sdk/src/index.ts +4 -0
- package/dist/bundling-sources/sdk/src/pagination.ts +25 -0
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
- package/dist/bundling-sources/sdk/src/release.ts +16 -9
- package/dist/bundling-sources/sdk/src/types.ts +23 -0
- package/dist/bundling-sources/shared_libs/monitors/analytics-public.ts +10 -0
- package/dist/bundling-sources/shared_libs/monitors/http-error-response.ts +3 -0
- package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +4 -4
- package/dist/bundling-sources/shared_libs/monitors/inventory-reconciliation.ts +59 -0
- package/dist/bundling-sources/shared_libs/monitors/lifecycle-notifications.ts +7 -9
- package/dist/bundling-sources/shared_libs/monitors/package.json +7 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +44 -2
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-consumer-health.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-identity-claims.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-public-aliases.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts +245 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-message.ts +38 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-ack-probe.ts +94 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-delivery.ts +60 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue.ts +60 -0
- package/dist/bundling-sources/shared_libs/monitors/snitcher-install-validation.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/snitcher-ip2company.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/snitcher-radar.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/tracking-provider-integrations.ts +1 -1
- package/dist/bundling-sources/shared_libs/observability/build-context.ts +13 -0
- package/dist/bundling-sources/shared_libs/observability/log-metadata.ts +160 -0
- package/dist/bundling-sources/shared_libs/observability/request-context.ts +68 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +654 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +35 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/neon-constants.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +272 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/async-play-invocation.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/bounceban-batching.ts +137 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +779 -140
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +46 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-payload-contract.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +725 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +442 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-protocol.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +66 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/ports.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/postgres-store.ts +170 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +1322 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/lane-retirement-gate.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +112 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/perf-trace.ts +127 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-artifact-runtime-contract.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +24 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/job.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/ports.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/contracts.ts +114 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/engine-suspension.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/index.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/job.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/ports.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/postgres-fence-adapter.ts +263 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/contracts.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/index.ts +15 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/job.ts +65 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/ports.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/index.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +142 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/ports.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/postgres-adapter.ts +102 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +60 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/index.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +5 -8
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +3 -5
- package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +168 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +72 -9
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +6 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +14 -44
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +57 -205
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts +201 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-env.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +67 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-release-environment.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-contract.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +14 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-run-task.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts +269 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-suffix.ts +73 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +6642 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +897 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-metrics.ts +120 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +76 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +476 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +147 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/index.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/lazy.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/play-run-recovery.ts +98 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts +260 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +1416 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +1255 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +942 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +14636 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +888 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +355 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +46 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runner-push.ts +60 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-api-origin.ts +33 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-gateway-health.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +835 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts +1218 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-operation.ts +207 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-scheduler-schema.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts +123 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-launch-read-observability.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +169 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/terminal-contract.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-queue-metrics.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +249 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +296 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +20 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +60 -15
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +14 -1
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +165 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +15 -23
- package/dist/bundling-sources/shared_libs/play-runtime/workflow-executor-auth.ts +169 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +122 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/deployment-status.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/docflow.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/play-grep-search.ts +39 -0
- package/dist/bundling-sources/shared_libs/plays/share/copy.ts +271 -0
- package/dist/bundling-sources/shared_libs/plays/share/metrics.ts +140 -0
- package/dist/bundling-sources/shared_libs/plays/share/public-read-model.ts +86 -0
- package/dist/bundling-sources/shared_libs/plays/share/seo-quality.ts +181 -0
- package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts +658 -0
- package/dist/bundling-sources/shared_libs/plays/share/types.ts +197 -0
- package/dist/bundling-sources/shared_libs/plays/slack-hitl-timeout-error.ts +37 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +6 -1
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/product-notifications/configuration.ts +7 -1
- package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +3 -2
- package/dist/bundling-sources/shared_libs/product-notifications/overflow.ts +18 -0
- package/dist/bundling-sources/shared_libs/product-notifications/rolling-limit.ts +24 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-delivery.ts +624 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-payload.ts +140 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-transport.ts +75 -0
- package/dist/bundling-sources/shared_libs/runtime-env.ts +6 -0
- package/dist/bundling-sources/shared_libs/security/email-blacklist.ts +42 -0
- package/dist/bundling-sources/shared_libs/security/header-safe-env.ts +10 -3
- package/dist/bundling-sources/shared_libs/security/slack-sanitize.ts +133 -0
- package/dist/cli/getting-started.d.mts +52 -0
- package/dist/cli/getting-started.d.ts +52 -0
- package/dist/cli/getting-started.js +65 -0
- package/dist/cli/getting-started.mjs +36 -0
- package/dist/cli/index.d.mts +9 -0
- package/dist/cli/index.d.ts +9 -0
- package/dist/cli/index.js +545 -264
- package/dist/cli/index.mjs +543 -271
- package/dist/{compiler-manifest-DsLbQaew.d.mts → compiler-manifest-BaJfE-5H.d.mts} +5 -3
- package/dist/{compiler-manifest-DsLbQaew.d.ts → compiler-manifest-BaJfE-5H.d.ts} +5 -3
- package/dist/index.d.mts +81 -148
- package/dist/index.d.ts +81 -148
- package/dist/index.js +83 -26
- package/dist/index.mjs +83 -26
- package/dist/install-integrity.json +130 -2
- package/dist/monitor-fleet-contract.d.mts +138 -0
- package/dist/monitor-fleet-contract.d.ts +138 -0
- package/dist/monitor-fleet-contract.js +404 -0
- package/dist/monitor-fleet-contract.mjs +367 -0
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/release.d.mts +255 -0
- package/dist/release.d.ts +255 -0
- package/dist/release.js +215 -0
- package/dist/release.mjs +188 -0
- package/package.json +21 -4
package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/engine-suspension.ts
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
PlayRunAttemptClaim,
|
|
3
|
+
PlayRunAttemptLaunch,
|
|
4
|
+
PlayRunAttemptLifecycle,
|
|
5
|
+
} from '../../play-runs/attempts/contracts';
|
|
6
|
+
import type { PlayRunAttemptStore } from '../../play-runs/attempts/ports';
|
|
7
|
+
import type { PlayExecutionSuspension } from '../../suspension';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The durable task engine is an Adapter, not Attempt policy. It maps a
|
|
11
|
+
* persisted Play suspension to a sleep/event/heartbeat primitive and reports
|
|
12
|
+
* whether the primitive parked this handler or resolved on this invocation.
|
|
13
|
+
*
|
|
14
|
+
* An Absurd Adapter retains the actual control signal until its ingress has
|
|
15
|
+
* handled the semantic `parked` outcome, then rethrows it. This separates the
|
|
16
|
+
* engine's transport exception from the Attempt Module's transition law.
|
|
17
|
+
*/
|
|
18
|
+
export type PlayRunAttemptEngineSuspensionPort = {
|
|
19
|
+
resolve(input: {
|
|
20
|
+
run: PlayRunAttemptLaunch;
|
|
21
|
+
attempt: PlayRunAttemptClaim;
|
|
22
|
+
checkpoint: unknown;
|
|
23
|
+
suspension: PlayExecutionSuspension;
|
|
24
|
+
}): Promise<{ kind: 'parked' } | { kind: 'resolved'; checkpoint: unknown }>;
|
|
25
|
+
/** Re-emit the engine-specific park signal captured by `resolve`. */
|
|
26
|
+
rethrowParked(): never;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export type PlayRunAttemptSuspensionOutcome =
|
|
30
|
+
| { kind: 'fence_lost'; stage: 'park' | 'resume' | 'handoff' }
|
|
31
|
+
| { kind: 'parked' }
|
|
32
|
+
| { kind: 'resumed'; checkpoint: unknown }
|
|
33
|
+
| { kind: 'continuation_handed_off'; checkpoint: unknown };
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Owns the durable transition law around an engine suspension:
|
|
37
|
+
*
|
|
38
|
+
* running -> waiting -> engine park|wake -> running|fresh continuation
|
|
39
|
+
*
|
|
40
|
+
* A detached runner which itself reaches a normal Play suspension cannot
|
|
41
|
+
* reuse its terminal slot; it is handed off to a fresh attempt. All other
|
|
42
|
+
* wakes retain their exact fence.
|
|
43
|
+
*/
|
|
44
|
+
export function createPlayRunAttemptSuspensionJob(ports: {
|
|
45
|
+
store: PlayRunAttemptStore;
|
|
46
|
+
engine: PlayRunAttemptEngineSuspensionPort;
|
|
47
|
+
lifecycle?: {
|
|
48
|
+
waiting(lifecycle: PlayRunAttemptLifecycle): Promise<void>;
|
|
49
|
+
resumed(lifecycle: PlayRunAttemptLifecycle): Promise<void>;
|
|
50
|
+
};
|
|
51
|
+
}) {
|
|
52
|
+
return {
|
|
53
|
+
async resolve(input: {
|
|
54
|
+
run: PlayRunAttemptLaunch;
|
|
55
|
+
attempt: PlayRunAttemptClaim;
|
|
56
|
+
checkpoint: unknown;
|
|
57
|
+
suspension: PlayExecutionSuspension;
|
|
58
|
+
/** The current runner result was read from a previously detached runner. */
|
|
59
|
+
fromDetachedRunner: boolean;
|
|
60
|
+
}): Promise<PlayRunAttemptSuspensionOutcome> {
|
|
61
|
+
const parked = await ports.store.park({
|
|
62
|
+
claim: input.attempt,
|
|
63
|
+
checkpoint: input.checkpoint,
|
|
64
|
+
suspension: input.suspension,
|
|
65
|
+
});
|
|
66
|
+
if (parked.kind === 'fence_lost') {
|
|
67
|
+
return { kind: 'fence_lost', stage: 'park' };
|
|
68
|
+
}
|
|
69
|
+
await ports.lifecycle?.waiting(parked.lifecycle);
|
|
70
|
+
|
|
71
|
+
const engineOutcome = await ports.engine.resolve({
|
|
72
|
+
run: input.run,
|
|
73
|
+
attempt: input.attempt,
|
|
74
|
+
checkpoint: input.checkpoint,
|
|
75
|
+
suspension: input.suspension,
|
|
76
|
+
});
|
|
77
|
+
if (engineOutcome.kind === 'parked') return engineOutcome;
|
|
78
|
+
|
|
79
|
+
if (
|
|
80
|
+
input.fromDetachedRunner &&
|
|
81
|
+
input.suspension.kind !== 'detached_runner'
|
|
82
|
+
) {
|
|
83
|
+
const handedOff = await ports.store.handoffDetachedContinuation({
|
|
84
|
+
claim: input.attempt,
|
|
85
|
+
checkpoint: engineOutcome.checkpoint,
|
|
86
|
+
});
|
|
87
|
+
if (handedOff.kind === 'fence_lost') {
|
|
88
|
+
return { kind: 'fence_lost', stage: 'handoff' };
|
|
89
|
+
}
|
|
90
|
+
await ports.lifecycle?.resumed(handedOff.lifecycle);
|
|
91
|
+
return {
|
|
92
|
+
kind: 'continuation_handed_off',
|
|
93
|
+
checkpoint: engineOutcome.checkpoint,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const resumed = await ports.store.resumeAfterPark({
|
|
98
|
+
claim: input.attempt,
|
|
99
|
+
suspension: input.suspension,
|
|
100
|
+
});
|
|
101
|
+
if (resumed.kind === 'fence_lost') {
|
|
102
|
+
return { kind: 'fence_lost', stage: 'resume' };
|
|
103
|
+
}
|
|
104
|
+
await ports.lifecycle?.resumed(resumed.lifecycle);
|
|
105
|
+
return { kind: 'resumed', checkpoint: engineOutcome.checkpoint };
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export { createPlayRunAttemptJob } from '../../play-runs/attempts/job';
|
|
2
|
+
export {
|
|
3
|
+
createPlayRunAttemptSuspensionJob,
|
|
4
|
+
type PlayRunAttemptEngineSuspensionPort,
|
|
5
|
+
type PlayRunAttemptSuspensionOutcome,
|
|
6
|
+
} from '../../play-runs/attempts/engine-suspension';
|
|
7
|
+
export {
|
|
8
|
+
createPlayRunAttemptWorkerTerminalProjection,
|
|
9
|
+
type PlayRunAttemptTerminalCommit,
|
|
10
|
+
type PlayRunAttemptWorkerTerminalProjectionPort,
|
|
11
|
+
type PlayRunAttemptWorkerTerminalProjectionPorts,
|
|
12
|
+
} from '../../play-runs/attempts/worker-terminal-projection';
|
|
13
|
+
export {
|
|
14
|
+
createPostgresPlayRunAttemptFence,
|
|
15
|
+
createPostgresPlayRunAttemptStore,
|
|
16
|
+
type PostgresPlayRunAttemptStore,
|
|
17
|
+
} from '../../play-runs/attempts/postgres-fence-adapter';
|
|
18
|
+
export type {
|
|
19
|
+
PlayRunAttemptClaim,
|
|
20
|
+
PlayRunAttemptInput,
|
|
21
|
+
PlayRunAttemptJob,
|
|
22
|
+
PlayRunAttemptLaunch,
|
|
23
|
+
PlayRunAttemptLifecycle,
|
|
24
|
+
PlayRunAttemptOutcome,
|
|
25
|
+
PlayRunAttemptPreCodeDeferral,
|
|
26
|
+
PlayRunAttemptSandboxOutcome,
|
|
27
|
+
PlayRunAttemptTerminal,
|
|
28
|
+
PlayRunAttemptTerminalEnvelope,
|
|
29
|
+
PlayRunAttemptTerminalHandoff,
|
|
30
|
+
} from '../../play-runs/attempts/contracts';
|
|
31
|
+
export type {
|
|
32
|
+
PlayRunAttemptPorts,
|
|
33
|
+
PlayRunAttemptStore,
|
|
34
|
+
PlayRunTerminalPort,
|
|
35
|
+
SandboxRunsPort,
|
|
36
|
+
} from '../../play-runs/attempts/ports';
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
PlayRunAttemptInput,
|
|
3
|
+
PlayRunAttemptJob,
|
|
4
|
+
PlayRunAttemptOutcome,
|
|
5
|
+
} from '../../play-runs/attempts/contracts';
|
|
6
|
+
import type { PlayRunAttemptPorts } from '../../play-runs/attempts/ports';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Builds the Play Run Attempt job.
|
|
10
|
+
*
|
|
11
|
+
* The ordering is the contract:
|
|
12
|
+
*
|
|
13
|
+
* terminal read -> fenced claim -> Sandbox Runs -> Terminal | park | defer
|
|
14
|
+
*
|
|
15
|
+
* A post-code-unknown outcome is intentionally returned to the caller rather
|
|
16
|
+
* than converted to a retry. A later recovery leg must resolve durable
|
|
17
|
+
* runner/resource facts; replaying here risks duplicate provider work.
|
|
18
|
+
*/
|
|
19
|
+
export function createPlayRunAttemptJob(
|
|
20
|
+
ports: PlayRunAttemptPorts,
|
|
21
|
+
): PlayRunAttemptJob {
|
|
22
|
+
return {
|
|
23
|
+
async execute(input: PlayRunAttemptInput): Promise<PlayRunAttemptOutcome> {
|
|
24
|
+
validate(input);
|
|
25
|
+
const terminal = await ports.store.readTerminal({
|
|
26
|
+
runId: input.run.runId,
|
|
27
|
+
});
|
|
28
|
+
if (terminal) return { kind: 'already_terminal', terminal };
|
|
29
|
+
|
|
30
|
+
// A detached runner owns an attempt-scoped executor token. Its heartbeat
|
|
31
|
+
// wake must see that exact claim before ordinary allocation is allowed.
|
|
32
|
+
const claim =
|
|
33
|
+
(await ports.store.resumeDetached(input.run)) ??
|
|
34
|
+
(await ports.store.claim(input.run));
|
|
35
|
+
if (claim === null) return { kind: 'claim_rejected' };
|
|
36
|
+
|
|
37
|
+
const sandbox = await ports.sandboxRuns.execute({
|
|
38
|
+
run: input.run,
|
|
39
|
+
attempt: claim,
|
|
40
|
+
});
|
|
41
|
+
switch (sandbox.kind) {
|
|
42
|
+
case 'pre_code_deferred':
|
|
43
|
+
const deferred = await ports.store.deferBeforeCode({
|
|
44
|
+
claim,
|
|
45
|
+
deferral: sandbox.deferral,
|
|
46
|
+
});
|
|
47
|
+
if (deferred.kind === 'deferred') return sandbox;
|
|
48
|
+
if (deferred.kind === 'fence_lost') return { kind: 'claim_rejected' };
|
|
49
|
+
const terminalResult = await ports.terminal.accept({
|
|
50
|
+
attempt: claim,
|
|
51
|
+
result: deferred.terminal,
|
|
52
|
+
});
|
|
53
|
+
return terminalResult.kind === 'committed'
|
|
54
|
+
? { kind: 'terminal_committed' }
|
|
55
|
+
: terminalResult.kind === 'already_terminal'
|
|
56
|
+
? { kind: 'terminal_already_committed' }
|
|
57
|
+
: { kind: 'terminal_fence_lost' };
|
|
58
|
+
case 'suspended': {
|
|
59
|
+
const parked = await ports.store.park({
|
|
60
|
+
claim,
|
|
61
|
+
checkpoint: sandbox.checkpoint,
|
|
62
|
+
suspension: sandbox.suspension,
|
|
63
|
+
});
|
|
64
|
+
return parked.kind === 'parked'
|
|
65
|
+
? {
|
|
66
|
+
kind: 'parked',
|
|
67
|
+
checkpoint: sandbox.checkpoint,
|
|
68
|
+
suspension: sandbox.suspension,
|
|
69
|
+
}
|
|
70
|
+
: { kind: 'claim_rejected' };
|
|
71
|
+
}
|
|
72
|
+
case 'post_code_unknown':
|
|
73
|
+
return sandbox;
|
|
74
|
+
case 'terminal': {
|
|
75
|
+
const terminalResult = await ports.terminal.accept({
|
|
76
|
+
attempt: claim,
|
|
77
|
+
result: sandbox.result,
|
|
78
|
+
});
|
|
79
|
+
switch (terminalResult.kind) {
|
|
80
|
+
case 'committed':
|
|
81
|
+
return { kind: 'terminal_committed' };
|
|
82
|
+
case 'already_terminal':
|
|
83
|
+
return { kind: 'terminal_already_committed' };
|
|
84
|
+
case 'fence_lost':
|
|
85
|
+
return { kind: 'terminal_fence_lost' };
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function validate(input: PlayRunAttemptInput): void {
|
|
94
|
+
if (!input.run.runId.trim()) {
|
|
95
|
+
throw new Error('Play Run Attempt requires a non-empty run id.');
|
|
96
|
+
}
|
|
97
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
PlayRunAttemptClaim,
|
|
3
|
+
PlayRunAttemptLaunch,
|
|
4
|
+
PlayRunAttemptLifecycle,
|
|
5
|
+
PlayRunAttemptPreCodeDeferral,
|
|
6
|
+
PlayRunAttemptSandboxOutcome,
|
|
7
|
+
PlayRunAttemptTerminalEnvelope,
|
|
8
|
+
PlayRunAttemptTerminal,
|
|
9
|
+
PlayRunAttemptTerminalHandoff,
|
|
10
|
+
} from '../../play-runs/attempts/contracts';
|
|
11
|
+
import type { PlayExecutionSuspension } from '../../suspension';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Scheduler state belongs on this seam. It intentionally combines allocate
|
|
15
|
+
* and assign: an attempt number without its write fence is not usable work.
|
|
16
|
+
*/
|
|
17
|
+
export type PlayRunAttemptStore = {
|
|
18
|
+
readTerminal(input: {
|
|
19
|
+
runId: string;
|
|
20
|
+
}): Promise<PlayRunAttemptTerminal | null>;
|
|
21
|
+
claim(input: PlayRunAttemptLaunch): Promise<PlayRunAttemptClaim | null>;
|
|
22
|
+
/**
|
|
23
|
+
* Reuses the fence held by a parked detached runner. This read is separate
|
|
24
|
+
* from `claim` because every other parked suspension must receive a fresh
|
|
25
|
+
* generation after its engine wake.
|
|
26
|
+
*/
|
|
27
|
+
resumeDetached(
|
|
28
|
+
input: PlayRunAttemptLaunch,
|
|
29
|
+
): Promise<PlayRunAttemptClaim | null>;
|
|
30
|
+
/** Only proven pre-code work may be returned to scheduler admission. */
|
|
31
|
+
deferBeforeCode(input: {
|
|
32
|
+
claim: PlayRunAttemptClaim;
|
|
33
|
+
deferral: PlayRunAttemptPreCodeDeferral;
|
|
34
|
+
}): Promise<
|
|
35
|
+
| { kind: 'deferred' }
|
|
36
|
+
| { kind: 'exhausted'; terminal: PlayRunAttemptTerminalEnvelope }
|
|
37
|
+
| { kind: 'fence_lost' }
|
|
38
|
+
>;
|
|
39
|
+
/**
|
|
40
|
+
* Stores a safe continuation without inventing a new attempt generation.
|
|
41
|
+
* The store, rather than callers, derives detached-runner heartbeat rules
|
|
42
|
+
* from the typed suspension.
|
|
43
|
+
*/
|
|
44
|
+
park(input: {
|
|
45
|
+
claim: PlayRunAttemptClaim;
|
|
46
|
+
checkpoint: unknown;
|
|
47
|
+
suspension: PlayExecutionSuspension;
|
|
48
|
+
}): Promise<
|
|
49
|
+
| { kind: 'parked'; lifecycle: PlayRunAttemptLifecycle }
|
|
50
|
+
| { kind: 'fence_lost' }
|
|
51
|
+
>;
|
|
52
|
+
/** Transition a non-detached parked attempt back to running. */
|
|
53
|
+
resumeAfterPark(input: {
|
|
54
|
+
claim: PlayRunAttemptClaim;
|
|
55
|
+
suspension: PlayExecutionSuspension;
|
|
56
|
+
}): Promise<
|
|
57
|
+
| { kind: 'resumed'; lifecycle: PlayRunAttemptLifecycle }
|
|
58
|
+
| { kind: 'fence_lost' }
|
|
59
|
+
>;
|
|
60
|
+
/**
|
|
61
|
+
* A detached runner that produces another suspension cannot reuse its
|
|
62
|
+
* terminal slot. Retire it and return its checkpoint to durable admission.
|
|
63
|
+
*/
|
|
64
|
+
handoffDetachedContinuation(input: {
|
|
65
|
+
claim: PlayRunAttemptClaim;
|
|
66
|
+
checkpoint: unknown;
|
|
67
|
+
}): Promise<
|
|
68
|
+
| { kind: 'handed_off'; lifecycle: PlayRunAttemptLifecycle }
|
|
69
|
+
| { kind: 'fence_lost' }
|
|
70
|
+
>;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The physical execution-resource Module; it never settles the Play Run.
|
|
75
|
+
*
|
|
76
|
+
* This is the proposed live-worker routing seam. An Absurd Adapter will own
|
|
77
|
+
* one complete runner leg: config/materialization, resource facts, runner
|
|
78
|
+
* start or detached-terminal observation, and engine-context suspension
|
|
79
|
+
* resolution. It reports only these semantic outcomes here. In particular it
|
|
80
|
+
* must report `pre_code_deferred` only with proof that no runner/customer code
|
|
81
|
+
* could have started, and must report `post_code_unknown` rather than replay
|
|
82
|
+
* an ambiguous runner command. The Attempt Module then owns the only durable
|
|
83
|
+
* transition allowed for each outcome.
|
|
84
|
+
*/
|
|
85
|
+
export type SandboxRunsPort = {
|
|
86
|
+
execute(input: {
|
|
87
|
+
run: PlayRunAttemptLaunch;
|
|
88
|
+
attempt: PlayRunAttemptClaim;
|
|
89
|
+
}): Promise<PlayRunAttemptSandboxOutcome>;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
/** The durable terminal Module; it owns first-write-wins terminal truth. */
|
|
93
|
+
export type PlayRunTerminalPort = {
|
|
94
|
+
accept(input: {
|
|
95
|
+
attempt: PlayRunAttemptClaim;
|
|
96
|
+
result: PlayRunAttemptTerminalEnvelope;
|
|
97
|
+
}): Promise<PlayRunAttemptTerminalHandoff>;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
export type PlayRunAttemptPorts = {
|
|
101
|
+
store: PlayRunAttemptStore;
|
|
102
|
+
sandboxRuns: SandboxRunsPort;
|
|
103
|
+
terminal: PlayRunTerminalPort;
|
|
104
|
+
};
|
package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/postgres-fence-adapter.ts
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
allocatePostgresSchedulerNextAttempt,
|
|
5
|
+
assignPostgresSchedulerRunAttempt,
|
|
6
|
+
deferPostgresSchedulerAttemptForCapacity,
|
|
7
|
+
deferPostgresSchedulerAttemptForRecovery,
|
|
8
|
+
handoffPostgresSchedulerAttemptContinuation,
|
|
9
|
+
markPostgresSchedulerAttemptRunning,
|
|
10
|
+
markPostgresSchedulerAttemptWaiting,
|
|
11
|
+
readPostgresSchedulerResult,
|
|
12
|
+
resumePostgresSchedulerDetachedRunnerAttempt,
|
|
13
|
+
type PostgresSchedulerOptions,
|
|
14
|
+
type PostgresSchedulerQueryClient,
|
|
15
|
+
} from '../../scheduler-backends/postgres';
|
|
16
|
+
import { PLAY_RUN_RECOVERY_DEFER_POLICY } from '../../play-run-recovery-policy';
|
|
17
|
+
import type {
|
|
18
|
+
PlayRunAttemptClaim,
|
|
19
|
+
PlayRunAttemptTerminalEnvelope,
|
|
20
|
+
} from '../../play-runs/attempts/contracts';
|
|
21
|
+
import type { PlayRunAttemptStore } from '../../play-runs/attempts/ports';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Runtime-Postgres Adapter for Play Run Attempt.
|
|
25
|
+
*
|
|
26
|
+
* The Adapter intentionally owns the scheduler's real transition vocabulary:
|
|
27
|
+
* fresh versus detached-fence claim, finite pre-code recovery, typed parking,
|
|
28
|
+
* ordinary wake, and detached continuation handoff. It does not execute
|
|
29
|
+
* sandboxes or shape runner-returned terminal output; those remain the
|
|
30
|
+
* Sandbox Runs and Terminal Modules behind their own seams. A bounded
|
|
31
|
+
* pre-code recovery exhaustion is the one scheduler-owned terminal envelope
|
|
32
|
+
* this Adapter constructs.
|
|
33
|
+
*/
|
|
34
|
+
export type PostgresPlayRunAttemptStore = PlayRunAttemptStore;
|
|
35
|
+
|
|
36
|
+
export function createPostgresPlayRunAttemptStore(input: {
|
|
37
|
+
client: PostgresSchedulerQueryClient;
|
|
38
|
+
schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
|
|
39
|
+
/** Injectable only to make Adapter conformance vectors deterministic. */
|
|
40
|
+
createFenceId?: () => string;
|
|
41
|
+
nowMs?: () => number;
|
|
42
|
+
}): PostgresPlayRunAttemptStore {
|
|
43
|
+
const options = input.schedulerOptions;
|
|
44
|
+
const createFenceId = input.createFenceId ?? randomUUID;
|
|
45
|
+
const nowMs = input.nowMs ?? Date.now;
|
|
46
|
+
const freshClaim = async (run: { runId: string }) => {
|
|
47
|
+
// This intentionally retains the worker's allocate-then-assign law.
|
|
48
|
+
// Races may choose the same generation, but the assignment SQL admits
|
|
49
|
+
// only one of them; a loser observes null and must retire without work.
|
|
50
|
+
const number = await allocatePostgresSchedulerNextAttempt(
|
|
51
|
+
input.client,
|
|
52
|
+
run.runId,
|
|
53
|
+
options,
|
|
54
|
+
);
|
|
55
|
+
const fenceId = createFenceId();
|
|
56
|
+
const assigned = await assignPostgresSchedulerRunAttempt(
|
|
57
|
+
input.client,
|
|
58
|
+
{ runId: run.runId, attempt: number, taskToken: fenceId },
|
|
59
|
+
options,
|
|
60
|
+
);
|
|
61
|
+
if (assigned === null) return null;
|
|
62
|
+
return {
|
|
63
|
+
runId: run.runId,
|
|
64
|
+
number,
|
|
65
|
+
fenceId,
|
|
66
|
+
checkpoint: assigned.checkpoint ?? null,
|
|
67
|
+
origin: 'fresh' as const,
|
|
68
|
+
};
|
|
69
|
+
};
|
|
70
|
+
return {
|
|
71
|
+
async readTerminal({ runId }) {
|
|
72
|
+
const terminal = await readPostgresSchedulerResult(
|
|
73
|
+
input.client,
|
|
74
|
+
runId,
|
|
75
|
+
options,
|
|
76
|
+
);
|
|
77
|
+
if (terminal === null) return null;
|
|
78
|
+
return {
|
|
79
|
+
status: terminal.status,
|
|
80
|
+
...(Object.hasOwn(terminal, 'output')
|
|
81
|
+
? { output: terminal.output }
|
|
82
|
+
: {}),
|
|
83
|
+
...(Object.hasOwn(terminal, 'error') ? { error: terminal.error } : {}),
|
|
84
|
+
};
|
|
85
|
+
},
|
|
86
|
+
async resumeDetached(run) {
|
|
87
|
+
const resumed = await resumePostgresSchedulerDetachedRunnerAttempt(
|
|
88
|
+
input.client,
|
|
89
|
+
{ runId: run.runId },
|
|
90
|
+
options,
|
|
91
|
+
);
|
|
92
|
+
if (!resumed) return null;
|
|
93
|
+
const suspension = detachedRunnerSuspension(resumed.suspension);
|
|
94
|
+
return {
|
|
95
|
+
runId: run.runId,
|
|
96
|
+
number: resumed.attempt,
|
|
97
|
+
fenceId: resumed.leaseToken,
|
|
98
|
+
checkpoint: resumed.checkpoint ?? null,
|
|
99
|
+
origin: 'detached_runner_resume' as const,
|
|
100
|
+
suspension,
|
|
101
|
+
};
|
|
102
|
+
},
|
|
103
|
+
claim: freshClaim,
|
|
104
|
+
async deferBeforeCode({ claim, deferral }) {
|
|
105
|
+
if (deferral.kind === 'capacity') {
|
|
106
|
+
const deferred = await deferPostgresSchedulerAttemptForCapacity(
|
|
107
|
+
input.client,
|
|
108
|
+
{
|
|
109
|
+
runId: claim.runId,
|
|
110
|
+
attempt: claim.number,
|
|
111
|
+
leaseToken: claim.fenceId,
|
|
112
|
+
reason: deferral.reason,
|
|
113
|
+
},
|
|
114
|
+
options,
|
|
115
|
+
);
|
|
116
|
+
return deferred
|
|
117
|
+
? { kind: 'deferred' as const }
|
|
118
|
+
: { kind: 'fence_lost' as const };
|
|
119
|
+
}
|
|
120
|
+
const decision = deferral.decision;
|
|
121
|
+
if (decision.stage !== 'pre_code' || decision.action !== 'defer') {
|
|
122
|
+
throw new Error(
|
|
123
|
+
'Play Run Attempt recovery deferral requires a pre-code defer decision.',
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
const recovery = await deferPostgresSchedulerAttemptForRecovery(
|
|
127
|
+
input.client,
|
|
128
|
+
{
|
|
129
|
+
runId: claim.runId,
|
|
130
|
+
attempt: claim.number,
|
|
131
|
+
leaseToken: claim.fenceId,
|
|
132
|
+
reason: deferral.reason,
|
|
133
|
+
dependency: decision.dependency,
|
|
134
|
+
openCircuit:
|
|
135
|
+
decision.dependency === 'modal' &&
|
|
136
|
+
decision.reason !== 'modal_circuit_open_before_execution',
|
|
137
|
+
nowMs: nowMs(),
|
|
138
|
+
maxDefers: PLAY_RUN_RECOVERY_DEFER_POLICY.maxDefers,
|
|
139
|
+
horizonMs: PLAY_RUN_RECOVERY_DEFER_POLICY.horizonMs,
|
|
140
|
+
deferBaseMs: PLAY_RUN_RECOVERY_DEFER_POLICY.deferBaseSeconds * 1_000,
|
|
141
|
+
deferMaxMs: PLAY_RUN_RECOVERY_DEFER_POLICY.deferMaxSeconds * 1_000,
|
|
142
|
+
},
|
|
143
|
+
options,
|
|
144
|
+
);
|
|
145
|
+
if (recovery.deferred) return { kind: 'deferred' as const };
|
|
146
|
+
if (!recovery.exhausted) return { kind: 'fence_lost' as const };
|
|
147
|
+
return {
|
|
148
|
+
kind: 'exhausted' as const,
|
|
149
|
+
terminal: exhaustedRecoveryTerminal({
|
|
150
|
+
dependency: decision.dependency,
|
|
151
|
+
checkpoint: claim.checkpoint,
|
|
152
|
+
}),
|
|
153
|
+
};
|
|
154
|
+
},
|
|
155
|
+
async park({ claim, checkpoint, suspension }) {
|
|
156
|
+
const parked = await markPostgresSchedulerAttemptWaiting(
|
|
157
|
+
input.client,
|
|
158
|
+
{
|
|
159
|
+
runId: claim.runId,
|
|
160
|
+
attempt: claim.number,
|
|
161
|
+
leaseToken: claim.fenceId,
|
|
162
|
+
checkpoint,
|
|
163
|
+
suspension,
|
|
164
|
+
preserveHeartbeatAt: suspension.kind === 'detached_runner',
|
|
165
|
+
},
|
|
166
|
+
options,
|
|
167
|
+
);
|
|
168
|
+
return parked
|
|
169
|
+
? { kind: 'parked', lifecycle: parked }
|
|
170
|
+
: { kind: 'fence_lost' };
|
|
171
|
+
},
|
|
172
|
+
async resumeAfterPark({ claim, suspension }) {
|
|
173
|
+
const resumed = await markPostgresSchedulerAttemptRunning(
|
|
174
|
+
input.client,
|
|
175
|
+
{
|
|
176
|
+
runId: claim.runId,
|
|
177
|
+
attempt: claim.number,
|
|
178
|
+
leaseToken: claim.fenceId,
|
|
179
|
+
preserveHeartbeatAt: suspension.kind === 'detached_runner',
|
|
180
|
+
},
|
|
181
|
+
options,
|
|
182
|
+
);
|
|
183
|
+
return resumed
|
|
184
|
+
? { kind: 'resumed', lifecycle: resumed }
|
|
185
|
+
: { kind: 'fence_lost' };
|
|
186
|
+
},
|
|
187
|
+
async handoffDetachedContinuation({ claim, checkpoint }) {
|
|
188
|
+
if (claim.origin !== 'detached_runner_resume') {
|
|
189
|
+
throw new Error(
|
|
190
|
+
'Only a resumed detached-runner attempt may hand off a fresh continuation.',
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
const handedOff = await handoffPostgresSchedulerAttemptContinuation(
|
|
194
|
+
input.client,
|
|
195
|
+
{
|
|
196
|
+
runId: claim.runId,
|
|
197
|
+
attempt: claim.number,
|
|
198
|
+
leaseToken: claim.fenceId,
|
|
199
|
+
checkpoint,
|
|
200
|
+
},
|
|
201
|
+
options,
|
|
202
|
+
);
|
|
203
|
+
return handedOff
|
|
204
|
+
? { kind: 'handed_off', lifecycle: handedOff }
|
|
205
|
+
: { kind: 'fence_lost' };
|
|
206
|
+
},
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function detachedRunnerSuspension(
|
|
211
|
+
value: unknown,
|
|
212
|
+
): NonNullable<PlayRunAttemptClaim['suspension']> {
|
|
213
|
+
if (
|
|
214
|
+
!value ||
|
|
215
|
+
typeof value !== 'object' ||
|
|
216
|
+
Array.isArray(value) ||
|
|
217
|
+
(value as { kind?: unknown }).kind !== 'detached_runner'
|
|
218
|
+
) {
|
|
219
|
+
throw new Error(
|
|
220
|
+
'Detached Attempt resume was selected without a detached_runner suspension.',
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
const candidate = value as Record<string, unknown>;
|
|
224
|
+
if (
|
|
225
|
+
!Number.isSafeInteger(candidate.runnerAttempt) ||
|
|
226
|
+
typeof candidate.boundaryId !== 'string' ||
|
|
227
|
+
typeof candidate.sandboxId !== 'string' ||
|
|
228
|
+
typeof candidate.sessionId !== 'string' ||
|
|
229
|
+
typeof candidate.cmdId !== 'string' ||
|
|
230
|
+
typeof candidate.outputPath !== 'string' ||
|
|
231
|
+
typeof candidate.exitCodePath !== 'string' ||
|
|
232
|
+
!Number.isFinite(candidate.startedAtMs) ||
|
|
233
|
+
!Number.isFinite(candidate.heartbeatTimeoutMs) ||
|
|
234
|
+
!Number.isFinite(candidate.ceilingMs)
|
|
235
|
+
) {
|
|
236
|
+
throw new Error(
|
|
237
|
+
'Detached Attempt resume found an invalid persisted detached_runner suspension.',
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
return value as NonNullable<PlayRunAttemptClaim['suspension']>;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** @deprecated Use `createPostgresPlayRunAttemptStore`; the old name retained
|
|
244
|
+
* for the migration while callers move from the claim-only Interface. */
|
|
245
|
+
export const createPostgresPlayRunAttemptFence =
|
|
246
|
+
createPostgresPlayRunAttemptStore;
|
|
247
|
+
|
|
248
|
+
function exhaustedRecoveryTerminal(input: {
|
|
249
|
+
dependency: 'modal' | 'runtime';
|
|
250
|
+
checkpoint: unknown;
|
|
251
|
+
}): PlayRunAttemptTerminalEnvelope {
|
|
252
|
+
// The checkpoint is intentionally not returned to customer output. It is
|
|
253
|
+
// already durable on the run row; the Terminal Module only needs a typed
|
|
254
|
+
// failure envelope to settle first-write-wins truth.
|
|
255
|
+
void input.checkpoint;
|
|
256
|
+
return {
|
|
257
|
+
status: 'failed',
|
|
258
|
+
error:
|
|
259
|
+
`PLATFORM_RECOVERY_EXHAUSTED: ${input.dependency} remained unavailable ` +
|
|
260
|
+
'before customer execution began through the bounded recovery horizon. ' +
|
|
261
|
+
'No customer code was started; inspect and redrive this run after the dependency is healthy.',
|
|
262
|
+
};
|
|
263
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { PlayRunAttemptTerminalEnvelope } from '../../play-runs/attempts/contracts';
|
|
2
|
+
|
|
3
|
+
/** The committed terminal fact used by worker-side read-model projection. */
|
|
4
|
+
export type PlayRunAttemptTerminalCommit =
|
|
5
|
+
| { status: 'completed'; output: unknown }
|
|
6
|
+
| { status: 'failed'; error: string };
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Post-commit effects are deliberately separate from terminal truth. Terminal
|
|
10
|
+
* has already won first-write-wins before this Interface is invoked; these
|
|
11
|
+
* effects improve observability and release ephemeral receipt leases, but
|
|
12
|
+
* cannot alter customer-visible terminal state.
|
|
13
|
+
*/
|
|
14
|
+
export type PlayRunAttemptWorkerTerminalProjectionPort = {
|
|
15
|
+
observeFenceLost(): Promise<void>;
|
|
16
|
+
projectCommitted(input: {
|
|
17
|
+
runnerResult: PlayRunAttemptTerminalEnvelope;
|
|
18
|
+
terminal: PlayRunAttemptTerminalCommit;
|
|
19
|
+
}): Promise<void>;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export type PlayRunAttemptWorkerTerminalProjectionPorts = {
|
|
23
|
+
reconcileTerminalTruth(input: {
|
|
24
|
+
runnerResult: PlayRunAttemptTerminalEnvelope;
|
|
25
|
+
terminal: PlayRunAttemptTerminalCommit;
|
|
26
|
+
}): Promise<void>;
|
|
27
|
+
releaseFailedAttempt(input: {
|
|
28
|
+
runnerResult: PlayRunAttemptTerminalEnvelope;
|
|
29
|
+
terminal: Extract<PlayRunAttemptTerminalCommit, { status: 'failed' }>;
|
|
30
|
+
}): Promise<void>;
|
|
31
|
+
queueProgressFinalization(input: {
|
|
32
|
+
runnerResult: PlayRunAttemptTerminalEnvelope;
|
|
33
|
+
terminal: PlayRunAttemptTerminalCommit;
|
|
34
|
+
}): void;
|
|
35
|
+
requestTerminalProjection(input: {
|
|
36
|
+
terminal: PlayRunAttemptTerminalCommit;
|
|
37
|
+
}): void;
|
|
38
|
+
observeFenceLost(): Promise<void>;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Builds the worker projection Module. The ordering is its Interface:
|
|
43
|
+
* reconcile durable truth, release failed-attempt leases, then enqueue
|
|
44
|
+
* best-effort progress/outbox projection. A stale terminal fence observes the
|
|
45
|
+
* winner only; it never emits a second projection sequence.
|
|
46
|
+
*/
|
|
47
|
+
export function createPlayRunAttemptWorkerTerminalProjection(
|
|
48
|
+
ports: PlayRunAttemptWorkerTerminalProjectionPorts,
|
|
49
|
+
): PlayRunAttemptWorkerTerminalProjectionPort {
|
|
50
|
+
return {
|
|
51
|
+
observeFenceLost: ports.observeFenceLost,
|
|
52
|
+
async projectCommitted(input) {
|
|
53
|
+
await ports.reconcileTerminalTruth(input);
|
|
54
|
+
if (input.terminal.status === 'failed') {
|
|
55
|
+
await ports.releaseFailedAttempt({
|
|
56
|
+
runnerResult: input.runnerResult,
|
|
57
|
+
terminal: input.terminal,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
ports.queueProgressFinalization(input);
|
|
61
|
+
ports.requestTerminalProjection({ terminal: input.terminal });
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|