deepline 0.3.89 → 0.3.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/contracts/install-commands.json +29 -0
- package/dist/bundling-sources/sdk/src/client.ts +221 -43
- package/dist/bundling-sources/sdk/src/index.ts +4 -0
- package/dist/bundling-sources/sdk/src/monitor-fleet-contract.ts +2 -556
- package/dist/bundling-sources/sdk/src/monitor-fleets.ts +2 -2
- package/dist/bundling-sources/sdk/src/pagination.ts +25 -0
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
- package/dist/bundling-sources/sdk/src/release.ts +11 -9
- package/dist/bundling-sources/sdk/src/types.ts +29 -1
- package/dist/bundling-sources/shared_libs/integration-contracts/monitor-fleet-contract.ts +549 -0
- package/dist/bundling-sources/shared_libs/monitors/analytics-public.ts +10 -0
- package/dist/bundling-sources/shared_libs/monitors/fleets/errors.ts +96 -0
- package/dist/bundling-sources/shared_libs/monitors/fleets/limits.ts +8 -0
- package/dist/bundling-sources/shared_libs/monitors/fleets/plan.ts +320 -0
- package/dist/bundling-sources/shared_libs/monitors/fleets/verdict.ts +106 -0
- package/dist/bundling-sources/shared_libs/monitors/http-error-response.ts +8 -1
- package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +4 -4
- package/dist/bundling-sources/shared_libs/monitors/inventory-reconciliation.ts +59 -0
- package/dist/bundling-sources/shared_libs/monitors/lifecycle-notifications.ts +7 -9
- package/dist/bundling-sources/shared_libs/monitors/monitor-fleet-limits.ts +2 -12
- package/dist/bundling-sources/shared_libs/monitors/package.json +113 -4
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +56 -9
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-consumer-health.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-identity-claims.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-public-aliases.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts +245 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-message.ts +38 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-ack-probe.ts +94 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-delivery.ts +60 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue.ts +60 -0
- package/dist/bundling-sources/shared_libs/monitors/snitcher-install-validation.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/snitcher-ip2company.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/snitcher-radar.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/tracking-provider-integrations.ts +1 -1
- package/dist/bundling-sources/shared_libs/observability/build-context.ts +13 -0
- package/dist/bundling-sources/shared_libs/observability/log-metadata.ts +160 -0
- package/dist/bundling-sources/shared_libs/observability/request-context.ts +68 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +690 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +35 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/neon-constants.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +272 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +46 -1
- package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/async-play-invocation.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/bounceban-batching.ts +137 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +779 -140
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +46 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-payload-contract.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +725 -0
- package/dist/bundling-sources/shared_libs/play-runtime/db-session-lease-store.ts +578 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +521 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-receipt-gateway.ts +1378 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/runtime-topology.ts +1024 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-gateway-route-probe.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machine-lifecycle.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +1844 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/process-tree.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +262 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-surface.ts +98 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +303 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-tuple-publication-guardian.ts +70 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-tuple-recovery.ts +519 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/vercel-fly-rollback-alignment.ts +280 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/vercel-production-gate.ts +159 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +442 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-protocol.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +66 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/memory-store.ts +114 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/ports.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/postgres-store.ts +170 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +1322 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/lane-retirement-gate.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +112 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/perf-trace.ts +127 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-artifact-runtime-contract.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +24 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/job.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/ports.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/contracts.ts +114 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/engine-suspension.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/index.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/job.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/ports.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/postgres-fence-adapter.ts +263 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/contracts.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/index.ts +15 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/job.ts +65 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/ports.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/index.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +142 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/ports.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/postgres-adapter.ts +102 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +60 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/index.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +5 -8
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +3 -5
- package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +168 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +44 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/child-play-resolver.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +3662 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/progress-sidecar.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +84 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-gateway-transport.ts +91 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-heartbeat.ts +424 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +321 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-sheet-persistence-retry.ts +110 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +625 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +72 -9
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +6 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +14 -44
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +103 -208
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts +201 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-env.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +67 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-release-environment.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-contract.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +14 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-run-task.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts +269 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-suffix.ts +73 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +6642 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +897 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-metrics.ts +120 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +76 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +476 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +147 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/index.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/lazy.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/play-run-recovery.ts +98 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts +260 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +1416 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +1255 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +942 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +14636 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +888 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +355 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +46 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runner-push.ts +60 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-api-origin.ts +33 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-gateway-health.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +835 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts +1218 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-operation.ts +207 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-scheduler-schema.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts +123 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-launch-read-observability.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +169 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/terminal-contract.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-queue-metrics.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +249 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +296 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +20 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +60 -15
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +14 -1
- package/dist/bundling-sources/shared_libs/play-runtime/webhook-receipt-retry.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +165 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +15 -23
- package/dist/bundling-sources/shared_libs/play-runtime/workflow-executor-auth.ts +169 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +122 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/bundling/repository-adapter.ts +24 -0
- package/dist/bundling-sources/shared_libs/plays/child-run-projection.ts +174 -0
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/deploy/api-client.ts +267 -0
- package/dist/bundling-sources/shared_libs/plays/deploy/control-plane-types.ts +73 -0
- package/dist/bundling-sources/shared_libs/plays/deploy/manifest.ts +162 -0
- package/dist/bundling-sources/shared_libs/plays/deploy/project-serializer.ts +87 -0
- package/dist/bundling-sources/shared_libs/plays/deploy/source-file-path.ts +26 -0
- package/dist/bundling-sources/shared_libs/plays/deploy/target-paths.ts +36 -0
- package/dist/bundling-sources/shared_libs/plays/deployment-status.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/docflow.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/launch/input-contract.ts +400 -0
- package/dist/bundling-sources/shared_libs/plays/launch/prepare.ts +96 -0
- package/dist/bundling-sources/shared_libs/plays/local-file-discovery.ts +385 -0
- package/dist/bundling-sources/shared_libs/plays/play-grep-search.ts +112 -0
- package/dist/bundling-sources/shared_libs/plays/play-reference.ts +117 -0
- package/dist/bundling-sources/shared_libs/plays/run-input-summary.ts +260 -0
- package/dist/bundling-sources/shared_libs/plays/run-summary-types.ts +49 -0
- package/dist/bundling-sources/shared_libs/plays/secrets/errors.ts +19 -0
- package/dist/bundling-sources/shared_libs/plays/share/copy.ts +271 -0
- package/dist/bundling-sources/shared_libs/plays/share/metrics.ts +140 -0
- package/dist/bundling-sources/shared_libs/plays/share/public-read-model.ts +86 -0
- package/dist/bundling-sources/shared_libs/plays/share/seo-quality.ts +181 -0
- package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts +658 -0
- package/dist/bundling-sources/shared_libs/plays/share/types.ts +197 -0
- package/dist/bundling-sources/shared_libs/plays/sheet-filter.ts +347 -0
- package/dist/bundling-sources/shared_libs/plays/slack-hitl-timeout-error.ts +37 -0
- package/dist/bundling-sources/shared_libs/plays/standard-webhooks.ts +117 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +6 -1
- package/dist/bundling-sources/shared_libs/plays/status-result.ts +388 -0
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/transform-factor-structure.ts +377 -0
- package/dist/bundling-sources/shared_libs/plays/webhook-payload.ts +113 -0
- package/dist/bundling-sources/shared_libs/product-notifications/configuration.ts +7 -1
- package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +3 -2
- package/dist/bundling-sources/shared_libs/product-notifications/overflow.ts +18 -0
- package/dist/bundling-sources/shared_libs/product-notifications/rolling-limit.ts +24 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-delivery.ts +624 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-payload.ts +140 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-transport.ts +75 -0
- package/dist/bundling-sources/shared_libs/runtime-env.ts +6 -0
- package/dist/bundling-sources/shared_libs/security/email-blacklist.ts +42 -0
- package/dist/bundling-sources/shared_libs/security/header-safe-env.ts +10 -3
- package/dist/bundling-sources/shared_libs/security/slack-sanitize.ts +133 -0
- package/dist/cli/getting-started.d.mts +52 -0
- package/dist/cli/getting-started.d.ts +52 -0
- package/dist/cli/getting-started.js +65 -0
- package/dist/cli/getting-started.mjs +36 -0
- package/dist/cli/index.d.mts +9 -0
- package/dist/cli/index.d.ts +9 -0
- package/dist/cli/index.js +895 -407
- package/dist/cli/index.mjs +893 -414
- package/dist/{compiler-manifest-DsLbQaew.d.mts → compiler-manifest-x14KL83E.d.mts} +4 -2
- package/dist/{compiler-manifest-DsLbQaew.d.ts → compiler-manifest-x14KL83E.d.ts} +4 -2
- package/dist/index.d.mts +124 -154
- package/dist/index.d.ts +124 -154
- package/dist/index.js +132 -28
- package/dist/index.mjs +132 -28
- package/dist/install-integrity.json +182 -2
- package/dist/monitor-fleet-contract.d.mts +131 -0
- package/dist/monitor-fleet-contract.d.ts +131 -0
- package/dist/monitor-fleet-contract.js +406 -0
- package/dist/monitor-fleet-contract.mjs +367 -0
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/release.d.mts +255 -0
- package/dist/release.d.ts +255 -0
- package/dist/release.js +210 -0
- package/dist/release.mjs +183 -0
- package/package.json +21 -4
|
@@ -0,0 +1,3662 @@
|
|
|
1
|
+
import { appendFileSync, createReadStream, writeFileSync } from 'node:fs';
|
|
2
|
+
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
4
|
+
import { createRequire } from 'node:module';
|
|
5
|
+
import { createHash } from 'node:crypto';
|
|
6
|
+
import { formatWithOptions } from 'node:util';
|
|
7
|
+
import { compileFunction } from 'node:vm';
|
|
8
|
+
import { gunzipSync } from 'node:zlib';
|
|
9
|
+
import { Readable } from 'node:stream';
|
|
10
|
+
import { parse } from 'csv-parse';
|
|
11
|
+
import { install as installSourceMapSupport } from 'source-map-support';
|
|
12
|
+
import {
|
|
13
|
+
createDeferredPlayDataset,
|
|
14
|
+
isPlayDataset,
|
|
15
|
+
isSerializedPlayDataset,
|
|
16
|
+
iteratePlayDatasetInputPages,
|
|
17
|
+
type PlayDataset,
|
|
18
|
+
} from '../../plays/dataset';
|
|
19
|
+
import type { PlayQueueHint } from '../governor/rate-state-backend';
|
|
20
|
+
import { AppRuntimeRateStateBackend } from '../governor/app-runtime-rate-state-backend';
|
|
21
|
+
import { AppRuntimeBudgetStateBackend } from '../governor/app-runtime-budget-state-backend';
|
|
22
|
+
import {
|
|
23
|
+
parseCtxFetchHttpFailureDiagnostic,
|
|
24
|
+
type CtxFetchHttpFailureDiagnostic,
|
|
25
|
+
} from '../log-provenance';
|
|
26
|
+
import type {
|
|
27
|
+
PlayRunnerEvent,
|
|
28
|
+
PlayRunnerExecutionConfig,
|
|
29
|
+
PlayRunOutputWarning,
|
|
30
|
+
PlayRunnerResult,
|
|
31
|
+
} from '../protocol';
|
|
32
|
+
import {
|
|
33
|
+
DURABLE_INVOCATION_FENCE_ATTESTATION_HEADER,
|
|
34
|
+
DURABLE_INVOCATION_FENCE_ATTESTATION_VALUE,
|
|
35
|
+
} from '../durable-invocation-fence';
|
|
36
|
+
import type { PlayRunInputPayload } from '../play-input';
|
|
37
|
+
import { playRunInputObjectView } from '../play-input';
|
|
38
|
+
import {
|
|
39
|
+
normalizePlayRunFailure,
|
|
40
|
+
runtimeLimitExceededCause,
|
|
41
|
+
runtimeLimitExceededFailure,
|
|
42
|
+
} from '../run-failure';
|
|
43
|
+
import { createPlayContext } from '../context';
|
|
44
|
+
import {
|
|
45
|
+
shouldEmitProgressSidecarEvent,
|
|
46
|
+
shouldEmitStdoutEvent,
|
|
47
|
+
} from './progress-sidecar';
|
|
48
|
+
import {
|
|
49
|
+
startRunnerHeartbeatLoop,
|
|
50
|
+
type RunnerHeartbeatStop,
|
|
51
|
+
} from './runtime-heartbeat';
|
|
52
|
+
import {
|
|
53
|
+
formatRunnerGatewayTransportSummary,
|
|
54
|
+
type RunnerGatewayTransportEvent,
|
|
55
|
+
} from './runtime-gateway-transport';
|
|
56
|
+
import { startConfiguredPlayRunnerRuntimeDeadline } from './runtime-deadline';
|
|
57
|
+
import {
|
|
58
|
+
createRunnerProgressPusher,
|
|
59
|
+
type RunnerProgressPusher,
|
|
60
|
+
} from './runtime-progress-push';
|
|
61
|
+
import {
|
|
62
|
+
pushRunnerTerminal,
|
|
63
|
+
runnerTerminalResultForTransport,
|
|
64
|
+
type PushRunnerTerminalOptions,
|
|
65
|
+
type PlayRunnerTerminalTransportResult,
|
|
66
|
+
} from './runtime-terminal-push';
|
|
67
|
+
import { vercelProtectionBypassHeader } from '../vercel-protection';
|
|
68
|
+
import { getPlayRuntimeBatchStrategy } from '../play-runtime-batching-registry';
|
|
69
|
+
import type { ToolResultMetadataInput } from '../tool-result';
|
|
70
|
+
import type {
|
|
71
|
+
ContextOptions,
|
|
72
|
+
IntegrationEventWaitBoundary,
|
|
73
|
+
IntegrationEventWaitHandler,
|
|
74
|
+
IntegrationEventWaitRuntimeContext,
|
|
75
|
+
PlayRowUpdate,
|
|
76
|
+
ResolvedPlayExecution,
|
|
77
|
+
RuntimeStepReceipt,
|
|
78
|
+
PlayExecutionEvent,
|
|
79
|
+
} from '../ctx-types';
|
|
80
|
+
import {
|
|
81
|
+
applyCsvRenameProjection,
|
|
82
|
+
type CsvRenameOptions,
|
|
83
|
+
} from '../csv-rename';
|
|
84
|
+
import { isPlayExecutionSuspendedError } from '../suspension';
|
|
85
|
+
import { assertNoSecretTaint } from '../secret-capability';
|
|
86
|
+
import { PLAY_ARTIFACT_KINDS } from '../backend';
|
|
87
|
+
import { assertNodeRunnableArtifact } from '../../plays/artifact-kind-guard';
|
|
88
|
+
import { createRuntimeDatasetId } from '../dataset-id';
|
|
89
|
+
import {
|
|
90
|
+
formatDatasetAvailabilityLog,
|
|
91
|
+
formatDatasetRegistrationLog,
|
|
92
|
+
} from '../dataset-lifecycle-log';
|
|
93
|
+
import {
|
|
94
|
+
assertCustomerPlayReturnWithinLimit,
|
|
95
|
+
RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES,
|
|
96
|
+
} from '../output-size-limits';
|
|
97
|
+
import {
|
|
98
|
+
resolveRuntimeReferencedPlay,
|
|
99
|
+
createCompleteRuntimeMapRowsOperation,
|
|
100
|
+
readRuntimeSheetDatasetRowKeys,
|
|
101
|
+
readRuntimeSheetDatasetRows,
|
|
102
|
+
releaseRuntimeWorkReceipts,
|
|
103
|
+
startRuntimeSheetDataset,
|
|
104
|
+
} from '../runtime-api';
|
|
105
|
+
import { installPgRuntimePoolDriver } from '../runtime-pg-driver-pg';
|
|
106
|
+
import { createRuntimeToolAuthScopeDigestResolver } from '../auth-scope-resolver';
|
|
107
|
+
import { resolveMapRowOutcomeKey } from '../map-row-outcome';
|
|
108
|
+
import { createRunScopedChildPlayResolver } from './child-play-resolver';
|
|
109
|
+
import { normalizePlayContractCompatibility } from '../../plays/contracts';
|
|
110
|
+
import {
|
|
111
|
+
SLACK_HITL_TIMEOUT_ERROR_CODE,
|
|
112
|
+
SlackHitlTimeoutError,
|
|
113
|
+
} from '../../plays/slack-hitl-timeout-error';
|
|
114
|
+
import { resolvePlayArtifactRuntimeContract } from '../play-artifact-runtime-contract';
|
|
115
|
+
import { validateOptionalPlayAuthoringField } from '../../plays/authoring-contract';
|
|
116
|
+
import { retryRuntimeSheetPersistence } from './runtime-sheet-persistence-retry';
|
|
117
|
+
import {
|
|
118
|
+
resolveMaxConcurrentExternalCalls,
|
|
119
|
+
resolveMaxConcurrentRows,
|
|
120
|
+
} from '../governor/policy';
|
|
121
|
+
import {
|
|
122
|
+
STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
123
|
+
validatePlaySandboxRuntimeLimits,
|
|
124
|
+
} from '../sandbox-runtime-limits';
|
|
125
|
+
import {
|
|
126
|
+
runtimeSheetPageTailHoldCompletedEvent,
|
|
127
|
+
runtimeSheetPageTailWriteMarkerEvent,
|
|
128
|
+
} from '../test-runtime-seams';
|
|
129
|
+
|
|
130
|
+
// Node-runtime play-runner uses the `pg` library for direct Postgres writes
|
|
131
|
+
// against per-run scoped Neon roles. Register the driver eagerly at module
|
|
132
|
+
// load so the first call into runtime-api.ts has a pool factory available.
|
|
133
|
+
installPgRuntimePoolDriver();
|
|
134
|
+
|
|
135
|
+
type SafeOutboundFetchModule =
|
|
136
|
+
typeof import('../../security/safe-outbound-fetch');
|
|
137
|
+
let safeOutboundFetchModule: Promise<SafeOutboundFetchModule> | null = null;
|
|
138
|
+
|
|
139
|
+
function loadSafeOutboundFetch(): Promise<SafeOutboundFetchModule> {
|
|
140
|
+
safeOutboundFetchModule ??= import('../../security/safe-outbound-fetch');
|
|
141
|
+
return safeOutboundFetchModule;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async function safeNodeOutboundFetch(
|
|
145
|
+
input: string | URL,
|
|
146
|
+
init?: RequestInit,
|
|
147
|
+
): Promise<Response> {
|
|
148
|
+
const { safeOutboundFetch } = await loadSafeOutboundFetch();
|
|
149
|
+
return safeOutboundFetch(input, init);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async function safeNodeStreamingOutboundFetch(
|
|
153
|
+
input: string | URL,
|
|
154
|
+
init?: RequestInit,
|
|
155
|
+
): Promise<Response> {
|
|
156
|
+
const { safeOutboundFetch } = await loadSafeOutboundFetch();
|
|
157
|
+
return safeOutboundFetch(input, init, { streamResponseBody: true });
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
import {
|
|
161
|
+
resolveSheetContractForTableNamespace,
|
|
162
|
+
sqlSafePlayColumnName,
|
|
163
|
+
type PlaySheetContract,
|
|
164
|
+
} from '../../plays/static-pipeline';
|
|
165
|
+
import { isIntegrationEventWaitToolId } from '../event-wait-tools';
|
|
166
|
+
import { type PlayBundleArtifact } from '../../plays/artifact-types';
|
|
167
|
+
import {
|
|
168
|
+
RuntimeReceiptStoreAdapter,
|
|
169
|
+
type RuntimeReceiptStoreHandlers,
|
|
170
|
+
} from '../runtime-receipt-store-adapter';
|
|
171
|
+
import {
|
|
172
|
+
applyRowUpdatesViaAppRuntime,
|
|
173
|
+
launchPlayAsyncViaAppRuntime,
|
|
174
|
+
observeDocflowNodeViaAppRuntime,
|
|
175
|
+
} from '../app-runtime-api';
|
|
176
|
+
import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
|
|
177
|
+
import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
|
|
178
|
+
|
|
179
|
+
const playRuntimeRequire = createRequire(`${process.cwd()}/package.json`);
|
|
180
|
+
const moduleRuntime = playRuntimeRequire('module') as {
|
|
181
|
+
Module: {
|
|
182
|
+
new (
|
|
183
|
+
id: string,
|
|
184
|
+
parent?: unknown,
|
|
185
|
+
): {
|
|
186
|
+
exports: unknown;
|
|
187
|
+
filename: string;
|
|
188
|
+
paths: string[];
|
|
189
|
+
require: (id: string) => unknown;
|
|
190
|
+
};
|
|
191
|
+
_nodeModulePaths: (from: string) => string[];
|
|
192
|
+
};
|
|
193
|
+
};
|
|
194
|
+
let sourceMapSupportInstalled = false;
|
|
195
|
+
const runtimeSourceMaps = new Map<string, string>();
|
|
196
|
+
const runtimeSourceFiles = new Map<string, string>();
|
|
197
|
+
const runtimeArtifactReferences = new Map<
|
|
198
|
+
string,
|
|
199
|
+
{ artifactHash: string; sourceMapHash: string; count: number }
|
|
200
|
+
>();
|
|
201
|
+
const CSV_NAMESPACE_PREFIX = 'csv';
|
|
202
|
+
|
|
203
|
+
function assertBundledRuntimeRequire(id: string): void {
|
|
204
|
+
if (
|
|
205
|
+
id === '.' ||
|
|
206
|
+
id.startsWith('./') ||
|
|
207
|
+
id.startsWith('../') ||
|
|
208
|
+
isAbsolute(id) ||
|
|
209
|
+
id.startsWith('file:')
|
|
210
|
+
) {
|
|
211
|
+
throw new Error(
|
|
212
|
+
`Runtime require("${id}") cannot load a local file. Local play dependencies must be bundled before execution.`,
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function createPlayRuntimeRequire(): ReturnType<typeof createRequire> {
|
|
218
|
+
const runtimeRequire = ((id: string) => {
|
|
219
|
+
assertBundledRuntimeRequire(id);
|
|
220
|
+
return playRuntimeRequire(id);
|
|
221
|
+
}) as ReturnType<typeof createRequire>;
|
|
222
|
+
runtimeRequire.resolve = Object.assign(
|
|
223
|
+
(id: string, options?: { paths?: string[] }) => {
|
|
224
|
+
assertBundledRuntimeRequire(id);
|
|
225
|
+
return playRuntimeRequire.resolve(id, options);
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
paths: (id: string) => {
|
|
229
|
+
assertBundledRuntimeRequire(id);
|
|
230
|
+
return playRuntimeRequire.resolve.paths(id);
|
|
231
|
+
},
|
|
232
|
+
},
|
|
233
|
+
);
|
|
234
|
+
runtimeRequire.main = playRuntimeRequire.main;
|
|
235
|
+
runtimeRequire.extensions = playRuntimeRequire.extensions;
|
|
236
|
+
runtimeRequire.cache = playRuntimeRequire.cache;
|
|
237
|
+
return runtimeRequire;
|
|
238
|
+
}
|
|
239
|
+
const WAIT_CONFIG_POLL_MS = 25;
|
|
240
|
+
const WAIT_CONFIG_TIMEOUT_MS = 120_000;
|
|
241
|
+
export const RUNNER_EXECUTION_ACK_TIMEOUT_MS = 75_000;
|
|
242
|
+
const RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS =
|
|
243
|
+
RUNTIME_RELIABILITY_POLICY.gateway.runnerControlMaxAttempts;
|
|
244
|
+
const RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS = [100, 250] as const;
|
|
245
|
+
export const RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS =
|
|
246
|
+
RUNTIME_RELIABILITY_POLICY.gateway.runnerControlRequestTimeoutMs;
|
|
247
|
+
const RUNTIME_RECEIPT_GATEWAY_BATCH_SIZE = 200;
|
|
248
|
+
const RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_WAIT_MS = 15;
|
|
249
|
+
const RUNTIME_RECEIPT_GATEWAY_REQUEST_TIMEOUT_MS =
|
|
250
|
+
RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs;
|
|
251
|
+
const RETRY_SAFE_EVENT_WAIT_ARM_TOOLS = new Set(['test_wait_for_event']);
|
|
252
|
+
|
|
253
|
+
// The one receipt backend: one Play Durability Store Adapter bound to the
|
|
254
|
+
// co-located receipt gateway. There is no host-injected or app-runtime path.
|
|
255
|
+
type RuntimeStepReceiptHandlers = RuntimeReceiptStoreHandlers;
|
|
256
|
+
|
|
257
|
+
type RuntimeToolMetadata = {
|
|
258
|
+
provider?: string | null;
|
|
259
|
+
operation?: string | null;
|
|
260
|
+
extractors?: ToolResultMetadataInput['extractors'];
|
|
261
|
+
actionCacheVersion: string;
|
|
262
|
+
actionCacheReadVersions?: string[];
|
|
263
|
+
queueHints?: unknown;
|
|
264
|
+
retry?: {
|
|
265
|
+
retrySafeTransientHttp?: boolean;
|
|
266
|
+
requiresExecutionFence?: boolean;
|
|
267
|
+
};
|
|
268
|
+
targetGetters?: Record<string, string[]>;
|
|
269
|
+
listExtractorPaths?: string[];
|
|
270
|
+
listIdentityGetters?: Record<string, string[]>;
|
|
271
|
+
eventWait?: boolean;
|
|
272
|
+
asyncOperation?:
|
|
273
|
+
| import('../async-operation').AsyncOperationContractWire
|
|
274
|
+
| null;
|
|
275
|
+
asyncOperationPoll?:
|
|
276
|
+
| import('../async-operation').AsyncOperationPollExecution
|
|
277
|
+
| null;
|
|
278
|
+
};
|
|
279
|
+
|
|
280
|
+
type RuntimeToolAuthScope = {
|
|
281
|
+
digest?: unknown;
|
|
282
|
+
rateScope?: {
|
|
283
|
+
bucketId?: unknown;
|
|
284
|
+
provider?: unknown;
|
|
285
|
+
token?: unknown;
|
|
286
|
+
};
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
export function assertRuntimeSheetAttemptPersistedTerminalRows(input: {
|
|
290
|
+
tableNamespace: string;
|
|
291
|
+
runId: string;
|
|
292
|
+
attemptId?: string | null;
|
|
293
|
+
updatedRows: number;
|
|
294
|
+
expectedRows: number;
|
|
295
|
+
persistedTerminalRows?: number | null;
|
|
296
|
+
fencedKeys?: string[];
|
|
297
|
+
}): void {
|
|
298
|
+
// Kept as a temporary source-compatible seam for callers compiled against
|
|
299
|
+
// the lease model. A partial update now means a newer scheduled write won;
|
|
300
|
+
// it is an observable stale drop, not a run-fatal ownership error.
|
|
301
|
+
void input;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
export function deterministicRuntimeSheetAttemptId(input: {
|
|
305
|
+
playName: string;
|
|
306
|
+
runId: string;
|
|
307
|
+
tableNamespace: string;
|
|
308
|
+
attemptSeq: number;
|
|
309
|
+
}): string {
|
|
310
|
+
const digest = createHash('sha256')
|
|
311
|
+
.update(input.playName)
|
|
312
|
+
.update('\0')
|
|
313
|
+
.update(input.runId)
|
|
314
|
+
.update('\0')
|
|
315
|
+
.update(input.tableNamespace)
|
|
316
|
+
.update('\0')
|
|
317
|
+
.update(String(input.attemptSeq))
|
|
318
|
+
.digest('hex')
|
|
319
|
+
.slice(0, 32);
|
|
320
|
+
return `sheet-attempt:${digest}`;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async function vercelBypassHeaders(
|
|
324
|
+
config: PlayRunnerExecutionConfig,
|
|
325
|
+
): Promise<Record<string, string>> {
|
|
326
|
+
return vercelProtectionBypassHeader(
|
|
327
|
+
config.context.vercelProtectionBypassToken,
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function isRetryableRuntimeControlStatus(status: number): boolean {
|
|
332
|
+
return (
|
|
333
|
+
status === 408 ||
|
|
334
|
+
status === 429 ||
|
|
335
|
+
(status >= 520 && status <= 524) ||
|
|
336
|
+
status === 502 ||
|
|
337
|
+
status === 503 ||
|
|
338
|
+
status === 504
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
async function sleepRunnerRuntimeControlRetry(ms: number): Promise<void> {
|
|
343
|
+
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function runnerRuntimeControlUrlHost(url: string): string {
|
|
347
|
+
try {
|
|
348
|
+
return new URL(url).host;
|
|
349
|
+
} catch {
|
|
350
|
+
return '<unparseable-url>';
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Thrown when a sandbox -> receipt-gateway runtime-control call is redirected
|
|
356
|
+
* (any 3xx). This is a deterministic wiring failure, never a transient one, so
|
|
357
|
+
* it must escape the retry loop immediately without being re-wrapped as a
|
|
358
|
+
* transport error.
|
|
359
|
+
*/
|
|
360
|
+
export class RunnerRuntimeControlRedirectError extends Error {
|
|
361
|
+
constructor(message: string) {
|
|
362
|
+
super(message);
|
|
363
|
+
this.name = 'RunnerRuntimeControlRedirectError';
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
export async function fetchRunnerRuntimeControl(input: {
|
|
368
|
+
url: string;
|
|
369
|
+
init: RequestInit;
|
|
370
|
+
description: string;
|
|
371
|
+
retrySafe: boolean;
|
|
372
|
+
/** Injectable fetch for unit tests; defaults to the global fetch. */
|
|
373
|
+
fetchImpl?: typeof fetch;
|
|
374
|
+
/** Injectable retry sleep for unit tests; defaults to a real timer. */
|
|
375
|
+
sleepImpl?: (ms: number) => Promise<void>;
|
|
376
|
+
/** Per-attempt transport deadline; injectable only to keep timeout tests fast. */
|
|
377
|
+
timeoutMs?: number;
|
|
378
|
+
}): Promise<Response> {
|
|
379
|
+
const fetchImpl = input.fetchImpl ?? fetch;
|
|
380
|
+
const sleepImpl = input.sleepImpl ?? sleepRunnerRuntimeControlRetry;
|
|
381
|
+
for (
|
|
382
|
+
let attempt = 1;
|
|
383
|
+
attempt <= RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS;
|
|
384
|
+
attempt += 1
|
|
385
|
+
) {
|
|
386
|
+
const timeoutMs =
|
|
387
|
+
input.timeoutMs ?? RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS;
|
|
388
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
389
|
+
const signal = input.init.signal
|
|
390
|
+
? AbortSignal.any([input.init.signal, timeoutSignal])
|
|
391
|
+
: timeoutSignal;
|
|
392
|
+
try {
|
|
393
|
+
// Never follow redirects on the sandbox -> receipt-gateway control plane.
|
|
394
|
+
// Every runtime-control call from the Daytona sandbox targets the gateway
|
|
395
|
+
// (context.baseUrl is rewritten to the gateway URL, and the sandbox holds
|
|
396
|
+
// NO Vercel protection-bypass credential). A 3xx therefore means the
|
|
397
|
+
// request escaped the gateway and hit Vercel Deployment Protection bare,
|
|
398
|
+
// whose SSO redirect drops our Authorization/bypass headers. Following it
|
|
399
|
+
// would swallow the failure into an opaque "307 Redirecting..." body from
|
|
400
|
+
// the wrong host. Surface it loudly with the target host instead.
|
|
401
|
+
const response = await fetchImpl(input.url, {
|
|
402
|
+
...input.init,
|
|
403
|
+
redirect: 'manual',
|
|
404
|
+
signal,
|
|
405
|
+
});
|
|
406
|
+
if (response.status >= 300 && response.status < 400) {
|
|
407
|
+
const location = response.headers.get('location') ?? '<none>';
|
|
408
|
+
await response.text().catch(() => '');
|
|
409
|
+
throw new RunnerRuntimeControlRedirectError(
|
|
410
|
+
`${input.description} was redirected (${response.status}) by host ` +
|
|
411
|
+
`${runnerRuntimeControlUrlHost(input.url)} -> ${location}. Sandbox ` +
|
|
412
|
+
`runtime-control calls must terminate at the receipt gateway; a ` +
|
|
413
|
+
`redirect means the request reached Vercel Deployment Protection ` +
|
|
414
|
+
`without a bypass credential. Confirm context.baseUrl points at the ` +
|
|
415
|
+
`gateway and the launch carries vercelProtectionBypassToken.`,
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
if (
|
|
419
|
+
input.retrySafe &&
|
|
420
|
+
attempt < RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS &&
|
|
421
|
+
isRetryableRuntimeControlStatus(response.status)
|
|
422
|
+
) {
|
|
423
|
+
await response.text().catch(() => '');
|
|
424
|
+
await sleepImpl(
|
|
425
|
+
RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS[attempt - 1] ?? 250,
|
|
426
|
+
);
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
// `fetch()` resolves once headers arrive. Leaving the body for callers
|
|
430
|
+
// means this attempt's timeout signal can fire during `response.json()`
|
|
431
|
+
// outside this retry loop. That turns a recoverable, partially delivered
|
|
432
|
+
// gateway response into a terminal child-run failure under a large HITL
|
|
433
|
+
// fanout. Buffer the deliberately small runtime-control response before
|
|
434
|
+
// returning it so one attempt owns both headers and body.
|
|
435
|
+
const body = await response.arrayBuffer();
|
|
436
|
+
return new Response(body, {
|
|
437
|
+
status: response.status,
|
|
438
|
+
statusText: response.statusText,
|
|
439
|
+
headers: response.headers,
|
|
440
|
+
});
|
|
441
|
+
} catch (error) {
|
|
442
|
+
// A redirect is a deterministic wiring failure, not a transient transport
|
|
443
|
+
// fault. Surface it immediately with its host-named message intact.
|
|
444
|
+
if (error instanceof RunnerRuntimeControlRedirectError) {
|
|
445
|
+
throw error;
|
|
446
|
+
}
|
|
447
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
448
|
+
const transportMessage = timeoutSignal.aborted
|
|
449
|
+
? `timed out after ${timeoutMs}ms`
|
|
450
|
+
: message;
|
|
451
|
+
if (!input.retrySafe || attempt >= RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS) {
|
|
452
|
+
throw new Error(
|
|
453
|
+
`${input.description} transport failed: ${transportMessage}`,
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
await sleepImpl(
|
|
457
|
+
RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS[attempt - 1] ?? 250,
|
|
458
|
+
);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
throw new Error(`${input.description} transport failed.`);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function createRuntimeToolMetadataResolver(config: PlayRunnerExecutionConfig) {
|
|
465
|
+
const cached = new Map<string, Promise<RuntimeToolMetadata>>();
|
|
466
|
+
|
|
467
|
+
return async (toolId: string): Promise<RuntimeToolMetadata> => {
|
|
468
|
+
const normalizedToolId = String(toolId || '').trim();
|
|
469
|
+
if (
|
|
470
|
+
!normalizedToolId ||
|
|
471
|
+
!config.context.executorToken ||
|
|
472
|
+
!config.context.baseUrl
|
|
473
|
+
) {
|
|
474
|
+
throw new Error(
|
|
475
|
+
`Runtime metadata lookup for tool ${normalizedToolId || '<empty>'} requires executorToken and baseUrl.`,
|
|
476
|
+
);
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const cachedMetadata = cached.get(normalizedToolId);
|
|
480
|
+
if (cachedMetadata) {
|
|
481
|
+
return await cachedMetadata;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
const promise = (async () => {
|
|
485
|
+
const url = `${config.context.baseUrl}/api/v2/plays/runtime-tools/${encodeURIComponent(normalizedToolId)}`;
|
|
486
|
+
const response = await fetchRunnerRuntimeControl({
|
|
487
|
+
url,
|
|
488
|
+
description: `Runtime metadata lookup for tool ${normalizedToolId}`,
|
|
489
|
+
retrySafe: true,
|
|
490
|
+
init: {
|
|
491
|
+
method: 'GET',
|
|
492
|
+
headers: {
|
|
493
|
+
Authorization: `Bearer ${config.context.executorToken}`,
|
|
494
|
+
...(await vercelBypassHeaders(config)),
|
|
495
|
+
},
|
|
496
|
+
},
|
|
497
|
+
});
|
|
498
|
+
if (!response.ok) {
|
|
499
|
+
throw new Error(
|
|
500
|
+
`Runtime metadata lookup for tool ${normalizedToolId} failed (${response.status}): ${await response.text()}`,
|
|
501
|
+
);
|
|
502
|
+
}
|
|
503
|
+
return (await response.json()) as RuntimeToolMetadata;
|
|
504
|
+
})();
|
|
505
|
+
|
|
506
|
+
cached.set(normalizedToolId, promise);
|
|
507
|
+
return await promise;
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
function createAppRuntimeToolAuthScopeDigestResolver(
|
|
512
|
+
config: PlayRunnerExecutionConfig,
|
|
513
|
+
) {
|
|
514
|
+
const rateScopes = new Map<
|
|
515
|
+
string,
|
|
516
|
+
{ bucketId: string; provider: string; token: string }
|
|
517
|
+
>();
|
|
518
|
+
const digestResolver = createRuntimeToolAuthScopeDigestResolver({
|
|
519
|
+
missingToolIdMessage: 'Runtime auth scope lookup requires tool id.',
|
|
520
|
+
retryDelaysMs: [],
|
|
521
|
+
fetchDigest: async (normalizedToolId) => {
|
|
522
|
+
if (!config.context.executorToken || !config.context.baseUrl) {
|
|
523
|
+
throw new Error(
|
|
524
|
+
`Runtime auth scope lookup for tool ${normalizedToolId || '<empty>'} requires executorToken and baseUrl.`,
|
|
525
|
+
);
|
|
526
|
+
}
|
|
527
|
+
const url = `${config.context.baseUrl}/api/v2/plays/runtime-tools/${encodeURIComponent(normalizedToolId)}/auth-scope`;
|
|
528
|
+
const response = await fetchRunnerRuntimeControl({
|
|
529
|
+
url,
|
|
530
|
+
description: `Runtime auth scope lookup for tool ${normalizedToolId}`,
|
|
531
|
+
retrySafe: true,
|
|
532
|
+
init: {
|
|
533
|
+
method: 'GET',
|
|
534
|
+
headers: {
|
|
535
|
+
Authorization: `Bearer ${config.context.executorToken}`,
|
|
536
|
+
...(await vercelBypassHeaders(config)),
|
|
537
|
+
},
|
|
538
|
+
},
|
|
539
|
+
});
|
|
540
|
+
if (!response.ok) {
|
|
541
|
+
throw new Error(
|
|
542
|
+
`Runtime auth scope lookup for tool ${normalizedToolId} failed (${response.status}): ${await response.text()}`,
|
|
543
|
+
);
|
|
544
|
+
}
|
|
545
|
+
const body = (await response.json()) as RuntimeToolAuthScope;
|
|
546
|
+
const digest = typeof body.digest === 'string' ? body.digest.trim() : '';
|
|
547
|
+
if (!digest) {
|
|
548
|
+
throw new Error(
|
|
549
|
+
`Runtime auth scope lookup for tool ${normalizedToolId} returned no digest.`,
|
|
550
|
+
);
|
|
551
|
+
}
|
|
552
|
+
const rateScope = body.rateScope;
|
|
553
|
+
const bucketId =
|
|
554
|
+
typeof rateScope?.bucketId === 'string'
|
|
555
|
+
? rateScope.bucketId.trim()
|
|
556
|
+
: '';
|
|
557
|
+
const provider =
|
|
558
|
+
typeof rateScope?.provider === 'string'
|
|
559
|
+
? rateScope.provider.trim()
|
|
560
|
+
: '';
|
|
561
|
+
const token =
|
|
562
|
+
typeof rateScope?.token === 'string' ? rateScope.token.trim() : '';
|
|
563
|
+
if (!bucketId || !provider || !token) {
|
|
564
|
+
throw new Error(
|
|
565
|
+
`Runtime auth scope lookup for tool ${normalizedToolId} returned no rate scope.`,
|
|
566
|
+
);
|
|
567
|
+
}
|
|
568
|
+
rateScopes.set(normalizedToolId, { bucketId, provider, token });
|
|
569
|
+
return digest;
|
|
570
|
+
},
|
|
571
|
+
});
|
|
572
|
+
const invalidateDigest = digestResolver.invalidate;
|
|
573
|
+
return Object.assign(digestResolver, {
|
|
574
|
+
invalidate: (toolId?: string | null) => {
|
|
575
|
+
invalidateDigest(toolId);
|
|
576
|
+
const normalized = String(toolId || '').trim();
|
|
577
|
+
if (normalized) rateScopes.delete(normalized);
|
|
578
|
+
else rateScopes.clear();
|
|
579
|
+
},
|
|
580
|
+
rateScope: async (toolId: string, expectedProvider: string) => {
|
|
581
|
+
await digestResolver(toolId);
|
|
582
|
+
const scope = rateScopes.get(toolId.trim());
|
|
583
|
+
if (!scope || scope.provider !== expectedProvider) {
|
|
584
|
+
throw new Error(
|
|
585
|
+
`Runtime rate scope provider mismatch for tool ${toolId}.`,
|
|
586
|
+
);
|
|
587
|
+
}
|
|
588
|
+
return { bucketId: scope.bucketId, token: scope.token };
|
|
589
|
+
},
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function createRateStateBackendForConfig(
|
|
594
|
+
config: PlayRunnerExecutionConfig,
|
|
595
|
+
): Pick<
|
|
596
|
+
ContextOptions,
|
|
597
|
+
'rateState' | 'budgetState' | 'requireSharedRateState'
|
|
598
|
+
> {
|
|
599
|
+
const backend = config.context.rateStateBackend;
|
|
600
|
+
if (!backend || backend.kind === 'in_memory') return {};
|
|
601
|
+
const baseUrl = config.context.baseUrl?.trim();
|
|
602
|
+
const executorToken = config.context.executorToken?.trim();
|
|
603
|
+
if (!baseUrl || !executorToken) {
|
|
604
|
+
throw new Error(
|
|
605
|
+
'Node play runtime requires baseUrl and executorToken for shared rate state.',
|
|
606
|
+
);
|
|
607
|
+
}
|
|
608
|
+
const runtimeApi = {
|
|
609
|
+
baseUrl,
|
|
610
|
+
executorToken,
|
|
611
|
+
vercelProtectionBypassToken:
|
|
612
|
+
config.context.vercelProtectionBypassToken ?? null,
|
|
613
|
+
integrationMode: config.context.integrationMode,
|
|
614
|
+
};
|
|
615
|
+
return {
|
|
616
|
+
requireSharedRateState: true,
|
|
617
|
+
rateState: new AppRuntimeRateStateBackend(runtimeApi, {
|
|
618
|
+
schedulerSchema: backend.schedulerSchema ?? null,
|
|
619
|
+
}),
|
|
620
|
+
budgetState: new AppRuntimeBudgetStateBackend(
|
|
621
|
+
runtimeApi,
|
|
622
|
+
backend.schedulerSchema ?? null,
|
|
623
|
+
),
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
export function createRuntimeEventWaitHandler(
|
|
628
|
+
config: PlayRunnerExecutionConfig,
|
|
629
|
+
toolId: string,
|
|
630
|
+
): IntegrationEventWaitHandler | null {
|
|
631
|
+
const normalizedToolId = String(toolId || '').trim();
|
|
632
|
+
if (
|
|
633
|
+
!normalizedToolId ||
|
|
634
|
+
!config.context.executorToken ||
|
|
635
|
+
!config.context.baseUrl
|
|
636
|
+
) {
|
|
637
|
+
return null;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
async function callRuntimeEventWaitEndpoint(
|
|
641
|
+
input:
|
|
642
|
+
| {
|
|
643
|
+
phase: 'prepare';
|
|
644
|
+
payload: Record<string, unknown>;
|
|
645
|
+
context: IntegrationEventWaitRuntimeContext;
|
|
646
|
+
}
|
|
647
|
+
| {
|
|
648
|
+
phase: 'arm';
|
|
649
|
+
payload: Record<string, unknown>;
|
|
650
|
+
context: IntegrationEventWaitRuntimeContext;
|
|
651
|
+
boundary: IntegrationEventWaitBoundary;
|
|
652
|
+
}
|
|
653
|
+
| {
|
|
654
|
+
phase: 'timeout';
|
|
655
|
+
payload: Record<string, unknown>;
|
|
656
|
+
context: IntegrationEventWaitRuntimeContext;
|
|
657
|
+
boundary: IntegrationEventWaitBoundary;
|
|
658
|
+
},
|
|
659
|
+
): Promise<IntegrationEventWaitBoundary> {
|
|
660
|
+
const url = `${config.context.baseUrl}/api/v2/plays/runtime-tools/${encodeURIComponent(normalizedToolId)}/event-wait`;
|
|
661
|
+
const response = await fetchRunnerRuntimeControl({
|
|
662
|
+
url,
|
|
663
|
+
description: `Event wait runtime call for tool ${normalizedToolId}`,
|
|
664
|
+
retrySafe:
|
|
665
|
+
input.phase === 'prepare' ||
|
|
666
|
+
RETRY_SAFE_EVENT_WAIT_ARM_TOOLS.has(normalizedToolId),
|
|
667
|
+
init: {
|
|
668
|
+
method: 'POST',
|
|
669
|
+
headers: {
|
|
670
|
+
'Content-Type': 'application/json',
|
|
671
|
+
Authorization: `Bearer ${config.context.executorToken}`,
|
|
672
|
+
...(await vercelBypassHeaders(config)),
|
|
673
|
+
},
|
|
674
|
+
body: JSON.stringify(input),
|
|
675
|
+
},
|
|
676
|
+
});
|
|
677
|
+
if (response.status === 404) {
|
|
678
|
+
throw new Error(`No event wait handler for tool ${normalizedToolId}.`);
|
|
679
|
+
}
|
|
680
|
+
if (!response.ok) {
|
|
681
|
+
const bodyText = await response.text();
|
|
682
|
+
const timeoutError = runtimeControlSlackHitlTimeoutError({
|
|
683
|
+
toolId: normalizedToolId,
|
|
684
|
+
phase: input.phase,
|
|
685
|
+
status: response.status,
|
|
686
|
+
bodyText,
|
|
687
|
+
});
|
|
688
|
+
if (timeoutError) throw timeoutError;
|
|
689
|
+
throw new Error(
|
|
690
|
+
formatRuntimeControlFailure({
|
|
691
|
+
description: `Event wait runtime call for tool ${normalizedToolId}`,
|
|
692
|
+
status: response.status,
|
|
693
|
+
bodyText,
|
|
694
|
+
}),
|
|
695
|
+
);
|
|
696
|
+
}
|
|
697
|
+
return (await response.json()) as IntegrationEventWaitBoundary;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
const handler: IntegrationEventWaitHandler = {
|
|
701
|
+
provider: 'runtime',
|
|
702
|
+
toolId: normalizedToolId,
|
|
703
|
+
prepare: async ({ payload, context }) => {
|
|
704
|
+
const boundary = await callRuntimeEventWaitEndpoint({
|
|
705
|
+
phase: 'prepare',
|
|
706
|
+
payload,
|
|
707
|
+
context,
|
|
708
|
+
});
|
|
709
|
+
// The app owns provider-specific timeout semantics. An older app omits
|
|
710
|
+
// this additive field, retaining its legacy replay behavior safely.
|
|
711
|
+
handler.suspendAfterTimeoutMs =
|
|
712
|
+
boundary.timeoutPolicy?.suspendAfterTimeoutMs;
|
|
713
|
+
return boundary;
|
|
714
|
+
},
|
|
715
|
+
arm: async ({ payload, context, boundary }) =>
|
|
716
|
+
await callRuntimeEventWaitEndpoint({
|
|
717
|
+
phase: 'arm',
|
|
718
|
+
payload,
|
|
719
|
+
context,
|
|
720
|
+
boundary,
|
|
721
|
+
}),
|
|
722
|
+
onTimeout: async ({ payload, context, boundary }) => {
|
|
723
|
+
// Only a newer app advertises this phase. Do not send a new request to an
|
|
724
|
+
// older app during the rollout-skew window.
|
|
725
|
+
if (boundary.timeoutPolicy?.finalizeOnTimeout !== true) return;
|
|
726
|
+
await callRuntimeEventWaitEndpoint({
|
|
727
|
+
phase: 'timeout',
|
|
728
|
+
payload,
|
|
729
|
+
context,
|
|
730
|
+
boundary,
|
|
731
|
+
});
|
|
732
|
+
},
|
|
733
|
+
};
|
|
734
|
+
return handler;
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
export function formatRuntimeControlFailure(input: {
|
|
738
|
+
description: string;
|
|
739
|
+
status: number;
|
|
740
|
+
bodyText: string;
|
|
741
|
+
}): string {
|
|
742
|
+
let message = input.bodyText;
|
|
743
|
+
try {
|
|
744
|
+
const parsed = JSON.parse(input.bodyText) as Record<string, unknown>;
|
|
745
|
+
const code =
|
|
746
|
+
typeof parsed.code === 'string' && parsed.code.trim()
|
|
747
|
+
? parsed.code.trim()
|
|
748
|
+
: null;
|
|
749
|
+
const error =
|
|
750
|
+
typeof parsed.error === 'string' && parsed.error.trim()
|
|
751
|
+
? parsed.error.trim()
|
|
752
|
+
: typeof parsed.message === 'string' && parsed.message.trim()
|
|
753
|
+
? parsed.message.trim()
|
|
754
|
+
: null;
|
|
755
|
+
const operatorHint =
|
|
756
|
+
typeof parsed.operator_hint === 'string' && parsed.operator_hint.trim()
|
|
757
|
+
? parsed.operator_hint.trim()
|
|
758
|
+
: null;
|
|
759
|
+
if (code && error) {
|
|
760
|
+
message = [code, error, operatorHint].filter(Boolean).join(': ');
|
|
761
|
+
}
|
|
762
|
+
} catch {
|
|
763
|
+
// Preserve the original runtime-control response for unstructured errors.
|
|
764
|
+
}
|
|
765
|
+
return `${input.description} failed (${input.status}): ${message}`;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
function runtimeControlSlackHitlTimeoutError(input: {
|
|
769
|
+
toolId: string;
|
|
770
|
+
phase: 'prepare' | 'arm' | 'timeout';
|
|
771
|
+
status: number;
|
|
772
|
+
bodyText: string;
|
|
773
|
+
}): SlackHitlTimeoutError | null {
|
|
774
|
+
if (
|
|
775
|
+
input.toolId !== 'slack_message_with_hitl' ||
|
|
776
|
+
input.phase !== 'timeout' ||
|
|
777
|
+
input.status !== 408
|
|
778
|
+
) {
|
|
779
|
+
return null;
|
|
780
|
+
}
|
|
781
|
+
try {
|
|
782
|
+
const body = JSON.parse(input.bodyText) as Record<string, unknown>;
|
|
783
|
+
if (body.code !== SLACK_HITL_TIMEOUT_ERROR_CODE) return null;
|
|
784
|
+
const message =
|
|
785
|
+
typeof body.message === 'string' && body.message.trim()
|
|
786
|
+
? body.message.trim()
|
|
787
|
+
: typeof body.error === 'string' && body.error.trim()
|
|
788
|
+
? body.error.trim()
|
|
789
|
+
: undefined;
|
|
790
|
+
return new SlackHitlTimeoutError(
|
|
791
|
+
message?.replace(
|
|
792
|
+
new RegExp(`^${SLACK_HITL_TIMEOUT_ERROR_CODE}:\\s*`),
|
|
793
|
+
'',
|
|
794
|
+
),
|
|
795
|
+
);
|
|
796
|
+
} catch {
|
|
797
|
+
return null;
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
function inferCsvSheetContract(input: {
|
|
802
|
+
tableNamespace: string;
|
|
803
|
+
rows: readonly Record<string, unknown>[];
|
|
804
|
+
}): PlaySheetContract {
|
|
805
|
+
const seen = new Set<string>();
|
|
806
|
+
const columns = input.rows.flatMap((row) =>
|
|
807
|
+
Object.keys(row).flatMap((field) => {
|
|
808
|
+
const normalizedField = field.trim();
|
|
809
|
+
if (!normalizedField || seen.has(normalizedField)) {
|
|
810
|
+
return [];
|
|
811
|
+
}
|
|
812
|
+
seen.add(normalizedField);
|
|
813
|
+
return [
|
|
814
|
+
{
|
|
815
|
+
id: normalizedField,
|
|
816
|
+
sqlName: sqlSafePlayColumnName(normalizedField),
|
|
817
|
+
source: 'input' as const,
|
|
818
|
+
field: normalizedField,
|
|
819
|
+
},
|
|
820
|
+
];
|
|
821
|
+
}),
|
|
822
|
+
);
|
|
823
|
+
return {
|
|
824
|
+
tableNamespace: input.tableNamespace,
|
|
825
|
+
columns,
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
type ReplayableCsvSource = {
|
|
830
|
+
iterate: () => AsyncIterable<Record<string, unknown>>;
|
|
831
|
+
};
|
|
832
|
+
|
|
833
|
+
async function createStreamingCsvDataset(input: {
|
|
834
|
+
playName: string;
|
|
835
|
+
tableNamespace: string;
|
|
836
|
+
source: ReplayableCsvSource;
|
|
837
|
+
sourceLabel: string;
|
|
838
|
+
options?: CsvRenameOptions;
|
|
839
|
+
}): Promise<PlayDataset<Record<string, unknown>>> {
|
|
840
|
+
let cachedCount: number | null = null;
|
|
841
|
+
const iterateProjected = () =>
|
|
842
|
+
({
|
|
843
|
+
async *[Symbol.asyncIterator]() {
|
|
844
|
+
for await (const row of input.source.iterate()) {
|
|
845
|
+
yield applyCsvRenameProjection([row], input.options)[0]!;
|
|
846
|
+
}
|
|
847
|
+
},
|
|
848
|
+
}) as AsyncIterable<Record<string, unknown>>;
|
|
849
|
+
const collect = async (limit?: number) => {
|
|
850
|
+
const rows: Record<string, unknown>[] = [];
|
|
851
|
+
if (limit !== undefined && limit <= 0) return rows;
|
|
852
|
+
for await (const row of iterateProjected()) {
|
|
853
|
+
rows.push(row);
|
|
854
|
+
if (limit !== undefined && rows.length >= limit) break;
|
|
855
|
+
}
|
|
856
|
+
return rows;
|
|
857
|
+
};
|
|
858
|
+
const previewRows = await collect(10);
|
|
859
|
+
return createDeferredPlayDataset({
|
|
860
|
+
datasetKind: 'csv',
|
|
861
|
+
datasetId: createRuntimeDatasetId(input.playName, input.tableNamespace),
|
|
862
|
+
count: 0,
|
|
863
|
+
knownCount: null,
|
|
864
|
+
previewRows,
|
|
865
|
+
sourceLabel: input.sourceLabel,
|
|
866
|
+
tableNamespace: input.tableNamespace,
|
|
867
|
+
resolvers: {
|
|
868
|
+
count: async () => {
|
|
869
|
+
if (cachedCount !== null) return cachedCount;
|
|
870
|
+
let count = 0;
|
|
871
|
+
for await (const _row of iterateProjected()) count += 1;
|
|
872
|
+
cachedCount = count;
|
|
873
|
+
return count;
|
|
874
|
+
},
|
|
875
|
+
peek: async (limit) => await collect(Math.max(0, limit)),
|
|
876
|
+
materialize: collect,
|
|
877
|
+
iterate: iterateProjected,
|
|
878
|
+
},
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
export interface ExecutePlayRunnerOptions {
|
|
883
|
+
emitEvent?: (event: PlayRunnerEvent) => void;
|
|
884
|
+
signal?: AbortSignal;
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
function emitRunnerEvent(
|
|
888
|
+
event: PlayRunnerEvent,
|
|
889
|
+
options?: ExecutePlayRunnerOptions,
|
|
890
|
+
): void {
|
|
891
|
+
if (options?.emitEvent) {
|
|
892
|
+
options.emitEvent(event);
|
|
893
|
+
return;
|
|
894
|
+
}
|
|
895
|
+
emit(event);
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* In-flight durable docflow observation posts (ADR 0016 rule 2). The wrapper
|
|
900
|
+
* fires them fire-and-forget, but a detached push run's terminal WAKES the
|
|
901
|
+
* parked worker, whose finalize immediately reads the observations table — so
|
|
902
|
+
* a post still in flight over the tunnel loses the race and the terminal
|
|
903
|
+
* ledger misses its nodeIo. Before the graceful terminal emit the runner
|
|
904
|
+
* drains these with a hard cap: the terminal is never failed or delayed
|
|
905
|
+
* beyond the cap, and hard-kill paths (runtime deadline, heartbeat stop,
|
|
906
|
+
* crash epilogue) never wait at all. Empty for undiagrammed plays.
|
|
907
|
+
*/
|
|
908
|
+
const pendingDocflowObservationPosts = new Set<Promise<unknown>>();
|
|
909
|
+
const DOCFLOW_OBSERVATION_TERMINAL_DRAIN_CAP_MS = 2_000;
|
|
910
|
+
|
|
911
|
+
async function drainPendingDocflowObservationPosts(
|
|
912
|
+
options?: ExecutePlayRunnerOptions,
|
|
913
|
+
): Promise<void> {
|
|
914
|
+
if (pendingDocflowObservationPosts.size === 0) {
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
const pendingCount = pendingDocflowObservationPosts.size;
|
|
918
|
+
let capTimer: ReturnType<typeof setTimeout> | null = null;
|
|
919
|
+
const capped = await Promise.race([
|
|
920
|
+
Promise.allSettled([...pendingDocflowObservationPosts]).then(
|
|
921
|
+
() => 'drained' as const,
|
|
922
|
+
),
|
|
923
|
+
new Promise<'capped'>((resolve) => {
|
|
924
|
+
capTimer = setTimeout(
|
|
925
|
+
() => resolve('capped'),
|
|
926
|
+
DOCFLOW_OBSERVATION_TERMINAL_DRAIN_CAP_MS,
|
|
927
|
+
);
|
|
928
|
+
}),
|
|
929
|
+
]).catch(() => 'drained' as const);
|
|
930
|
+
if (capTimer) clearTimeout(capTimer);
|
|
931
|
+
if (capped === 'capped') {
|
|
932
|
+
emitWorkerLog(
|
|
933
|
+
`[warn] docflow observation drain hit the ${DOCFLOW_OBSERVATION_TERMINAL_DRAIN_CAP_MS}ms terminal cap with ${pendingDocflowObservationPosts.size}/${pendingCount} posts still in flight; the worker-side retry fold covers the stragglers`,
|
|
934
|
+
options,
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
function emitWorkerLog(line: string, options?: ExecutePlayRunnerOptions): void {
|
|
940
|
+
emitRunnerEvent(
|
|
941
|
+
{
|
|
942
|
+
type: 'log',
|
|
943
|
+
at: new Date().toISOString(),
|
|
944
|
+
source: 'worker',
|
|
945
|
+
line,
|
|
946
|
+
},
|
|
947
|
+
options,
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
function emitSystemLog(line: string, options?: ExecutePlayRunnerOptions): void {
|
|
952
|
+
emitRunnerEvent(
|
|
953
|
+
{
|
|
954
|
+
type: 'log',
|
|
955
|
+
at: new Date().toISOString(),
|
|
956
|
+
source: 'system',
|
|
957
|
+
line,
|
|
958
|
+
},
|
|
959
|
+
options,
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
export function playRunnerContextCachePolicy(
|
|
964
|
+
context: PlayRunnerExecutionConfig['context'],
|
|
965
|
+
): ContextOptions['cachePolicy'] | undefined {
|
|
966
|
+
if (context.forceToolRefresh) {
|
|
967
|
+
return {
|
|
968
|
+
...(context.force ? { forceStepRefresh: true } : {}),
|
|
969
|
+
forceToolRefresh: true,
|
|
970
|
+
forceFailedToolRefresh: true,
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
if (context.force) {
|
|
974
|
+
return { forceStepRefresh: true, forceFailedToolRefresh: true };
|
|
975
|
+
}
|
|
976
|
+
return undefined;
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
export function playRunnerContextRuntimeSheetForce(
|
|
980
|
+
context: PlayRunnerExecutionConfig['context'],
|
|
981
|
+
): boolean {
|
|
982
|
+
return context.force === true || context.forceToolRefresh === true;
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
export function playRunnerInvocationFenceFlags(
|
|
986
|
+
context: PlayRunnerExecutionConfig['context'],
|
|
987
|
+
): Pick<
|
|
988
|
+
ContextOptions,
|
|
989
|
+
'requestDurableInvocationFence' | 'durableInvocationFence'
|
|
990
|
+
> {
|
|
991
|
+
return {
|
|
992
|
+
requestDurableInvocationFence:
|
|
993
|
+
context.requestDurableInvocationFence === true ||
|
|
994
|
+
(Boolean(context.receiptGatewayBaseUrl?.trim()) &&
|
|
995
|
+
context.baseUrl?.replace(/\/$/, '') ===
|
|
996
|
+
context.receiptGatewayBaseUrl?.trim().replace(/\/$/, '')),
|
|
997
|
+
durableInvocationFence: context.durableInvocationFence === true,
|
|
998
|
+
};
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
export function playRunnerRuntimeReceiptsEnabled(
|
|
1002
|
+
context: PlayRunnerExecutionConfig['context'],
|
|
1003
|
+
): boolean {
|
|
1004
|
+
return Boolean(
|
|
1005
|
+
context.orgId &&
|
|
1006
|
+
context.playName &&
|
|
1007
|
+
context.baseUrl &&
|
|
1008
|
+
context.executorToken &&
|
|
1009
|
+
context.baseUrl.trim(),
|
|
1010
|
+
);
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/**
|
|
1014
|
+
* Runtime Sheets belong to an organization and Play, not to the human who
|
|
1015
|
+
* initiated the run. Triggered runs intentionally have no actor email.
|
|
1016
|
+
*/
|
|
1017
|
+
export function assertPlayRunnerRuntimeSheetPersistenceContext(
|
|
1018
|
+
context: PlayRunnerExecutionConfig['context'],
|
|
1019
|
+
): asserts context is PlayRunnerExecutionConfig['context'] & {
|
|
1020
|
+
orgId: string;
|
|
1021
|
+
playName: string;
|
|
1022
|
+
baseUrl: string;
|
|
1023
|
+
executorToken: string;
|
|
1024
|
+
} {
|
|
1025
|
+
const missing = [
|
|
1026
|
+
!context.orgId?.trim() ? 'organization id' : null,
|
|
1027
|
+
!context.playName?.trim() ? 'Play name' : null,
|
|
1028
|
+
!(context.runId?.trim() || context.workflowId?.trim()) ? 'run id' : null,
|
|
1029
|
+
!context.baseUrl?.trim() ? 'runtime API origin' : null,
|
|
1030
|
+
!context.executorToken?.trim() ? 'executor token' : null,
|
|
1031
|
+
].filter((field): field is string => field !== null);
|
|
1032
|
+
if (missing.length > 0) {
|
|
1033
|
+
throw new Error(
|
|
1034
|
+
`Runtime Sheet persistence is required for ctx.dataset() and ctx.csv(); run launch is missing ${missing.join(', ')}.`,
|
|
1035
|
+
);
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
function collectReturnedDatasets(value: unknown): unknown[] {
|
|
1040
|
+
const datasets: unknown[] = [];
|
|
1041
|
+
const seen = new Set<unknown>();
|
|
1042
|
+
|
|
1043
|
+
const visit = (entry: unknown) => {
|
|
1044
|
+
if (!entry || typeof entry !== 'object' || seen.has(entry)) {
|
|
1045
|
+
return;
|
|
1046
|
+
}
|
|
1047
|
+
seen.add(entry);
|
|
1048
|
+
if (isPlayDataset(entry)) {
|
|
1049
|
+
datasets.push(entry);
|
|
1050
|
+
return;
|
|
1051
|
+
}
|
|
1052
|
+
if (Array.isArray(entry)) {
|
|
1053
|
+
for (const item of entry) {
|
|
1054
|
+
visit(item);
|
|
1055
|
+
}
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1058
|
+
for (const child of Object.values(entry as Record<string, unknown>)) {
|
|
1059
|
+
visit(child);
|
|
1060
|
+
}
|
|
1061
|
+
};
|
|
1062
|
+
|
|
1063
|
+
visit(value);
|
|
1064
|
+
return datasets;
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// Absorb ENOBUFS / EPIPE on stdout — the parent reads stdout chunks anyway,
|
|
1068
|
+
// and crashing the runner process on backpressure loses the entire run.
|
|
1069
|
+
process.stdout.on('error', (err: NodeJS.ErrnoException) => {
|
|
1070
|
+
if (err.code === 'ENOBUFS' || err.code === 'EPIPE') return;
|
|
1071
|
+
throw err;
|
|
1072
|
+
});
|
|
1073
|
+
|
|
1074
|
+
// Push-execution live progress channel. When the launch enables push execution
|
|
1075
|
+
// (Daytona sandbox runner), progress events are PUSHED to the receipt gateway
|
|
1076
|
+
// instead of polled out of a sidecar file by the worker. Set once in main().
|
|
1077
|
+
let runnerProgressPusher: RunnerProgressPusher | null = null;
|
|
1078
|
+
|
|
1079
|
+
function emit(event: PlayRunnerEvent): void {
|
|
1080
|
+
emitProgressSidecarEvent(event);
|
|
1081
|
+
if (runnerProgressPusher && shouldEmitProgressSidecarEvent(event)) {
|
|
1082
|
+
runnerProgressPusher.enqueue(event);
|
|
1083
|
+
}
|
|
1084
|
+
if (!shouldEmitStdoutEvent(event)) {
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
try {
|
|
1088
|
+
process.stdout.write(`${JSON.stringify(event)}\n`);
|
|
1089
|
+
} catch (err: unknown) {
|
|
1090
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
1091
|
+
if (code === 'ENOBUFS' || code === 'EPIPE') return;
|
|
1092
|
+
throw err;
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
const PROGRESS_SIDECAR_SEGMENT_BYTES = 256 * 1024;
|
|
1097
|
+
let progressSidecarState: {
|
|
1098
|
+
basePath: string;
|
|
1099
|
+
segment: number;
|
|
1100
|
+
segmentBytes: number;
|
|
1101
|
+
} | null = null;
|
|
1102
|
+
|
|
1103
|
+
function emitProgressSidecarEvent(event: PlayRunnerEvent): void {
|
|
1104
|
+
if (!shouldEmitProgressSidecarEvent(event)) {
|
|
1105
|
+
return;
|
|
1106
|
+
}
|
|
1107
|
+
const progressPath =
|
|
1108
|
+
process.env.DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH?.trim();
|
|
1109
|
+
if (!progressPath) {
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
1112
|
+
try {
|
|
1113
|
+
const line = `${JSON.stringify(event)}\n`;
|
|
1114
|
+
const lineBytes = Buffer.byteLength(line);
|
|
1115
|
+
if (
|
|
1116
|
+
!progressSidecarState ||
|
|
1117
|
+
progressSidecarState.basePath !== progressPath
|
|
1118
|
+
) {
|
|
1119
|
+
progressSidecarState = {
|
|
1120
|
+
basePath: progressPath,
|
|
1121
|
+
segment: 0,
|
|
1122
|
+
segmentBytes: 0,
|
|
1123
|
+
};
|
|
1124
|
+
writeFileSync(progressPath, '0', 'utf-8');
|
|
1125
|
+
}
|
|
1126
|
+
if (
|
|
1127
|
+
progressSidecarState.segmentBytes > 0 &&
|
|
1128
|
+
progressSidecarState.segmentBytes + lineBytes >
|
|
1129
|
+
PROGRESS_SIDECAR_SEGMENT_BYTES
|
|
1130
|
+
) {
|
|
1131
|
+
progressSidecarState.segment += 1;
|
|
1132
|
+
progressSidecarState.segmentBytes = 0;
|
|
1133
|
+
writeFileSync(
|
|
1134
|
+
progressPath,
|
|
1135
|
+
String(progressSidecarState.segment),
|
|
1136
|
+
'utf-8',
|
|
1137
|
+
);
|
|
1138
|
+
}
|
|
1139
|
+
const segmentPath = `${progressPath}.${String(
|
|
1140
|
+
progressSidecarState.segment,
|
|
1141
|
+
).padStart(6, '0')}`;
|
|
1142
|
+
appendFileSync(segmentPath, line, 'utf-8');
|
|
1143
|
+
progressSidecarState.segmentBytes += lineBytes;
|
|
1144
|
+
} catch {
|
|
1145
|
+
// Best-effort live observability. The normal stdout result stream remains
|
|
1146
|
+
// authoritative, so a sidecar write failure must not fail the play.
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
async function writeRunnerEventToStdout(
|
|
1151
|
+
event:
|
|
1152
|
+
| PlayRunnerEvent
|
|
1153
|
+
| { type: 'result'; result: PlayRunnerTerminalTransportResult },
|
|
1154
|
+
): Promise<void> {
|
|
1155
|
+
await new Promise<void>((resolve, reject) => {
|
|
1156
|
+
process.stdout.write(`${JSON.stringify(event)}\n`, (error) => {
|
|
1157
|
+
if (error) {
|
|
1158
|
+
reject(error);
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
resolve();
|
|
1162
|
+
});
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
function truncateUtf8Prefix(value: string, maxBytes: number): string {
|
|
1167
|
+
if (Buffer.byteLength(value, 'utf8') <= maxBytes) return value;
|
|
1168
|
+
const encoded = Buffer.from(value, 'utf8');
|
|
1169
|
+
for (let end = Math.min(encoded.length, maxBytes); end > 0; end -= 1) {
|
|
1170
|
+
const candidate = encoded.subarray(0, end).toString('utf8');
|
|
1171
|
+
if (Buffer.byteLength(candidate, 'utf8') <= maxBytes) return candidate;
|
|
1172
|
+
}
|
|
1173
|
+
return '';
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/**
|
|
1177
|
+
* The Daytona crash pusher reads a bounded tail of merged stdout/stderr. Once
|
|
1178
|
+
* its authoritative `result` event is written, no runner-owned diagnostic may
|
|
1179
|
+
* push that event out of the tail. The normal progress pusher is stopped before
|
|
1180
|
+
* the result; this bounds only terminal-push retry diagnostics that follow it.
|
|
1181
|
+
*/
|
|
1182
|
+
function createPostTerminalDiagnosticWriter(): NonNullable<
|
|
1183
|
+
PushRunnerTerminalOptions['warn']
|
|
1184
|
+
> {
|
|
1185
|
+
let remaining = RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES;
|
|
1186
|
+
return (message, detail) => {
|
|
1187
|
+
if (remaining <= 1) return;
|
|
1188
|
+
const prefix = `[play-runner.terminal-push] ${message} `;
|
|
1189
|
+
const prefixBytes = Buffer.byteLength(prefix, 'utf8');
|
|
1190
|
+
if (remaining <= prefixBytes + 1) return;
|
|
1191
|
+
const formatted = formatWithOptions(
|
|
1192
|
+
{
|
|
1193
|
+
breakLength: Infinity,
|
|
1194
|
+
colors: false,
|
|
1195
|
+
compact: true,
|
|
1196
|
+
depth: 4,
|
|
1197
|
+
maxArrayLength: 20,
|
|
1198
|
+
maxStringLength: 4_096,
|
|
1199
|
+
},
|
|
1200
|
+
detail,
|
|
1201
|
+
);
|
|
1202
|
+
const suffix = truncateUtf8Prefix(formatted, remaining - prefixBytes - 1);
|
|
1203
|
+
const line = `${prefix}${suffix}\n`;
|
|
1204
|
+
const bytes = Buffer.byteLength(line, 'utf8');
|
|
1205
|
+
if (bytes > remaining) return;
|
|
1206
|
+
remaining -= bytes;
|
|
1207
|
+
try {
|
|
1208
|
+
process.stderr.write(line);
|
|
1209
|
+
} catch (error: unknown) {
|
|
1210
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
1211
|
+
if (code !== 'ENOBUFS' && code !== 'EPIPE') throw error;
|
|
1212
|
+
}
|
|
1213
|
+
};
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
async function exitRunner(exitCode: number): Promise<never> {
|
|
1217
|
+
await Promise.race([
|
|
1218
|
+
Promise.resolve(),
|
|
1219
|
+
new Promise<void>((resolve) => setTimeout(resolve, 1_000)),
|
|
1220
|
+
]);
|
|
1221
|
+
|
|
1222
|
+
process.exit(exitCode);
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
async function emitAndExit(
|
|
1226
|
+
event:
|
|
1227
|
+
| PlayRunnerEvent
|
|
1228
|
+
| { type: 'result'; result: PlayRunnerTerminalTransportResult },
|
|
1229
|
+
exitCode: number,
|
|
1230
|
+
): Promise<never> {
|
|
1231
|
+
await writeRunnerEventToStdout(event);
|
|
1232
|
+
return exitRunner(exitCode);
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
function parseCsvStream(
|
|
1236
|
+
source: NodeJS.ReadableStream,
|
|
1237
|
+
): AsyncIterable<Record<string, unknown>> {
|
|
1238
|
+
const parser = parse({ columns: true, skip_empty_lines: true, trim: true });
|
|
1239
|
+
// `Readable.pipe()` does not forward source errors to the destination. A
|
|
1240
|
+
// missing staged file therefore used to emit an unhandled `error` from
|
|
1241
|
+
// createReadStream and kill the Daytona process outside main().catch().
|
|
1242
|
+
// Route it through the parser's async iterator so normal runner failure
|
|
1243
|
+
// handling can push an actionable terminal instead.
|
|
1244
|
+
source.once('error', (error) => parser.destroy(error));
|
|
1245
|
+
return source.pipe(parser) as AsyncIterable<Record<string, unknown>>;
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
function createCsvSource(input: {
|
|
1249
|
+
path?: string;
|
|
1250
|
+
url?: string;
|
|
1251
|
+
contentBase64?: string;
|
|
1252
|
+
}): ReplayableCsvSource {
|
|
1253
|
+
return {
|
|
1254
|
+
iterate: () => {
|
|
1255
|
+
if (input.contentBase64) {
|
|
1256
|
+
return parseCsvStream(
|
|
1257
|
+
Readable.from([Buffer.from(input.contentBase64, 'base64')]),
|
|
1258
|
+
);
|
|
1259
|
+
}
|
|
1260
|
+
if (input.url) {
|
|
1261
|
+
return {
|
|
1262
|
+
async *[Symbol.asyncIterator]() {
|
|
1263
|
+
const response = await fetch(input.url!);
|
|
1264
|
+
if (!response.ok || !response.body) {
|
|
1265
|
+
throw new Error(
|
|
1266
|
+
`CSV source URL failed with status ${response.status}.`,
|
|
1267
|
+
);
|
|
1268
|
+
}
|
|
1269
|
+
yield* parseCsvStream(
|
|
1270
|
+
Readable.fromWeb(
|
|
1271
|
+
response.body as Parameters<typeof Readable.fromWeb>[0],
|
|
1272
|
+
),
|
|
1273
|
+
);
|
|
1274
|
+
},
|
|
1275
|
+
} as AsyncIterable<Record<string, unknown>>;
|
|
1276
|
+
}
|
|
1277
|
+
if (!input.path) throw new Error('CSV source path is required.');
|
|
1278
|
+
return parseCsvStream(createReadStream(input.path));
|
|
1279
|
+
},
|
|
1280
|
+
};
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
/**
|
|
1284
|
+
* Serialize a runner failure with the message guaranteed present. Bun can
|
|
1285
|
+
* surface `error.stack` without the leading "Error: <message>" line (seen on
|
|
1286
|
+
* governor guard errors thrown deep in async chains), and the run's terminal
|
|
1287
|
+
* error text is what users and the guard e2e assertions read — so never trust
|
|
1288
|
+
* the stack alone to carry the message.
|
|
1289
|
+
*/
|
|
1290
|
+
function formatRunnerFailure(error: unknown): string {
|
|
1291
|
+
if (!(error instanceof Error)) {
|
|
1292
|
+
return String(error);
|
|
1293
|
+
}
|
|
1294
|
+
const stack = error.stack ?? '';
|
|
1295
|
+
if (!error.message.trim()) {
|
|
1296
|
+
return stack || String(error);
|
|
1297
|
+
}
|
|
1298
|
+
if (stack.includes(error.message)) {
|
|
1299
|
+
return stack;
|
|
1300
|
+
}
|
|
1301
|
+
const headline = `${error.name || 'Error'}: ${error.message}`;
|
|
1302
|
+
if (!stack) {
|
|
1303
|
+
return headline;
|
|
1304
|
+
}
|
|
1305
|
+
const frames = stack.startsWith(`${error.name || 'Error'}\n`)
|
|
1306
|
+
? stack.slice((error.name || 'Error').length + 1)
|
|
1307
|
+
: stack;
|
|
1308
|
+
return `${headline}\n${frames}`;
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
function ensureSourceMapSupport(): void {
|
|
1312
|
+
if (sourceMapSupportInstalled) {
|
|
1313
|
+
return;
|
|
1314
|
+
}
|
|
1315
|
+
installSourceMapSupport({
|
|
1316
|
+
environment: 'node',
|
|
1317
|
+
handleUncaughtExceptions: false,
|
|
1318
|
+
// A runner can execute many Play revisions. `source-map-support` otherwise
|
|
1319
|
+
// caches forever by virtual filename and can map a later revision through
|
|
1320
|
+
// an older artifact's map. Stack formatting is synchronous, so clearing
|
|
1321
|
+
// this internal cache between operations is concurrency-safe.
|
|
1322
|
+
emptyCacheBetweenOperations: true,
|
|
1323
|
+
retrieveFile(source) {
|
|
1324
|
+
return runtimeSourceFiles.get(source) ?? '';
|
|
1325
|
+
},
|
|
1326
|
+
retrieveSourceMap(source) {
|
|
1327
|
+
const map = runtimeSourceMaps.get(source);
|
|
1328
|
+
if (!map) {
|
|
1329
|
+
return null;
|
|
1330
|
+
}
|
|
1331
|
+
return {
|
|
1332
|
+
url: source,
|
|
1333
|
+
map,
|
|
1334
|
+
};
|
|
1335
|
+
},
|
|
1336
|
+
});
|
|
1337
|
+
sourceMapSupportInstalled = true;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
function retainRuntimeArtifact(artifact: PlayBundleArtifact): () => void {
|
|
1341
|
+
const existing = runtimeArtifactReferences.get(artifact.virtualFilename);
|
|
1342
|
+
if (
|
|
1343
|
+
existing &&
|
|
1344
|
+
(existing.artifactHash !== artifact.artifactHash ||
|
|
1345
|
+
existing.sourceMapHash !== artifact.sourceMapHash)
|
|
1346
|
+
) {
|
|
1347
|
+
throw new Error(
|
|
1348
|
+
`Concurrent Play artifacts cannot share virtual filename ${artifact.virtualFilename} with different contents.`,
|
|
1349
|
+
);
|
|
1350
|
+
}
|
|
1351
|
+
runtimeSourceFiles.set(artifact.virtualFilename, artifact.bundledCode);
|
|
1352
|
+
runtimeSourceMaps.set(artifact.virtualFilename, artifact.sourceMap);
|
|
1353
|
+
runtimeArtifactReferences.set(artifact.virtualFilename, {
|
|
1354
|
+
artifactHash: artifact.artifactHash,
|
|
1355
|
+
sourceMapHash: artifact.sourceMapHash,
|
|
1356
|
+
count: (existing?.count ?? 0) + 1,
|
|
1357
|
+
});
|
|
1358
|
+
|
|
1359
|
+
let released = false;
|
|
1360
|
+
return () => {
|
|
1361
|
+
if (released) return;
|
|
1362
|
+
released = true;
|
|
1363
|
+
const current = runtimeArtifactReferences.get(artifact.virtualFilename);
|
|
1364
|
+
if (
|
|
1365
|
+
!current ||
|
|
1366
|
+
current.artifactHash !== artifact.artifactHash ||
|
|
1367
|
+
current.sourceMapHash !== artifact.sourceMapHash
|
|
1368
|
+
) {
|
|
1369
|
+
return;
|
|
1370
|
+
}
|
|
1371
|
+
if (current.count > 1) {
|
|
1372
|
+
runtimeArtifactReferences.set(artifact.virtualFilename, {
|
|
1373
|
+
...current,
|
|
1374
|
+
count: current.count - 1,
|
|
1375
|
+
});
|
|
1376
|
+
return;
|
|
1377
|
+
}
|
|
1378
|
+
runtimeArtifactReferences.delete(artifact.virtualFilename);
|
|
1379
|
+
runtimeSourceFiles.delete(artifact.virtualFilename);
|
|
1380
|
+
runtimeSourceMaps.delete(artifact.virtualFilename);
|
|
1381
|
+
};
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
async function executeRuntimePlayCode(input: {
|
|
1385
|
+
artifact: PlayBundleArtifact;
|
|
1386
|
+
ctx: ReturnType<typeof createPlayContext>;
|
|
1387
|
+
runtimeInput: PlayRunInputPayload;
|
|
1388
|
+
options?: ExecutePlayRunnerOptions;
|
|
1389
|
+
}): Promise<unknown> {
|
|
1390
|
+
const compileStartedAt = Date.now();
|
|
1391
|
+
// Defense-in-depth at the load seam: this runner compiles play code with
|
|
1392
|
+
// Module._compile, which cannot resolve `cloudflare:workers` scheme imports.
|
|
1393
|
+
// If a historical or malformed bundle reaches here, fail loudly with a
|
|
1394
|
+
// directional artifact mismatch instead of the opaque scheme error thrown at
|
|
1395
|
+
// compile time.
|
|
1396
|
+
assertNodeRunnableArtifact({
|
|
1397
|
+
artifact: input.artifact,
|
|
1398
|
+
expectedKind: PLAY_ARTIFACT_KINDS.cjsNode20,
|
|
1399
|
+
context: `play-runner ${input.artifact.virtualFilename}`,
|
|
1400
|
+
});
|
|
1401
|
+
ensureSourceMapSupport();
|
|
1402
|
+
const releaseRuntimeArtifact = retainRuntimeArtifact(input.artifact);
|
|
1403
|
+
try {
|
|
1404
|
+
const runtimeModule = new moduleRuntime.Module(
|
|
1405
|
+
input.artifact.virtualFilename,
|
|
1406
|
+
);
|
|
1407
|
+
runtimeModule.filename = input.artifact.virtualFilename;
|
|
1408
|
+
runtimeModule.paths = moduleRuntime.Module._nodeModulePaths(process.cwd());
|
|
1409
|
+
// Bun's `Module` compatibility object does not install `module.require`.
|
|
1410
|
+
// Install a fresh package-only wrapper for both CommonJS entry points. Local
|
|
1411
|
+
// files belong in the bundle; resolving them from the host repo would cross
|
|
1412
|
+
// the artifact boundary, and sharing one function would leak mutations.
|
|
1413
|
+
const runtimeRequire = createPlayRuntimeRequire();
|
|
1414
|
+
runtimeModule.require = runtimeRequire;
|
|
1415
|
+
// Resolve free `console` references through a per-run context extension.
|
|
1416
|
+
// This keeps directive prologues and customer-declared top-level `console`
|
|
1417
|
+
// bindings intact, while avoiding a process-global console swap that could
|
|
1418
|
+
// mix concurrent Absurd claims.
|
|
1419
|
+
const compiled = compileFunction(
|
|
1420
|
+
input.artifact.bundledCode,
|
|
1421
|
+
['exports', 'require', 'module', '__filename', '__dirname'],
|
|
1422
|
+
{
|
|
1423
|
+
filename: input.artifact.virtualFilename,
|
|
1424
|
+
contextExtensions: [
|
|
1425
|
+
{ console: createCapturedCustomerConsole(input.ctx) },
|
|
1426
|
+
],
|
|
1427
|
+
},
|
|
1428
|
+
);
|
|
1429
|
+
compiled.call(
|
|
1430
|
+
runtimeModule.exports,
|
|
1431
|
+
runtimeModule.exports,
|
|
1432
|
+
runtimeRequire,
|
|
1433
|
+
runtimeModule,
|
|
1434
|
+
input.artifact.virtualFilename,
|
|
1435
|
+
dirname(input.artifact.virtualFilename),
|
|
1436
|
+
);
|
|
1437
|
+
emitWorkerLog(
|
|
1438
|
+
`[perf] artifact compile elapsed_ms=${Date.now() - compileStartedAt}`,
|
|
1439
|
+
input.options,
|
|
1440
|
+
);
|
|
1441
|
+
|
|
1442
|
+
const candidate =
|
|
1443
|
+
typeof runtimeModule.exports === 'function'
|
|
1444
|
+
? runtimeModule.exports
|
|
1445
|
+
: (runtimeModule.exports as { default?: unknown }).default;
|
|
1446
|
+
|
|
1447
|
+
if (typeof candidate !== 'function') {
|
|
1448
|
+
throw new Error(
|
|
1449
|
+
'Bundled play module must default export an async function.',
|
|
1450
|
+
);
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
const userFunctionStartedAt = Date.now();
|
|
1454
|
+
const result = await (
|
|
1455
|
+
candidate as (
|
|
1456
|
+
ctx: unknown,
|
|
1457
|
+
runtimeInput: PlayRunInputPayload,
|
|
1458
|
+
) => Promise<unknown>
|
|
1459
|
+
)(input.ctx, input.runtimeInput);
|
|
1460
|
+
// A Play may catch an individual tool failure, but a runner-owned fatal
|
|
1461
|
+
// tool failure still cannot settle as a normal result. Today this covers
|
|
1462
|
+
// Deepline's own insufficient-credit denial and a Slack HITL deadline
|
|
1463
|
+
// whose Slack-card replacement was confirmed. Provider capacity remains a
|
|
1464
|
+
// normal typed waterfall miss.
|
|
1465
|
+
input.ctx.assertNoRunFatalToolFailure();
|
|
1466
|
+
emitWorkerLog(
|
|
1467
|
+
`[perf] user function returned elapsed_ms=${Date.now() - userFunctionStartedAt}`,
|
|
1468
|
+
input.options,
|
|
1469
|
+
);
|
|
1470
|
+
return result;
|
|
1471
|
+
} catch (error) {
|
|
1472
|
+
// Error.stack is lazy in V8. Materialize it while this artifact's source
|
|
1473
|
+
// map is still registered; the mapped string remains on the Error after
|
|
1474
|
+
// the in-memory artifact is released and crosses the runner boundary.
|
|
1475
|
+
if (error instanceof Error) void error.stack;
|
|
1476
|
+
throw error;
|
|
1477
|
+
} finally {
|
|
1478
|
+
releaseRuntimeArtifact();
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
const CAPTURED_CONSOLE_MAX_CHARS = 2_000;
|
|
1483
|
+
|
|
1484
|
+
function logLevelForCapturedConsoleMethod(
|
|
1485
|
+
method: string,
|
|
1486
|
+
): 'debug' | 'info' | 'warn' | 'error' {
|
|
1487
|
+
if (method === 'debug' || method === 'trace') return 'debug';
|
|
1488
|
+
if (method === 'warn') return 'warn';
|
|
1489
|
+
if (method === 'error' || method === 'assert') return 'error';
|
|
1490
|
+
return 'info';
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
function createCapturedCustomerConsole(
|
|
1494
|
+
ctx: ReturnType<typeof createPlayContext>,
|
|
1495
|
+
): Console {
|
|
1496
|
+
const counters = new Map<string, number>();
|
|
1497
|
+
const timers = new Map<string, number>();
|
|
1498
|
+
const capture =
|
|
1499
|
+
(method: string) =>
|
|
1500
|
+
(...args: unknown[]) => {
|
|
1501
|
+
assertNoSecretTaint(args, `console.${method}`);
|
|
1502
|
+
const formatted = formatWithOptions(
|
|
1503
|
+
{
|
|
1504
|
+
breakLength: Infinity,
|
|
1505
|
+
colors: false,
|
|
1506
|
+
compact: true,
|
|
1507
|
+
depth: 4,
|
|
1508
|
+
maxArrayLength: 20,
|
|
1509
|
+
maxStringLength: 2_000,
|
|
1510
|
+
},
|
|
1511
|
+
...args,
|
|
1512
|
+
);
|
|
1513
|
+
const message =
|
|
1514
|
+
formatted.length > CAPTURED_CONSOLE_MAX_CHARS
|
|
1515
|
+
? `${formatted.slice(0, CAPTURED_CONSOLE_MAX_CHARS - 1)}…`
|
|
1516
|
+
: formatted;
|
|
1517
|
+
ctx.log(`[console.${method}] ${message}`, {
|
|
1518
|
+
level: logLevelForCapturedConsoleMethod(method),
|
|
1519
|
+
});
|
|
1520
|
+
};
|
|
1521
|
+
|
|
1522
|
+
const captured = {
|
|
1523
|
+
assert(condition?: boolean, ...args: unknown[]) {
|
|
1524
|
+
if (!condition) {
|
|
1525
|
+
capture('assert')('Assertion failed', ...args);
|
|
1526
|
+
}
|
|
1527
|
+
},
|
|
1528
|
+
clear() {},
|
|
1529
|
+
count(label = 'default') {
|
|
1530
|
+
const next = (counters.get(label) ?? 0) + 1;
|
|
1531
|
+
counters.set(label, next);
|
|
1532
|
+
capture('count')(`${label}: ${next}`);
|
|
1533
|
+
},
|
|
1534
|
+
countReset(label = 'default') {
|
|
1535
|
+
counters.delete(label);
|
|
1536
|
+
},
|
|
1537
|
+
debug: capture('debug'),
|
|
1538
|
+
dir: capture('dir'),
|
|
1539
|
+
dirxml: capture('dirxml'),
|
|
1540
|
+
error: capture('error'),
|
|
1541
|
+
group: capture('group'),
|
|
1542
|
+
groupCollapsed: capture('groupCollapsed'),
|
|
1543
|
+
groupEnd() {},
|
|
1544
|
+
info: capture('info'),
|
|
1545
|
+
log: capture('log'),
|
|
1546
|
+
profile() {},
|
|
1547
|
+
profileEnd() {},
|
|
1548
|
+
table: capture('table'),
|
|
1549
|
+
time(label = 'default') {
|
|
1550
|
+
timers.set(label, Date.now());
|
|
1551
|
+
},
|
|
1552
|
+
timeEnd(label = 'default') {
|
|
1553
|
+
const startedAt = timers.get(label);
|
|
1554
|
+
timers.delete(label);
|
|
1555
|
+
capture('timeEnd')(
|
|
1556
|
+
`${label}: ${startedAt === undefined ? 0 : Date.now() - startedAt}ms`,
|
|
1557
|
+
);
|
|
1558
|
+
},
|
|
1559
|
+
timeLog(label = 'default', ...args: unknown[]) {
|
|
1560
|
+
const startedAt = timers.get(label);
|
|
1561
|
+
capture('timeLog')(
|
|
1562
|
+
`${label}: ${startedAt === undefined ? 0 : Date.now() - startedAt}ms`,
|
|
1563
|
+
...args,
|
|
1564
|
+
);
|
|
1565
|
+
},
|
|
1566
|
+
timeStamp() {},
|
|
1567
|
+
trace: capture('trace'),
|
|
1568
|
+
warn: capture('warn'),
|
|
1569
|
+
};
|
|
1570
|
+
|
|
1571
|
+
// Deliberately use no Console prototype. Any method inherited from Node's
|
|
1572
|
+
// process-global console can write to shared stdout/stderr, bypassing both
|
|
1573
|
+
// the run log and secret-taint validation.
|
|
1574
|
+
return Object.assign(Object.create(null), captured) as Console;
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
/**
|
|
1578
|
+
* Release attempt-scoped leases when a run settles. Terminal runs no longer own
|
|
1579
|
+
* rows, so their sheet fences must not delay an immediate repair run. A
|
|
1580
|
+
* suspension keeps its sheet fences because the same run will resume.
|
|
1581
|
+
*/
|
|
1582
|
+
async function releaseRuntimeLeasesOnAppRunnerSettlement(input: {
|
|
1583
|
+
config: PlayRunnerExecutionConfig;
|
|
1584
|
+
runtimeSheetAttempts: Map<
|
|
1585
|
+
string,
|
|
1586
|
+
{
|
|
1587
|
+
attemptId: string;
|
|
1588
|
+
attemptOwnerRunId: string;
|
|
1589
|
+
attemptExpiresAt: string;
|
|
1590
|
+
attemptSeq: number;
|
|
1591
|
+
writeVersion?: number;
|
|
1592
|
+
}
|
|
1593
|
+
>;
|
|
1594
|
+
runnerRunAttempt: number;
|
|
1595
|
+
preloadedDbSessions: PlayRunnerExecutionConfig['context']['preloadedDbSessions'];
|
|
1596
|
+
postgresSessionUnwrapKey: string | null;
|
|
1597
|
+
emitWorkerLog: (message: string) => void;
|
|
1598
|
+
reason: 'completed' | 'suspension' | 'run-fatal';
|
|
1599
|
+
}): Promise<void> {
|
|
1600
|
+
const { config } = input;
|
|
1601
|
+
const reasonLabel =
|
|
1602
|
+
input.reason === 'completed'
|
|
1603
|
+
? 'completed settlement'
|
|
1604
|
+
: `${input.reason} teardown`;
|
|
1605
|
+
const orgId = config.context.orgId;
|
|
1606
|
+
const playName = config.context.playName;
|
|
1607
|
+
const runId =
|
|
1608
|
+
config.context.runId ?? config.context.workflowId ?? 'anonymous-run';
|
|
1609
|
+
if (!orgId || !playName) return;
|
|
1610
|
+
const runtimeContext = {
|
|
1611
|
+
executorToken: config.context.executorToken,
|
|
1612
|
+
baseUrl: config.context.baseUrl,
|
|
1613
|
+
...playRunnerInvocationFenceFlags(config.context),
|
|
1614
|
+
runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
|
|
1615
|
+
vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
|
|
1616
|
+
orgId,
|
|
1617
|
+
playName,
|
|
1618
|
+
runId,
|
|
1619
|
+
userEmail: config.context.userEmail,
|
|
1620
|
+
dbSessionStrategy: config.context.dbSessionStrategy,
|
|
1621
|
+
preloadedDbSessions: input.preloadedDbSessions,
|
|
1622
|
+
postgresSessionUnwrapKey: input.postgresSessionUnwrapKey ?? undefined,
|
|
1623
|
+
};
|
|
1624
|
+
// Runtime Sheet rows use schedule-time write versions. They have no lease
|
|
1625
|
+
// to heartbeat or release; late writers are rejected by the write fence.
|
|
1626
|
+
void input.runtimeSheetAttempts;
|
|
1627
|
+
try {
|
|
1628
|
+
const released = await releaseRuntimeWorkReceipts(runtimeContext, {
|
|
1629
|
+
playName,
|
|
1630
|
+
runId,
|
|
1631
|
+
runAttempt: input.runnerRunAttempt,
|
|
1632
|
+
});
|
|
1633
|
+
if (released.released > 0) {
|
|
1634
|
+
input.emitWorkerLog(
|
|
1635
|
+
`[info] released ${released.released} completed-work lease(s) on ${reasonLabel}`,
|
|
1636
|
+
);
|
|
1637
|
+
}
|
|
1638
|
+
} catch (releaseError) {
|
|
1639
|
+
input.emitWorkerLog(
|
|
1640
|
+
`[warn] completed-work lease release failed on ${reasonLabel} (rerun falls back to lease-TTL expiry): ${
|
|
1641
|
+
releaseError instanceof Error
|
|
1642
|
+
? releaseError.message
|
|
1643
|
+
: String(releaseError)
|
|
1644
|
+
}`,
|
|
1645
|
+
);
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
async function run(
|
|
1650
|
+
config: PlayRunnerExecutionConfig,
|
|
1651
|
+
options?: ExecutePlayRunnerOptions,
|
|
1652
|
+
): Promise<PlayRunnerResult> {
|
|
1653
|
+
const runStartedAt = Date.now();
|
|
1654
|
+
const persistenceSignal =
|
|
1655
|
+
options?.signal ??
|
|
1656
|
+
AbortSignal.timeout(
|
|
1657
|
+
validatePlaySandboxRuntimeLimits(
|
|
1658
|
+
config.context.sandboxRuntimeLimits ??
|
|
1659
|
+
STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
1660
|
+
).timeoutSeconds * 1_000,
|
|
1661
|
+
);
|
|
1662
|
+
const contextSetupStartedAt = Date.now();
|
|
1663
|
+
const getRuntimeToolMetadata = createRuntimeToolMetadataResolver(config);
|
|
1664
|
+
const getRuntimeToolAuthScopeDigest =
|
|
1665
|
+
createAppRuntimeToolAuthScopeDigestResolver(config);
|
|
1666
|
+
let totalRows = 0;
|
|
1667
|
+
let inserted = 0;
|
|
1668
|
+
let skipped = 0;
|
|
1669
|
+
const runtimePreloadedDbSessions = config.context.preloadedDbSessions;
|
|
1670
|
+
const runtimePostgresSessionUnwrapKey =
|
|
1671
|
+
// Preloaded sessions are sealed with the immutable submit-time token, which
|
|
1672
|
+
// may differ from the per-attempt `executorToken` durable schedulers
|
|
1673
|
+
// re-mint. Prefer the pinned unwrap key; fall back for legacy launches.
|
|
1674
|
+
config.context.postgresSessionUnwrapKey ??
|
|
1675
|
+
config.context.executorToken ??
|
|
1676
|
+
process.env.DEEPLINE_INTERNAL_TOKEN ??
|
|
1677
|
+
null;
|
|
1678
|
+
const runtimeCsvDatasets = new Map<
|
|
1679
|
+
string,
|
|
1680
|
+
{
|
|
1681
|
+
tableNamespace: string;
|
|
1682
|
+
dataset: PlayDataset<Record<string, unknown>>;
|
|
1683
|
+
sheetContract: PlaySheetContract;
|
|
1684
|
+
sourceLabel: string;
|
|
1685
|
+
}
|
|
1686
|
+
>();
|
|
1687
|
+
const appRuntimeApiContext = {
|
|
1688
|
+
baseUrl: config.context.baseUrl ?? '',
|
|
1689
|
+
executorToken: config.context.executorToken ?? '',
|
|
1690
|
+
vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
|
|
1691
|
+
runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
|
|
1692
|
+
};
|
|
1693
|
+
const receiptGatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
|
|
1694
|
+
// Receipt state has exactly two backends: handlers provided by the host
|
|
1695
|
+
// process (inline executor) or the co-located receipt gateway named by the
|
|
1696
|
+
// launch context. There is deliberately no fallback onto the app runtime
|
|
1697
|
+
// API and no env-var switch here.
|
|
1698
|
+
const runtimeReceiptApiContext = receiptGatewayBaseUrl
|
|
1699
|
+
? {
|
|
1700
|
+
...appRuntimeApiContext,
|
|
1701
|
+
baseUrl: receiptGatewayBaseUrl,
|
|
1702
|
+
boundary: 'receipt_gateway' as const,
|
|
1703
|
+
requestTimeoutMs: RUNTIME_RECEIPT_GATEWAY_REQUEST_TIMEOUT_MS,
|
|
1704
|
+
}
|
|
1705
|
+
: null;
|
|
1706
|
+
const canUseRuntimeStepReceipts = playRunnerRuntimeReceiptsEnabled(
|
|
1707
|
+
config.context,
|
|
1708
|
+
);
|
|
1709
|
+
const receiptStoreAdapter =
|
|
1710
|
+
runtimeReceiptApiContext &&
|
|
1711
|
+
canUseRuntimeStepReceipts &&
|
|
1712
|
+
config.context.playName &&
|
|
1713
|
+
config.context.orgId
|
|
1714
|
+
? new RuntimeReceiptStoreAdapter({
|
|
1715
|
+
context: runtimeReceiptApiContext,
|
|
1716
|
+
playName: config.context.playName,
|
|
1717
|
+
runId:
|
|
1718
|
+
config.context.runId ??
|
|
1719
|
+
config.context.workflowId ??
|
|
1720
|
+
'anonymous-run',
|
|
1721
|
+
maxBatchSize: RUNTIME_RECEIPT_GATEWAY_BATCH_SIZE,
|
|
1722
|
+
maxFlushMs: RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_WAIT_MS,
|
|
1723
|
+
onRetryTelemetry: (line) => emitSystemLog(line, options),
|
|
1724
|
+
})
|
|
1725
|
+
: null;
|
|
1726
|
+
const receiptGatewayHandlers: RuntimeStepReceiptHandlers | null =
|
|
1727
|
+
receiptStoreAdapter?.handlers ?? null;
|
|
1728
|
+
if (canUseRuntimeStepReceipts && !receiptGatewayHandlers) {
|
|
1729
|
+
throw new Error(
|
|
1730
|
+
'Play runner has no receipt backend: the launch context must carry ' +
|
|
1731
|
+
'receiptGatewayBaseUrl. Runtime receipts have exactly one path — ' +
|
|
1732
|
+
'the receipt gateway — never the app runtime API.',
|
|
1733
|
+
);
|
|
1734
|
+
}
|
|
1735
|
+
const receiptHandlers = receiptGatewayHandlers;
|
|
1736
|
+
emitWorkerLog(
|
|
1737
|
+
`[perf] runtime state=${
|
|
1738
|
+
receiptGatewayHandlers ? 'durable' : 'disabled'
|
|
1739
|
+
} db_session_strategy=${config.context.dbSessionStrategy ?? 'unset'} preloaded_db_sessions=${runtimePreloadedDbSessions?.length ?? 0}`,
|
|
1740
|
+
options,
|
|
1741
|
+
);
|
|
1742
|
+
const runtimeSheetAttempts = new Map<
|
|
1743
|
+
string,
|
|
1744
|
+
{
|
|
1745
|
+
attemptId: string;
|
|
1746
|
+
attemptOwnerRunId: string;
|
|
1747
|
+
attemptExpiresAt: string;
|
|
1748
|
+
attemptSeq: number;
|
|
1749
|
+
writeVersion?: number;
|
|
1750
|
+
}
|
|
1751
|
+
>();
|
|
1752
|
+
const runtimeMapDatasetSettlements = new Map<
|
|
1753
|
+
string,
|
|
1754
|
+
{
|
|
1755
|
+
datasetId: string;
|
|
1756
|
+
path: string;
|
|
1757
|
+
tableNamespace: string;
|
|
1758
|
+
runId: string;
|
|
1759
|
+
persistedRows: number;
|
|
1760
|
+
succeededRows: number;
|
|
1761
|
+
failedRows: number;
|
|
1762
|
+
}
|
|
1763
|
+
>();
|
|
1764
|
+
const runtimeSheetAttemptKey = (input: {
|
|
1765
|
+
playName: string;
|
|
1766
|
+
runId: string;
|
|
1767
|
+
tableNamespace: string;
|
|
1768
|
+
}) => `${input.playName}\u0000${input.runId}\u0000${input.tableNamespace}`;
|
|
1769
|
+
const runnerRunAttempt =
|
|
1770
|
+
typeof config.context.runAttempt === 'number' &&
|
|
1771
|
+
Number.isFinite(config.context.runAttempt)
|
|
1772
|
+
? Math.max(0, Math.floor(config.context.runAttempt))
|
|
1773
|
+
: 0;
|
|
1774
|
+
const cachePolicy = playRunnerContextCachePolicy(config.context);
|
|
1775
|
+
const rateStateBackend = createRateStateBackendForConfig(config);
|
|
1776
|
+
const runtimeInput = config.input ?? {};
|
|
1777
|
+
const runtimeInputObject: Record<string, unknown> =
|
|
1778
|
+
playRunInputObjectView(runtimeInput);
|
|
1779
|
+
const resolveChildPlay =
|
|
1780
|
+
createRunScopedChildPlayResolver<ResolvedPlayExecution>({
|
|
1781
|
+
resolvePlay: async (playRef) => {
|
|
1782
|
+
if (!config.context.orgId) {
|
|
1783
|
+
return null;
|
|
1784
|
+
}
|
|
1785
|
+
// Child plays always resolve through the server's CJS-only path.
|
|
1786
|
+
try {
|
|
1787
|
+
const resolved = await resolveRuntimeReferencedPlay(
|
|
1788
|
+
{
|
|
1789
|
+
executorToken: config.context.executorToken,
|
|
1790
|
+
baseUrl: config.context.baseUrl,
|
|
1791
|
+
vercelProtectionBypassToken:
|
|
1792
|
+
config.context.vercelProtectionBypassToken,
|
|
1793
|
+
playName: config.context.playName,
|
|
1794
|
+
userEmail: config.context.userEmail,
|
|
1795
|
+
},
|
|
1796
|
+
playRef,
|
|
1797
|
+
);
|
|
1798
|
+
if (resolved?.resolutionDiagnostics) {
|
|
1799
|
+
emitWorkerLog(
|
|
1800
|
+
`[runtime.resolve-play-contract] ${JSON.stringify({
|
|
1801
|
+
playRef,
|
|
1802
|
+
...resolved.resolutionDiagnostics,
|
|
1803
|
+
runnerPipeline: Boolean(resolved.staticPipeline),
|
|
1804
|
+
})}`,
|
|
1805
|
+
options,
|
|
1806
|
+
);
|
|
1807
|
+
}
|
|
1808
|
+
return resolved as ResolvedPlayExecution | null;
|
|
1809
|
+
} catch (error) {
|
|
1810
|
+
const message =
|
|
1811
|
+
error instanceof Error ? error.message : String(error);
|
|
1812
|
+
throw new Error(
|
|
1813
|
+
`ctx.runPlay failed to resolve child play "${playRef}" via runtime API baseUrl=${config.context.baseUrl ?? '<missing>'}: ${message}`,
|
|
1814
|
+
);
|
|
1815
|
+
}
|
|
1816
|
+
},
|
|
1817
|
+
});
|
|
1818
|
+
const maxConcurrentExternalCalls = resolveMaxConcurrentExternalCalls(
|
|
1819
|
+
config.context.maxConcurrentExternalCalls,
|
|
1820
|
+
);
|
|
1821
|
+
const maxConcurrentRows = resolveMaxConcurrentRows(
|
|
1822
|
+
config.context.maxConcurrentRows,
|
|
1823
|
+
);
|
|
1824
|
+
emitWorkerLog(
|
|
1825
|
+
`[runtime.resource-policy] max_concurrent_external_calls=${maxConcurrentExternalCalls} max_concurrent_rows=${maxConcurrentRows ?? 'map_default'}`,
|
|
1826
|
+
options,
|
|
1827
|
+
);
|
|
1828
|
+
const artifactCompatibility = normalizePlayContractCompatibility(
|
|
1829
|
+
config.artifact.compatibility,
|
|
1830
|
+
);
|
|
1831
|
+
const artifactRuntimeContract = resolvePlayArtifactRuntimeContract({
|
|
1832
|
+
authoringContractEdition: artifactCompatibility.authoringContractEdition,
|
|
1833
|
+
slackHitlTimeoutOutcome: artifactCompatibility.slackHitlTimeoutOutcome,
|
|
1834
|
+
toolResponseContract: config.context.toolResponseContract,
|
|
1835
|
+
toolResponseReceiptRevision: config.context.toolResponseReceiptRevision,
|
|
1836
|
+
});
|
|
1837
|
+
const ctx = createPlayContext({
|
|
1838
|
+
artifactRuntimeContract,
|
|
1839
|
+
executorToken: config.context.executorToken,
|
|
1840
|
+
baseUrl: config.context.baseUrl,
|
|
1841
|
+
// Runtime control stays on baseUrl (the receipt gateway); provider calls
|
|
1842
|
+
// alone use this separate relay origin.
|
|
1843
|
+
executionGatewayBaseUrl: config.context.executionGatewayBaseUrl,
|
|
1844
|
+
...playRunnerInvocationFenceFlags(config.context),
|
|
1845
|
+
runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
|
|
1846
|
+
integrationMode: config.context.integrationMode,
|
|
1847
|
+
docflowEnabled: config.context.docflowEnabled === true,
|
|
1848
|
+
fixtureBehavior: config.context.fixtureBehavior,
|
|
1849
|
+
maxConcurrentExternalCalls,
|
|
1850
|
+
maxConcurrentRows,
|
|
1851
|
+
verifyDurableInvocationFence: async (signal) => {
|
|
1852
|
+
const gatewayBaseUrl =
|
|
1853
|
+
config.context.executionGatewayBaseUrl?.trim().replace(/\/$/, '') ||
|
|
1854
|
+
config.context.receiptGatewayBaseUrl?.trim().replace(/\/$/, '') ||
|
|
1855
|
+
'';
|
|
1856
|
+
const hasExplicitFence =
|
|
1857
|
+
config.context.requestDurableInvocationFence === true;
|
|
1858
|
+
if (
|
|
1859
|
+
!gatewayBaseUrl ||
|
|
1860
|
+
(!hasExplicitFence &&
|
|
1861
|
+
config.context.baseUrl?.replace(/\/$/, '') !== gatewayBaseUrl)
|
|
1862
|
+
) {
|
|
1863
|
+
return false;
|
|
1864
|
+
}
|
|
1865
|
+
try {
|
|
1866
|
+
const timeoutSignal = AbortSignal.timeout(
|
|
1867
|
+
RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS,
|
|
1868
|
+
);
|
|
1869
|
+
const response = await safeNodeOutboundFetch(
|
|
1870
|
+
`${gatewayBaseUrl}/healthz`,
|
|
1871
|
+
{
|
|
1872
|
+
redirect: 'manual',
|
|
1873
|
+
signal: signal
|
|
1874
|
+
? AbortSignal.any([signal, timeoutSignal])
|
|
1875
|
+
: timeoutSignal,
|
|
1876
|
+
},
|
|
1877
|
+
);
|
|
1878
|
+
return (
|
|
1879
|
+
response.ok &&
|
|
1880
|
+
response.headers.get(DURABLE_INVOCATION_FENCE_ATTESTATION_HEADER) ===
|
|
1881
|
+
DURABLE_INVOCATION_FENCE_ATTESTATION_VALUE
|
|
1882
|
+
);
|
|
1883
|
+
} catch {
|
|
1884
|
+
return false;
|
|
1885
|
+
}
|
|
1886
|
+
},
|
|
1887
|
+
dbSessionStrategy: config.context.dbSessionStrategy,
|
|
1888
|
+
fetchImpl: safeNodeStreamingOutboundFetch,
|
|
1889
|
+
vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
|
|
1890
|
+
orgId: config.context.orgId,
|
|
1891
|
+
userEmail: config.context.userEmail,
|
|
1892
|
+
playName: config.context.playName,
|
|
1893
|
+
graphHash: config.artifact.graphHash,
|
|
1894
|
+
artifactHash: config.artifact.artifactHash,
|
|
1895
|
+
convexUrl: config.context.convexUrl,
|
|
1896
|
+
runtimeSchedulerSchema: config.context.runtimeSchedulerSchema ?? null,
|
|
1897
|
+
runtimeSheetBackedMapDatasets: true,
|
|
1898
|
+
runtimeReceiptClaimsEstablishExecutionFence:
|
|
1899
|
+
receiptStoreAdapter?.claimsEstablishExecutionFence === true,
|
|
1900
|
+
workflowId: config.context.workflowId,
|
|
1901
|
+
playId: config.context.playId ?? config.context.workflowId,
|
|
1902
|
+
runId: config.context.runId ?? config.context.workflowId,
|
|
1903
|
+
rootRunId:
|
|
1904
|
+
config.context.rootRunId ??
|
|
1905
|
+
config.context.runId ??
|
|
1906
|
+
config.context.workflowId,
|
|
1907
|
+
runAttempt: runnerRunAttempt,
|
|
1908
|
+
staticPipeline: config.context.staticPipeline,
|
|
1909
|
+
checkpoint: config.checkpoint ?? undefined,
|
|
1910
|
+
durableBoundaries: true,
|
|
1911
|
+
governance: config.context.governance ?? undefined,
|
|
1912
|
+
...rateStateBackend,
|
|
1913
|
+
...(cachePolicy ? { cachePolicy } : {}),
|
|
1914
|
+
...(receiptHandlers ?? {}),
|
|
1915
|
+
resolvePlay: resolveChildPlay,
|
|
1916
|
+
launchPlayAsync: async (request) =>
|
|
1917
|
+
await launchPlayAsyncViaAppRuntime(
|
|
1918
|
+
{
|
|
1919
|
+
...appRuntimeApiContext,
|
|
1920
|
+
requestTimeoutMs: RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS,
|
|
1921
|
+
},
|
|
1922
|
+
request,
|
|
1923
|
+
),
|
|
1924
|
+
onBatchComplete: (checkpoint) => {
|
|
1925
|
+
emitRunnerEvent(
|
|
1926
|
+
{
|
|
1927
|
+
type: 'checkpoint',
|
|
1928
|
+
checkpoint,
|
|
1929
|
+
},
|
|
1930
|
+
options,
|
|
1931
|
+
);
|
|
1932
|
+
},
|
|
1933
|
+
getToolQueueHints: async (toolId) => {
|
|
1934
|
+
const metadata = await getRuntimeToolMetadata(toolId);
|
|
1935
|
+
return Array.isArray(metadata.queueHints)
|
|
1936
|
+
? (metadata.queueHints as PlayQueueHint[])
|
|
1937
|
+
: [];
|
|
1938
|
+
},
|
|
1939
|
+
getToolProvider: async (toolId) => {
|
|
1940
|
+
const provider = (await getRuntimeToolMetadata(toolId)).provider;
|
|
1941
|
+
return typeof provider === 'string' && provider.trim()
|
|
1942
|
+
? provider.trim()
|
|
1943
|
+
: null;
|
|
1944
|
+
},
|
|
1945
|
+
getToolOperation: async (toolId) => {
|
|
1946
|
+
const operation = (await getRuntimeToolMetadata(toolId)).operation;
|
|
1947
|
+
return typeof operation === 'string' && operation.trim()
|
|
1948
|
+
? operation.trim()
|
|
1949
|
+
: null;
|
|
1950
|
+
},
|
|
1951
|
+
getToolRetryPolicy: async (toolId) => {
|
|
1952
|
+
const metadata = await getRuntimeToolMetadata(toolId);
|
|
1953
|
+
return {
|
|
1954
|
+
retrySafeTransientHttp: metadata.retry?.retrySafeTransientHttp === true,
|
|
1955
|
+
requiresExecutionFence: metadata.retry?.requiresExecutionFence === true,
|
|
1956
|
+
};
|
|
1957
|
+
},
|
|
1958
|
+
getToolActionCacheVersion: async (toolId) => {
|
|
1959
|
+
const metadata = await getRuntimeToolMetadata(toolId);
|
|
1960
|
+
const version = metadata.actionCacheVersion.trim();
|
|
1961
|
+
if (!version) {
|
|
1962
|
+
throw new Error(
|
|
1963
|
+
`Runtime metadata for ${toolId} is missing actionCacheVersion.`,
|
|
1964
|
+
);
|
|
1965
|
+
}
|
|
1966
|
+
return version;
|
|
1967
|
+
},
|
|
1968
|
+
getToolActionCacheReadVersions: async (toolId) => {
|
|
1969
|
+
const metadata = await getRuntimeToolMetadata(toolId);
|
|
1970
|
+
return [
|
|
1971
|
+
...new Set(
|
|
1972
|
+
(metadata.actionCacheReadVersions ?? [])
|
|
1973
|
+
.map((version) => String(version || '').trim())
|
|
1974
|
+
.filter(Boolean),
|
|
1975
|
+
),
|
|
1976
|
+
];
|
|
1977
|
+
},
|
|
1978
|
+
getToolAuthScopeDigest: getRuntimeToolAuthScopeDigest,
|
|
1979
|
+
getToolRateScope: getRuntimeToolAuthScopeDigest.rateScope,
|
|
1980
|
+
invalidateToolAuthScopeDigest: (toolId) =>
|
|
1981
|
+
getRuntimeToolAuthScopeDigest.invalidate?.(toolId),
|
|
1982
|
+
getToolTargetGetters: async (toolId, output) => {
|
|
1983
|
+
const metadata = await getRuntimeToolMetadata(toolId);
|
|
1984
|
+
return metadata.targetGetters?.[output] ?? [];
|
|
1985
|
+
},
|
|
1986
|
+
getToolResultMetadata: async (toolId) => {
|
|
1987
|
+
const metadata = await getRuntimeToolMetadata(toolId);
|
|
1988
|
+
return {
|
|
1989
|
+
toolId,
|
|
1990
|
+
extractors: metadata.extractors ?? {},
|
|
1991
|
+
targetGetters: metadata.targetGetters ?? {},
|
|
1992
|
+
listExtractorPaths: metadata.listExtractorPaths ?? [],
|
|
1993
|
+
listIdentityGetters: metadata.listIdentityGetters ?? {},
|
|
1994
|
+
};
|
|
1995
|
+
},
|
|
1996
|
+
getToolAsyncOperation: async (toolId) =>
|
|
1997
|
+
(await getRuntimeToolMetadata(toolId)).asyncOperation ?? null,
|
|
1998
|
+
getToolAsyncOperationPoll: async (toolId) =>
|
|
1999
|
+
(await getRuntimeToolMetadata(toolId)).asyncOperationPoll ?? null,
|
|
2000
|
+
getBatchOperationStrategy: (toolId) => getPlayRuntimeBatchStrategy(toolId),
|
|
2001
|
+
getIntegrationEventWaitHandler: async (toolId) => {
|
|
2002
|
+
if (isIntegrationEventWaitToolId(toolId)) {
|
|
2003
|
+
return createRuntimeEventWaitHandler(config, toolId);
|
|
2004
|
+
}
|
|
2005
|
+
const metadata = await getRuntimeToolMetadata(toolId);
|
|
2006
|
+
return metadata.eventWait
|
|
2007
|
+
? createRuntimeEventWaitHandler(config, toolId)
|
|
2008
|
+
: null;
|
|
2009
|
+
},
|
|
2010
|
+
onRowUpdate: (() => {
|
|
2011
|
+
let rowUpdateCount = 0;
|
|
2012
|
+
let lastEmitAt = 0;
|
|
2013
|
+
const ROW_UPDATE_EMIT_INTERVAL_MS = 200;
|
|
2014
|
+
const ROW_UPDATE_EMIT_SAMPLE_RATE = 100;
|
|
2015
|
+
const terminalCellStates = new Set([
|
|
2016
|
+
'completed',
|
|
2017
|
+
'failed',
|
|
2018
|
+
'cached',
|
|
2019
|
+
'missed',
|
|
2020
|
+
'skipped',
|
|
2021
|
+
]);
|
|
2022
|
+
return (update: PlayRowUpdate) => {
|
|
2023
|
+
rowUpdateCount++;
|
|
2024
|
+
const now = Date.now();
|
|
2025
|
+
const hasTerminalCellUpdate = Object.values(
|
|
2026
|
+
update.cellMetaPatch ?? {},
|
|
2027
|
+
).some((cell) => terminalCellStates.has(cell.status));
|
|
2028
|
+
const isTerminalUpdate =
|
|
2029
|
+
update.status === 'completed' ||
|
|
2030
|
+
update.status === 'failed' ||
|
|
2031
|
+
hasTerminalCellUpdate;
|
|
2032
|
+
if (
|
|
2033
|
+
isTerminalUpdate ||
|
|
2034
|
+
rowUpdateCount === 1 ||
|
|
2035
|
+
rowUpdateCount % ROW_UPDATE_EMIT_SAMPLE_RATE === 0 ||
|
|
2036
|
+
now - lastEmitAt >= ROW_UPDATE_EMIT_INTERVAL_MS
|
|
2037
|
+
) {
|
|
2038
|
+
lastEmitAt = now;
|
|
2039
|
+
emitRunnerEvent({ type: 'row_update', update }, options);
|
|
2040
|
+
}
|
|
2041
|
+
};
|
|
2042
|
+
})(),
|
|
2043
|
+
onExecutionEvent: (() => {
|
|
2044
|
+
// map.row.updated fires for every cell-state change of every row (~7x
|
|
2045
|
+
// per row in a multi-column map) with no terminal semantics — unsampled
|
|
2046
|
+
// it floods the event transport at 150k-row scale. Sample it like row
|
|
2047
|
+
// updates; every other event type passes through untouched.
|
|
2048
|
+
let mapRowUpdatedCount = 0;
|
|
2049
|
+
let lastMapRowUpdatedEmitAt = 0;
|
|
2050
|
+
const MAP_ROW_UPDATED_EMIT_INTERVAL_MS = 200;
|
|
2051
|
+
const MAP_ROW_UPDATED_EMIT_SAMPLE_RATE = 100;
|
|
2052
|
+
return (event: PlayExecutionEvent) => {
|
|
2053
|
+
if (event.type === 'map.row.updated') {
|
|
2054
|
+
mapRowUpdatedCount++;
|
|
2055
|
+
const now = Date.now();
|
|
2056
|
+
if (
|
|
2057
|
+
mapRowUpdatedCount !== 1 &&
|
|
2058
|
+
mapRowUpdatedCount % MAP_ROW_UPDATED_EMIT_SAMPLE_RATE !== 0 &&
|
|
2059
|
+
now - lastMapRowUpdatedEmitAt < MAP_ROW_UPDATED_EMIT_INTERVAL_MS
|
|
2060
|
+
) {
|
|
2061
|
+
return;
|
|
2062
|
+
}
|
|
2063
|
+
lastMapRowUpdatedEmitAt = now;
|
|
2064
|
+
}
|
|
2065
|
+
emitRunnerEvent(
|
|
2066
|
+
{
|
|
2067
|
+
type: 'execution_event',
|
|
2068
|
+
event,
|
|
2069
|
+
},
|
|
2070
|
+
options,
|
|
2071
|
+
);
|
|
2072
|
+
};
|
|
2073
|
+
})(),
|
|
2074
|
+
// Durable docflow observation sink (ADR 0016 rule 2). Present only when the
|
|
2075
|
+
// co-located receipt gateway is wired (same durable path as receipts) and
|
|
2076
|
+
// the play carries an org/play scope. The compiler injects the observation
|
|
2077
|
+
// wrapper ONLY into instrumented (diagrammed) plays, so an undiagrammed play
|
|
2078
|
+
// never invokes this sink — the gating is at the wrapper. Fire-and-forget:
|
|
2079
|
+
// any transport failure is swallowed-and-logged so the play body is never
|
|
2080
|
+
// affected. The live docflow.node.* event stream is unchanged.
|
|
2081
|
+
onDocflowObservation:
|
|
2082
|
+
runtimeReceiptApiContext && config.context.playName
|
|
2083
|
+
? (observation) => {
|
|
2084
|
+
// Tracked so the graceful terminal path can drain in-flight posts
|
|
2085
|
+
// (bounded) before the terminal wakes the parked worker's finalize
|
|
2086
|
+
// read. Still strictly fire-and-forget from the play body.
|
|
2087
|
+
const post = observeDocflowNodeViaAppRuntime(
|
|
2088
|
+
runtimeReceiptApiContext,
|
|
2089
|
+
{
|
|
2090
|
+
playName: config.context.playName!,
|
|
2091
|
+
runId:
|
|
2092
|
+
observation.runId ||
|
|
2093
|
+
config.context.runId ||
|
|
2094
|
+
config.context.workflowId ||
|
|
2095
|
+
'anonymous-run',
|
|
2096
|
+
nodeId: observation.nodeId,
|
|
2097
|
+
attempt: observation.attempt,
|
|
2098
|
+
invocationSeq: observation.invocationSeq,
|
|
2099
|
+
invocationId: observation.invocationId ?? null,
|
|
2100
|
+
status: observation.status,
|
|
2101
|
+
inputs: observation.inputs ?? null,
|
|
2102
|
+
outputs: observation.outputs ?? null,
|
|
2103
|
+
inputsTruncated: observation.inputsTruncated === true,
|
|
2104
|
+
outputsTruncated: observation.outputsTruncated === true,
|
|
2105
|
+
error: observation.error ?? null,
|
|
2106
|
+
at: observation.at,
|
|
2107
|
+
},
|
|
2108
|
+
).catch((error) => {
|
|
2109
|
+
emitWorkerLog(
|
|
2110
|
+
`[error] docflow observation write failed node=${observation.nodeId} status=${observation.status}: ${
|
|
2111
|
+
error instanceof Error ? error.message : String(error)
|
|
2112
|
+
}`,
|
|
2113
|
+
options,
|
|
2114
|
+
);
|
|
2115
|
+
});
|
|
2116
|
+
pendingDocflowObservationPosts.add(post);
|
|
2117
|
+
void post.finally(() => {
|
|
2118
|
+
pendingDocflowObservationPosts.delete(post);
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
: undefined,
|
|
2122
|
+
onLog: (line) => {
|
|
2123
|
+
const timestampMatch = line.match(/^\[([^\]]+)\]/);
|
|
2124
|
+
const timestamp = timestampMatch?.[1];
|
|
2125
|
+
const at =
|
|
2126
|
+
timestamp && !Number.isNaN(new Date(timestamp).getTime())
|
|
2127
|
+
? new Date(timestamp).toISOString()
|
|
2128
|
+
: new Date().toISOString();
|
|
2129
|
+
emitRunnerEvent(
|
|
2130
|
+
{
|
|
2131
|
+
type: 'log',
|
|
2132
|
+
at,
|
|
2133
|
+
source: 'play',
|
|
2134
|
+
line,
|
|
2135
|
+
},
|
|
2136
|
+
options,
|
|
2137
|
+
);
|
|
2138
|
+
},
|
|
2139
|
+
onMapStart: async (items, tableNamespace, mapContext) => {
|
|
2140
|
+
assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
|
|
2141
|
+
const { baseUrl, orgId } = config.context;
|
|
2142
|
+
const startedAt = Date.now();
|
|
2143
|
+
totalRows += items.length;
|
|
2144
|
+
const userEmail = config.context.userEmail;
|
|
2145
|
+
const playName = mapContext.playName ?? config.context.playName;
|
|
2146
|
+
const runId =
|
|
2147
|
+
mapContext.runId ??
|
|
2148
|
+
config.context.runId ??
|
|
2149
|
+
config.context.workflowId ??
|
|
2150
|
+
'anonymous-run';
|
|
2151
|
+
const executorToken =
|
|
2152
|
+
mapContext.executorToken ?? config.context.executorToken;
|
|
2153
|
+
const rootRunId =
|
|
2154
|
+
config.context.runId ?? config.context.workflowId ?? 'anonymous-run';
|
|
2155
|
+
const canUseRootPreloadedDbSessions =
|
|
2156
|
+
executorToken === config.context.executorToken && runId === rootRunId;
|
|
2157
|
+
|
|
2158
|
+
const sheetContract = resolveSheetContractForTableNamespace(
|
|
2159
|
+
mapContext.staticPipeline ?? config.context.staticPipeline,
|
|
2160
|
+
tableNamespace,
|
|
2161
|
+
);
|
|
2162
|
+
if (!sheetContract) {
|
|
2163
|
+
throw new Error(
|
|
2164
|
+
`Missing sheet contract for play "${playName}" and table namespace "${tableNamespace}".`,
|
|
2165
|
+
);
|
|
2166
|
+
}
|
|
2167
|
+
const mapStartResult = await retryRuntimeSheetPersistence({
|
|
2168
|
+
description: `runtime map start table=${tableNamespace}`,
|
|
2169
|
+
log: (line) => emitWorkerLog(line, options),
|
|
2170
|
+
signal: persistenceSignal,
|
|
2171
|
+
run: () =>
|
|
2172
|
+
startRuntimeSheetDataset(
|
|
2173
|
+
{
|
|
2174
|
+
executorToken,
|
|
2175
|
+
baseUrl,
|
|
2176
|
+
vercelProtectionBypassToken:
|
|
2177
|
+
config.context.vercelProtectionBypassToken,
|
|
2178
|
+
orgId,
|
|
2179
|
+
playName,
|
|
2180
|
+
runId,
|
|
2181
|
+
userEmail,
|
|
2182
|
+
dbSessionStrategy: config.context.dbSessionStrategy,
|
|
2183
|
+
preloadedDbSessions: canUseRootPreloadedDbSessions
|
|
2184
|
+
? runtimePreloadedDbSessions
|
|
2185
|
+
: undefined,
|
|
2186
|
+
postgresSessionUnwrapKey: canUseRootPreloadedDbSessions
|
|
2187
|
+
? runtimePostgresSessionUnwrapKey
|
|
2188
|
+
: executorToken,
|
|
2189
|
+
abortSignal: persistenceSignal,
|
|
2190
|
+
},
|
|
2191
|
+
{
|
|
2192
|
+
playName,
|
|
2193
|
+
tableNamespace,
|
|
2194
|
+
playInput: runtimeInputObject,
|
|
2195
|
+
sheetContract,
|
|
2196
|
+
rows: items,
|
|
2197
|
+
inputOffset: mapContext.inputOffset,
|
|
2198
|
+
mode: mapContext.mode,
|
|
2199
|
+
runId,
|
|
2200
|
+
attemptId: deterministicRuntimeSheetAttemptId({
|
|
2201
|
+
playName,
|
|
2202
|
+
runId,
|
|
2203
|
+
tableNamespace,
|
|
2204
|
+
attemptSeq: runnerRunAttempt,
|
|
2205
|
+
}),
|
|
2206
|
+
attemptOwnerRunId: runId,
|
|
2207
|
+
attemptSeq: runnerRunAttempt,
|
|
2208
|
+
...(playRunnerContextRuntimeSheetForce(config.context)
|
|
2209
|
+
? { force: true }
|
|
2210
|
+
: {}),
|
|
2211
|
+
},
|
|
2212
|
+
),
|
|
2213
|
+
});
|
|
2214
|
+
const datasetId = createRuntimeDatasetId(playName, tableNamespace);
|
|
2215
|
+
const datasetPath = `datasets.${tableNamespace}`;
|
|
2216
|
+
const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
|
|
2217
|
+
datasetId,
|
|
2218
|
+
path: datasetPath,
|
|
2219
|
+
tableNamespace,
|
|
2220
|
+
runId,
|
|
2221
|
+
persistedRows: 0,
|
|
2222
|
+
succeededRows: 0,
|
|
2223
|
+
failedRows: 0,
|
|
2224
|
+
};
|
|
2225
|
+
// Reused rows are already terminal for this run. No completion callback
|
|
2226
|
+
// follows for them, so include them in the eventual terminal settlement.
|
|
2227
|
+
const alreadyCompletedRows = mapStartResult.completedRows.length;
|
|
2228
|
+
settlement.persistedRows += alreadyCompletedRows;
|
|
2229
|
+
settlement.succeededRows += alreadyCompletedRows;
|
|
2230
|
+
runtimeMapDatasetSettlements.set(datasetId, settlement);
|
|
2231
|
+
emitRunnerEvent(
|
|
2232
|
+
{
|
|
2233
|
+
type: 'execution_event',
|
|
2234
|
+
event: {
|
|
2235
|
+
type: 'dataset.lifecycle',
|
|
2236
|
+
datasetId,
|
|
2237
|
+
path: datasetPath,
|
|
2238
|
+
tableNamespace,
|
|
2239
|
+
phase: 'registered',
|
|
2240
|
+
persistedRows: mapStartResult.completedRows.length,
|
|
2241
|
+
succeededRows: mapStartResult.completedRows.length,
|
|
2242
|
+
failedRows: 0,
|
|
2243
|
+
complete: false,
|
|
2244
|
+
// ADR 0019 dataset-grain row birth. These rows entered from
|
|
2245
|
+
// play-body code, so the only witnessed fact is how many arrived at
|
|
2246
|
+
// this table — never a per-row source mapping, which the runtime
|
|
2247
|
+
// does not see here. `rowCountIn` counts rows admitted through this
|
|
2248
|
+
// boundary so far, so a paged dataset converges upward instead of
|
|
2249
|
+
// reporting only its last page.
|
|
2250
|
+
bornFrom: {
|
|
2251
|
+
table: tableNamespace,
|
|
2252
|
+
rowCountIn: (mapContext.inputOffset ?? 0) + items.length,
|
|
2253
|
+
},
|
|
2254
|
+
at: Date.now(),
|
|
2255
|
+
},
|
|
2256
|
+
},
|
|
2257
|
+
options,
|
|
2258
|
+
);
|
|
2259
|
+
emitRunnerEvent(
|
|
2260
|
+
{
|
|
2261
|
+
type: 'log',
|
|
2262
|
+
at: new Date().toISOString(),
|
|
2263
|
+
source: 'play',
|
|
2264
|
+
line: formatDatasetRegistrationLog({
|
|
2265
|
+
dataset: { path: datasetPath, tableNamespace },
|
|
2266
|
+
rows: items.length,
|
|
2267
|
+
}),
|
|
2268
|
+
},
|
|
2269
|
+
options,
|
|
2270
|
+
);
|
|
2271
|
+
if (
|
|
2272
|
+
mapStartResult.attemptId &&
|
|
2273
|
+
mapStartResult.attemptOwnerRunId &&
|
|
2274
|
+
mapStartResult.attemptExpiresAt
|
|
2275
|
+
) {
|
|
2276
|
+
runtimeSheetAttempts.set(
|
|
2277
|
+
runtimeSheetAttemptKey({ playName, runId, tableNamespace }),
|
|
2278
|
+
{
|
|
2279
|
+
attemptId: mapStartResult.attemptId,
|
|
2280
|
+
attemptOwnerRunId: mapStartResult.attemptOwnerRunId,
|
|
2281
|
+
attemptExpiresAt: mapStartResult.attemptExpiresAt,
|
|
2282
|
+
attemptSeq: mapStartResult.attemptSeq ?? runnerRunAttempt,
|
|
2283
|
+
writeVersion: mapStartResult.writeVersion,
|
|
2284
|
+
},
|
|
2285
|
+
);
|
|
2286
|
+
}
|
|
2287
|
+
emitWorkerLog(
|
|
2288
|
+
`[perf] runtime map setup table=${tableNamespace} rows=${items.length} elapsed_ms=${Date.now() - startedAt}`,
|
|
2289
|
+
options,
|
|
2290
|
+
);
|
|
2291
|
+
inserted += mapStartResult.inserted;
|
|
2292
|
+
skipped += mapStartResult.skipped;
|
|
2293
|
+
return mapStartResult;
|
|
2294
|
+
},
|
|
2295
|
+
onMapRowsCheckpoint: async (input) => {
|
|
2296
|
+
if (input.updates.length === 0) return;
|
|
2297
|
+
assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
|
|
2298
|
+
const playName = input.playName ?? config.context.playName;
|
|
2299
|
+
const userEmail = config.context.userEmail;
|
|
2300
|
+
const { runId, tableNamespace, updates } = input;
|
|
2301
|
+
const baseUrl = config.context.baseUrl;
|
|
2302
|
+
const executorToken = input.executorToken ?? config.context.executorToken;
|
|
2303
|
+
if (!baseUrl || !executorToken) {
|
|
2304
|
+
throw new Error(
|
|
2305
|
+
`Runtime map checkpoint for "${playName}" requires the runtime API origin and executor token.`,
|
|
2306
|
+
);
|
|
2307
|
+
}
|
|
2308
|
+
const sheetContract = resolveSheetContractForTableNamespace(
|
|
2309
|
+
input.staticPipeline ?? config.context.staticPipeline,
|
|
2310
|
+
tableNamespace,
|
|
2311
|
+
);
|
|
2312
|
+
if (!sheetContract) {
|
|
2313
|
+
throw new Error(
|
|
2314
|
+
`Missing sheet contract for play "${playName}" and table namespace "${tableNamespace}" while checkpointing partial rows.`,
|
|
2315
|
+
);
|
|
2316
|
+
}
|
|
2317
|
+
const attempt = runtimeSheetAttempts.get(
|
|
2318
|
+
runtimeSheetAttemptKey({
|
|
2319
|
+
playName,
|
|
2320
|
+
runId,
|
|
2321
|
+
tableNamespace,
|
|
2322
|
+
}),
|
|
2323
|
+
);
|
|
2324
|
+
const checkpointUpdates = updates.map((update) => ({
|
|
2325
|
+
...update,
|
|
2326
|
+
attemptId: attempt?.attemptId ?? null,
|
|
2327
|
+
attemptOwnerRunId: attempt?.attemptOwnerRunId ?? null,
|
|
2328
|
+
attemptExpiresAt: attempt?.attemptExpiresAt ?? null,
|
|
2329
|
+
attemptSeq: attempt?.attemptSeq ?? runnerRunAttempt,
|
|
2330
|
+
writeVersion: attempt?.writeVersion ?? null,
|
|
2331
|
+
}));
|
|
2332
|
+
await retryRuntimeSheetPersistence({
|
|
2333
|
+
description: `runtime map checkpoint table=${tableNamespace}`,
|
|
2334
|
+
log: (line) => emitWorkerLog(line, options),
|
|
2335
|
+
signal: persistenceSignal,
|
|
2336
|
+
run: () =>
|
|
2337
|
+
applyRowUpdatesViaAppRuntime(
|
|
2338
|
+
{
|
|
2339
|
+
baseUrl,
|
|
2340
|
+
executorToken,
|
|
2341
|
+
vercelProtectionBypassToken:
|
|
2342
|
+
config.context.vercelProtectionBypassToken,
|
|
2343
|
+
signal: persistenceSignal,
|
|
2344
|
+
},
|
|
2345
|
+
{
|
|
2346
|
+
playName,
|
|
2347
|
+
tableNamespace,
|
|
2348
|
+
sheetContract,
|
|
2349
|
+
contractSnapshot: {
|
|
2350
|
+
staticPipeline:
|
|
2351
|
+
input.staticPipeline ?? config.context.staticPipeline ?? null,
|
|
2352
|
+
},
|
|
2353
|
+
runId,
|
|
2354
|
+
userEmail,
|
|
2355
|
+
updates: checkpointUpdates,
|
|
2356
|
+
},
|
|
2357
|
+
),
|
|
2358
|
+
});
|
|
2359
|
+
},
|
|
2360
|
+
onMapRowsCompleted: async (input) => {
|
|
2361
|
+
// Persist executed rows (data + cell meta) into the tenant sheet —
|
|
2362
|
+
// the sheet is the source of truth for dataset state, and the
|
|
2363
|
+
// deterministic row keys make this write idempotent across retries
|
|
2364
|
+
// and reruns. An actor email is optional attribution, never durability
|
|
2365
|
+
// authority for a scheduler-backed Play run.
|
|
2366
|
+
assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
|
|
2367
|
+
const playName = input.playName ?? config.context.playName;
|
|
2368
|
+
const runId =
|
|
2369
|
+
input.runId ??
|
|
2370
|
+
config.context.runId ??
|
|
2371
|
+
config.context.workflowId ??
|
|
2372
|
+
'anonymous-run';
|
|
2373
|
+
const executorToken = input.executorToken ?? config.context.executorToken;
|
|
2374
|
+
const rootRunId =
|
|
2375
|
+
config.context.runId ?? config.context.workflowId ?? 'anonymous-run';
|
|
2376
|
+
const canUseRootPreloadedDbSessions =
|
|
2377
|
+
executorToken === config.context.executorToken && runId === rootRunId;
|
|
2378
|
+
const sheetContract = resolveSheetContractForTableNamespace(
|
|
2379
|
+
input.staticPipeline ?? config.context.staticPipeline,
|
|
2380
|
+
input.tableNamespace,
|
|
2381
|
+
);
|
|
2382
|
+
if (!sheetContract) {
|
|
2383
|
+
throw new Error(
|
|
2384
|
+
`Missing sheet contract for play "${playName}" and table namespace "${input.tableNamespace}" while persisting executed rows.`,
|
|
2385
|
+
);
|
|
2386
|
+
}
|
|
2387
|
+
const persistStartedAt = Date.now();
|
|
2388
|
+
const attempt = runtimeSheetAttempts.get(
|
|
2389
|
+
runtimeSheetAttemptKey({
|
|
2390
|
+
playName,
|
|
2391
|
+
runId,
|
|
2392
|
+
tableNamespace: input.tableNamespace,
|
|
2393
|
+
}),
|
|
2394
|
+
);
|
|
2395
|
+
const runtimeSheetContext = {
|
|
2396
|
+
executorToken,
|
|
2397
|
+
baseUrl: config.context.baseUrl,
|
|
2398
|
+
vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
|
|
2399
|
+
orgId: config.context.orgId,
|
|
2400
|
+
playName,
|
|
2401
|
+
runId,
|
|
2402
|
+
userEmail: config.context.userEmail,
|
|
2403
|
+
dbSessionStrategy: config.context.dbSessionStrategy,
|
|
2404
|
+
runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
|
|
2405
|
+
preloadedDbSessions: canUseRootPreloadedDbSessions
|
|
2406
|
+
? runtimePreloadedDbSessions
|
|
2407
|
+
: undefined,
|
|
2408
|
+
postgresSessionUnwrapKey: canUseRootPreloadedDbSessions
|
|
2409
|
+
? runtimePostgresSessionUnwrapKey
|
|
2410
|
+
: executorToken,
|
|
2411
|
+
abortSignal: persistenceSignal,
|
|
2412
|
+
};
|
|
2413
|
+
const pageTailWriteMarker = runtimeSheetPageTailWriteMarkerEvent(
|
|
2414
|
+
config.context.runtimeTestFaultHeader,
|
|
2415
|
+
input.rows,
|
|
2416
|
+
input.tableNamespace,
|
|
2417
|
+
config.context.dbSessionStrategy,
|
|
2418
|
+
);
|
|
2419
|
+
if (pageTailWriteMarker) {
|
|
2420
|
+
// Authorized synthetic fault only. Source "play" is intentional: the
|
|
2421
|
+
// direct-worker reporter drops host diagnostics, while this one marker
|
|
2422
|
+
// is the durable proof that the page-tail terminal write was admitted.
|
|
2423
|
+
emitRunnerEvent(pageTailWriteMarker, options);
|
|
2424
|
+
}
|
|
2425
|
+
emitWorkerLog(
|
|
2426
|
+
`[perf] runtime map persist:start table=${input.tableNamespace} rows=${input.rows.length}`,
|
|
2427
|
+
options,
|
|
2428
|
+
);
|
|
2429
|
+
const completeRows = createCompleteRuntimeMapRowsOperation(
|
|
2430
|
+
runtimeSheetContext,
|
|
2431
|
+
{
|
|
2432
|
+
tableNamespace: input.tableNamespace,
|
|
2433
|
+
sheetContract,
|
|
2434
|
+
rows: input.rows,
|
|
2435
|
+
outputFields: input.outputFields,
|
|
2436
|
+
runId,
|
|
2437
|
+
attemptId: attempt?.attemptId ?? null,
|
|
2438
|
+
attemptOwnerRunId: attempt?.attemptOwnerRunId ?? null,
|
|
2439
|
+
attemptExpiresAt: attempt?.attemptExpiresAt ?? null,
|
|
2440
|
+
attemptSeq: attempt?.attemptSeq ?? runnerRunAttempt,
|
|
2441
|
+
writeVersion: attempt?.writeVersion ?? null,
|
|
2442
|
+
},
|
|
2443
|
+
);
|
|
2444
|
+
const result = await retryRuntimeSheetPersistence({
|
|
2445
|
+
description: `runtime map persist table=${input.tableNamespace}`,
|
|
2446
|
+
log: (line) => emitWorkerLog(line, options),
|
|
2447
|
+
signal: persistenceSignal,
|
|
2448
|
+
run: completeRows,
|
|
2449
|
+
});
|
|
2450
|
+
const pageTailHoldCompleted = runtimeSheetPageTailHoldCompletedEvent(
|
|
2451
|
+
result.runtimeTestPageTailHoldMs,
|
|
2452
|
+
input.tableNamespace,
|
|
2453
|
+
);
|
|
2454
|
+
if (pageTailHoldCompleted) {
|
|
2455
|
+
emitRunnerEvent(pageTailHoldCompleted, options);
|
|
2456
|
+
}
|
|
2457
|
+
emitWorkerLog(
|
|
2458
|
+
`[perf] runtime map persist:write_done table=${input.tableNamespace} rows=${input.rows.length} updated=${result.updated} elapsed_ms=${Date.now() - persistStartedAt}`,
|
|
2459
|
+
options,
|
|
2460
|
+
);
|
|
2461
|
+
const staleKeys = new Set(result.staleKeys);
|
|
2462
|
+
const staleDropped = staleKeys.size;
|
|
2463
|
+
let recoveredStaleKeys: string[] = [];
|
|
2464
|
+
if (staleDropped > 0) {
|
|
2465
|
+
// A terminal retry can lose the SQL mutation race to its own earlier
|
|
2466
|
+
// commit. The Context must not turn that already-durable row into an
|
|
2467
|
+
// empty successful dataset. Verify ownership through the runner's
|
|
2468
|
+
// preloaded session (never by minting a new database authority), then
|
|
2469
|
+
// return only the same-run terminal keys as idempotent settlements.
|
|
2470
|
+
const recovered = await retryRuntimeSheetPersistence({
|
|
2471
|
+
description: `runtime map stale settlement reconcile table=${input.tableNamespace}`,
|
|
2472
|
+
log: (line) => emitWorkerLog(line, options),
|
|
2473
|
+
signal: persistenceSignal,
|
|
2474
|
+
run: () =>
|
|
2475
|
+
readRuntimeSheetDatasetRowKeys(runtimeSheetContext, {
|
|
2476
|
+
tableNamespace: input.tableNamespace,
|
|
2477
|
+
runId,
|
|
2478
|
+
keys: [...staleKeys],
|
|
2479
|
+
rowMode: 'all',
|
|
2480
|
+
}),
|
|
2481
|
+
});
|
|
2482
|
+
recoveredStaleKeys = recovered.keys;
|
|
2483
|
+
for (const key of recoveredStaleKeys) staleKeys.delete(key);
|
|
2484
|
+
}
|
|
2485
|
+
if (staleDropped > 0) {
|
|
2486
|
+
emitWorkerLog(
|
|
2487
|
+
`[info] dropped ${staleKeys.size} stale runtime sheet write(s) for table=${input.tableNamespace}; ${recoveredStaleKeys.length} same-run terminal write(s) were accepted idempotently`,
|
|
2488
|
+
options,
|
|
2489
|
+
);
|
|
2490
|
+
}
|
|
2491
|
+
const failedRows = input.rows.filter(
|
|
2492
|
+
(row) =>
|
|
2493
|
+
row.status === 'failed' &&
|
|
2494
|
+
!staleKeys.has(resolveMapRowOutcomeKey(row) ?? String(row.key ?? '')),
|
|
2495
|
+
).length;
|
|
2496
|
+
// The fenced commit response is the persistence barrier. An exact-version
|
|
2497
|
+
// replay may report zero physical updates while still being an accepted,
|
|
2498
|
+
// durable settlement, so derive accepted rows from submitted minus stale
|
|
2499
|
+
// keys instead of issuing a read-after-write visibility audit.
|
|
2500
|
+
const committedTerminalRows = result.committedKeys.length;
|
|
2501
|
+
const succeededRows = Math.max(0, committedTerminalRows - failedRows);
|
|
2502
|
+
const datasetPath = `datasets.${input.tableNamespace}`;
|
|
2503
|
+
const datasetId = createRuntimeDatasetId(playName, input.tableNamespace);
|
|
2504
|
+
const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
|
|
2505
|
+
datasetId,
|
|
2506
|
+
path: datasetPath,
|
|
2507
|
+
tableNamespace: input.tableNamespace,
|
|
2508
|
+
runId,
|
|
2509
|
+
persistedRows: 0,
|
|
2510
|
+
succeededRows: 0,
|
|
2511
|
+
failedRows: 0,
|
|
2512
|
+
};
|
|
2513
|
+
// Only exact-version accepted rows belong to this run's lifecycle
|
|
2514
|
+
// settlement. Stale rows belong to the newer writer and stay excluded.
|
|
2515
|
+
settlement.persistedRows += committedTerminalRows;
|
|
2516
|
+
settlement.succeededRows += succeededRows;
|
|
2517
|
+
settlement.failedRows += failedRows;
|
|
2518
|
+
runtimeMapDatasetSettlements.set(datasetId, settlement);
|
|
2519
|
+
emitWorkerLog(
|
|
2520
|
+
`[perf] runtime map persist table=${input.tableNamespace} rows=${input.rows.length} updated=${result.updated} accepted=${committedTerminalRows} stale=${staleKeys.size} elapsed_ms=${Date.now() - persistStartedAt}`,
|
|
2521
|
+
options,
|
|
2522
|
+
);
|
|
2523
|
+
return {
|
|
2524
|
+
updated: result.updated,
|
|
2525
|
+
staleDroppedKeys: [...staleKeys],
|
|
2526
|
+
...(recoveredStaleKeys.length > 0 ? { recoveredStaleKeys } : {}),
|
|
2527
|
+
};
|
|
2528
|
+
},
|
|
2529
|
+
});
|
|
2530
|
+
|
|
2531
|
+
const csvSourceCache = new Map<string, ReplayableCsvSource>();
|
|
2532
|
+
const csvDatasetCache = new Map<
|
|
2533
|
+
string,
|
|
2534
|
+
import('../../plays/dataset').PlayDataset<Record<string, unknown>>
|
|
2535
|
+
>();
|
|
2536
|
+
const csvDatasetNamespaceByCacheKey = new Map<string, string>();
|
|
2537
|
+
const runtimeInputAlias = config.runtimeInputAlias ?? null;
|
|
2538
|
+
const csvSourcePath = config.csvSourcePath ?? null;
|
|
2539
|
+
const csvSourceUrl = config.csvSourceUrl ?? null;
|
|
2540
|
+
const csvSourceContentBase64 = config.csvSourceContentBase64 ?? null;
|
|
2541
|
+
|
|
2542
|
+
(
|
|
2543
|
+
ctx as unknown as {
|
|
2544
|
+
csv: (
|
|
2545
|
+
requestedPath: string,
|
|
2546
|
+
options?: CsvRenameOptions,
|
|
2547
|
+
) => Promise<
|
|
2548
|
+
import('../../plays/dataset').PlayDataset<Record<string, unknown>>
|
|
2549
|
+
>;
|
|
2550
|
+
}
|
|
2551
|
+
).csv = async (requestedPath: string, csvOptions?: CsvRenameOptions) => {
|
|
2552
|
+
if (csvOptions) {
|
|
2553
|
+
for (const [field, value] of [
|
|
2554
|
+
['ctx.csv.options.columns', csvOptions.columns],
|
|
2555
|
+
['ctx.csv.options.rename', csvOptions.rename],
|
|
2556
|
+
['ctx.csv.options.required', csvOptions.required],
|
|
2557
|
+
] as const) {
|
|
2558
|
+
validateOptionalPlayAuthoringField(field, value);
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
const resolvedRequestedPath =
|
|
2562
|
+
typeof requestedPath === 'string' && requestedPath.trim().length > 0
|
|
2563
|
+
? requestedPath
|
|
2564
|
+
: runtimeInputAlias;
|
|
2565
|
+
|
|
2566
|
+
if (!resolvedRequestedPath) {
|
|
2567
|
+
throw new Error(
|
|
2568
|
+
'ctx.csv(...) requires a path, or the play run must provide an input file.',
|
|
2569
|
+
);
|
|
2570
|
+
}
|
|
2571
|
+
|
|
2572
|
+
const shouldUseInlineCsvSource =
|
|
2573
|
+
Boolean(csvSourceContentBase64) &&
|
|
2574
|
+
(!runtimeInputAlias || resolvedRequestedPath === runtimeInputAlias);
|
|
2575
|
+
const shouldUseCsvSourceUrl =
|
|
2576
|
+
Boolean(csvSourceUrl) &&
|
|
2577
|
+
(!runtimeInputAlias || resolvedRequestedPath === runtimeInputAlias);
|
|
2578
|
+
const targetPath = shouldUseInlineCsvSource
|
|
2579
|
+
? `inline:${resolvedRequestedPath}`
|
|
2580
|
+
: shouldUseCsvSourceUrl
|
|
2581
|
+
? csvSourceUrl!
|
|
2582
|
+
: (config.materializedFiles[resolvedRequestedPath] ??
|
|
2583
|
+
config.materializedFiles[
|
|
2584
|
+
resolvedRequestedPath.replace(/^\.\//, '')
|
|
2585
|
+
] ??
|
|
2586
|
+
resolve(config.workspaceRoot, resolvedRequestedPath));
|
|
2587
|
+
|
|
2588
|
+
if (
|
|
2589
|
+
!shouldUseInlineCsvSource &&
|
|
2590
|
+
!shouldUseCsvSourceUrl &&
|
|
2591
|
+
!targetPath.startsWith(config.workspaceRoot)
|
|
2592
|
+
) {
|
|
2593
|
+
throw new Error(
|
|
2594
|
+
`ctx.csv("${resolvedRequestedPath}") resolved outside the packaged play workspace.`,
|
|
2595
|
+
);
|
|
2596
|
+
}
|
|
2597
|
+
|
|
2598
|
+
if (!csvSourceCache.has(targetPath)) {
|
|
2599
|
+
try {
|
|
2600
|
+
csvSourceCache.set(
|
|
2601
|
+
targetPath,
|
|
2602
|
+
createCsvSource(
|
|
2603
|
+
shouldUseInlineCsvSource
|
|
2604
|
+
? { contentBase64: csvSourceContentBase64! }
|
|
2605
|
+
: shouldUseCsvSourceUrl
|
|
2606
|
+
? { url: targetPath }
|
|
2607
|
+
: { path: targetPath },
|
|
2608
|
+
),
|
|
2609
|
+
);
|
|
2610
|
+
} catch {
|
|
2611
|
+
const knownFiles = Object.keys(config.materializedFiles)
|
|
2612
|
+
.filter((key) => !key.includes('/'))
|
|
2613
|
+
.sort()
|
|
2614
|
+
.join(', ');
|
|
2615
|
+
throw new Error(
|
|
2616
|
+
`ctx.csv("${resolvedRequestedPath}") could not be resolved in the packaged play workspace. ` +
|
|
2617
|
+
`Known packaged files: ${knownFiles || 'none'}.`,
|
|
2618
|
+
);
|
|
2619
|
+
}
|
|
2620
|
+
}
|
|
2621
|
+
|
|
2622
|
+
const csvProjectionIdentity = `${JSON.stringify(csvOptions?.columns ?? {})}:${JSON.stringify(csvOptions?.rename ?? {})}:${JSON.stringify(csvOptions?.required ?? [])}`;
|
|
2623
|
+
const datasetCacheKey = `${resolvedRequestedPath}:${targetPath}:${csvProjectionIdentity}`;
|
|
2624
|
+
let tableNamespace = csvDatasetNamespaceByCacheKey.get(datasetCacheKey);
|
|
2625
|
+
if (!tableNamespace) {
|
|
2626
|
+
const baseTableNamespace = `${CSV_NAMESPACE_PREFIX}_${createHash('sha1').update(resolvedRequestedPath).digest('hex').slice(0, 12)}`;
|
|
2627
|
+
const baseAlreadyAssigned = [
|
|
2628
|
+
...csvDatasetNamespaceByCacheKey.values(),
|
|
2629
|
+
].some((assigned) => assigned === baseTableNamespace);
|
|
2630
|
+
tableNamespace = baseAlreadyAssigned
|
|
2631
|
+
? `${baseTableNamespace}_${createHash('sha1').update(csvProjectionIdentity).digest('hex').slice(0, 12)}`
|
|
2632
|
+
: baseTableNamespace;
|
|
2633
|
+
csvDatasetNamespaceByCacheKey.set(datasetCacheKey, tableNamespace);
|
|
2634
|
+
}
|
|
2635
|
+
|
|
2636
|
+
assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
|
|
2637
|
+
|
|
2638
|
+
assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
|
|
2639
|
+
const cachedDataset = csvDatasetCache.get(datasetCacheKey);
|
|
2640
|
+
if (cachedDataset) {
|
|
2641
|
+
return cachedDataset;
|
|
2642
|
+
}
|
|
2643
|
+
|
|
2644
|
+
const playName = config.context.playName;
|
|
2645
|
+
const dataset = await createStreamingCsvDataset({
|
|
2646
|
+
playName,
|
|
2647
|
+
tableNamespace,
|
|
2648
|
+
source: csvSourceCache.get(targetPath)!,
|
|
2649
|
+
sourceLabel: resolvedRequestedPath,
|
|
2650
|
+
options: csvOptions,
|
|
2651
|
+
});
|
|
2652
|
+
const sheetContract = inferCsvSheetContract({
|
|
2653
|
+
tableNamespace,
|
|
2654
|
+
rows: await dataset.peek(10),
|
|
2655
|
+
});
|
|
2656
|
+
runtimeCsvDatasets.set(dataset.datasetId, {
|
|
2657
|
+
tableNamespace,
|
|
2658
|
+
dataset,
|
|
2659
|
+
sheetContract,
|
|
2660
|
+
sourceLabel: resolvedRequestedPath,
|
|
2661
|
+
});
|
|
2662
|
+
csvDatasetCache.set(datasetCacheKey, dataset);
|
|
2663
|
+
return dataset;
|
|
2664
|
+
};
|
|
2665
|
+
emitWorkerLog(
|
|
2666
|
+
`[perf] context setup elapsed_ms=${Date.now() - contextSetupStartedAt}`,
|
|
2667
|
+
options,
|
|
2668
|
+
);
|
|
2669
|
+
|
|
2670
|
+
const playInput: PlayRunInputPayload =
|
|
2671
|
+
runtimeInputAlias && !Array.isArray(runtimeInput)
|
|
2672
|
+
? {
|
|
2673
|
+
...runtimeInputObject,
|
|
2674
|
+
file: runtimeInputAlias,
|
|
2675
|
+
}
|
|
2676
|
+
: runtimeInput;
|
|
2677
|
+
|
|
2678
|
+
try {
|
|
2679
|
+
emitWorkerLog('Play started', options);
|
|
2680
|
+
const rawResult = await executeRuntimePlayCode({
|
|
2681
|
+
artifact: config.artifact,
|
|
2682
|
+
ctx,
|
|
2683
|
+
runtimeInput: playInput,
|
|
2684
|
+
options,
|
|
2685
|
+
});
|
|
2686
|
+
// A durable ctx boundary deliberately unwinds execution. Customer code can
|
|
2687
|
+
// catch that error, so the context keeps a sticky handoff that remains
|
|
2688
|
+
// authoritative until this invocation returns it to the scheduler.
|
|
2689
|
+
ctx.assertNoPendingDurableSuspension();
|
|
2690
|
+
emitWorkerLog(
|
|
2691
|
+
`[perf] play body returned elapsed_ms=${Date.now() - runStartedAt}`,
|
|
2692
|
+
options,
|
|
2693
|
+
);
|
|
2694
|
+
for (const settlement of runtimeMapDatasetSettlements.values()) {
|
|
2695
|
+
emitRunnerEvent(
|
|
2696
|
+
{
|
|
2697
|
+
type: 'execution_event',
|
|
2698
|
+
event: {
|
|
2699
|
+
type: 'dataset.lifecycle',
|
|
2700
|
+
datasetId: settlement.datasetId,
|
|
2701
|
+
path: settlement.path,
|
|
2702
|
+
tableNamespace: settlement.tableNamespace,
|
|
2703
|
+
phase: 'available',
|
|
2704
|
+
persistedRows: settlement.persistedRows,
|
|
2705
|
+
succeededRows: settlement.succeededRows,
|
|
2706
|
+
failedRows: settlement.failedRows,
|
|
2707
|
+
complete: true,
|
|
2708
|
+
at: Date.now(),
|
|
2709
|
+
},
|
|
2710
|
+
},
|
|
2711
|
+
options,
|
|
2712
|
+
);
|
|
2713
|
+
emitRunnerEvent(
|
|
2714
|
+
{
|
|
2715
|
+
type: 'log',
|
|
2716
|
+
at: new Date().toISOString(),
|
|
2717
|
+
source: 'play',
|
|
2718
|
+
line: formatDatasetAvailabilityLog({
|
|
2719
|
+
runId: settlement.runId,
|
|
2720
|
+
dataset: {
|
|
2721
|
+
path: settlement.path,
|
|
2722
|
+
tableNamespace: settlement.tableNamespace,
|
|
2723
|
+
},
|
|
2724
|
+
persistedRows: settlement.persistedRows,
|
|
2725
|
+
}),
|
|
2726
|
+
},
|
|
2727
|
+
options,
|
|
2728
|
+
);
|
|
2729
|
+
}
|
|
2730
|
+
await receiptStoreAdapter?.close();
|
|
2731
|
+
const result = normalizeTopLevelPlayReturn(rawResult);
|
|
2732
|
+
for (const returnedDataset of collectReturnedDatasets(result)) {
|
|
2733
|
+
if (!isPlayDataset<Record<string, unknown>>(returnedDataset)) {
|
|
2734
|
+
continue;
|
|
2735
|
+
}
|
|
2736
|
+
const csvDataset = runtimeCsvDatasets.get(returnedDataset.datasetId);
|
|
2737
|
+
if (!csvDataset) {
|
|
2738
|
+
continue;
|
|
2739
|
+
}
|
|
2740
|
+
const persistStartedAt = Date.now();
|
|
2741
|
+
const csvRunId =
|
|
2742
|
+
config.context.runId ?? config.context.workflowId ?? 'anonymous-run';
|
|
2743
|
+
const csvRuntimeContext = {
|
|
2744
|
+
executorToken: config.context.executorToken,
|
|
2745
|
+
baseUrl: config.context.baseUrl,
|
|
2746
|
+
vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
|
|
2747
|
+
orgId: config.context.orgId,
|
|
2748
|
+
playName: config.context.playName!,
|
|
2749
|
+
runId: csvRunId,
|
|
2750
|
+
userEmail: config.context.userEmail,
|
|
2751
|
+
dbSessionStrategy: config.context.dbSessionStrategy,
|
|
2752
|
+
preloadedDbSessions: runtimePreloadedDbSessions,
|
|
2753
|
+
postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
|
|
2754
|
+
abortSignal: persistenceSignal,
|
|
2755
|
+
};
|
|
2756
|
+
let csvRowsPersisted = 0;
|
|
2757
|
+
for await (const page of iteratePlayDatasetInputPages(
|
|
2758
|
+
csvDataset.dataset,
|
|
2759
|
+
)) {
|
|
2760
|
+
await retryRuntimeSheetPersistence({
|
|
2761
|
+
description: `runtime csv persist table=${csvDataset.tableNamespace}`,
|
|
2762
|
+
log: (line) => emitWorkerLog(line, options),
|
|
2763
|
+
signal: persistenceSignal,
|
|
2764
|
+
run: () =>
|
|
2765
|
+
startRuntimeSheetDataset(csvRuntimeContext, {
|
|
2766
|
+
playName: config.context.playName!,
|
|
2767
|
+
tableNamespace: csvDataset.tableNamespace,
|
|
2768
|
+
playInput: runtimeInputObject,
|
|
2769
|
+
sheetContract: csvDataset.sheetContract,
|
|
2770
|
+
rows: page.rows,
|
|
2771
|
+
runId: csvRunId,
|
|
2772
|
+
inputOffset: page.offset,
|
|
2773
|
+
}),
|
|
2774
|
+
});
|
|
2775
|
+
csvRowsPersisted += page.rows.length;
|
|
2776
|
+
page.rows.length = 0;
|
|
2777
|
+
}
|
|
2778
|
+
emitWorkerLog(
|
|
2779
|
+
`[perf] runtime csv persist table=${csvDataset.tableNamespace} rows=${csvRowsPersisted} elapsed_ms=${Date.now() - persistStartedAt}`,
|
|
2780
|
+
options,
|
|
2781
|
+
);
|
|
2782
|
+
const csvDatasetPath = `datasets.${csvDataset.tableNamespace}`;
|
|
2783
|
+
emitRunnerEvent(
|
|
2784
|
+
{
|
|
2785
|
+
type: 'execution_event',
|
|
2786
|
+
event: {
|
|
2787
|
+
type: 'dataset.lifecycle',
|
|
2788
|
+
datasetId: createRuntimeDatasetId(
|
|
2789
|
+
config.context.playName!,
|
|
2790
|
+
csvDataset.tableNamespace,
|
|
2791
|
+
),
|
|
2792
|
+
path: csvDatasetPath,
|
|
2793
|
+
tableNamespace: csvDataset.tableNamespace,
|
|
2794
|
+
phase: 'available',
|
|
2795
|
+
persistedRows: csvRowsPersisted,
|
|
2796
|
+
succeededRows: csvRowsPersisted,
|
|
2797
|
+
failedRows: 0,
|
|
2798
|
+
complete: true,
|
|
2799
|
+
at: Date.now(),
|
|
2800
|
+
},
|
|
2801
|
+
},
|
|
2802
|
+
options,
|
|
2803
|
+
);
|
|
2804
|
+
emitRunnerEvent(
|
|
2805
|
+
{
|
|
2806
|
+
type: 'log',
|
|
2807
|
+
at: new Date().toISOString(),
|
|
2808
|
+
source: 'play',
|
|
2809
|
+
line: formatDatasetAvailabilityLog({
|
|
2810
|
+
runId: csvRunId,
|
|
2811
|
+
dataset: {
|
|
2812
|
+
path: csvDatasetPath,
|
|
2813
|
+
tableNamespace: csvDataset.tableNamespace,
|
|
2814
|
+
},
|
|
2815
|
+
persistedRows: csvRowsPersisted,
|
|
2816
|
+
}),
|
|
2817
|
+
},
|
|
2818
|
+
options,
|
|
2819
|
+
);
|
|
2820
|
+
}
|
|
2821
|
+
emitWorkerLog(`Play finished in ${Date.now() - runStartedAt}ms`, options);
|
|
2822
|
+
const normalizeStartedAt = Date.now();
|
|
2823
|
+
const outputRows = normalizeOutputRows(result);
|
|
2824
|
+
const outputRowCount = deriveOutputRowCount(result, outputRows);
|
|
2825
|
+
emitWorkerLog(
|
|
2826
|
+
`[perf] output normalize rows=${outputRowCount} elapsed_ms=${Date.now() - normalizeStartedAt}`,
|
|
2827
|
+
options,
|
|
2828
|
+
);
|
|
2829
|
+
const serializeStartedAt = Date.now();
|
|
2830
|
+
const projectedResult = projectPlayResultForTransport(result);
|
|
2831
|
+
const compactedResult = projectedResult.output;
|
|
2832
|
+
const serializedResult = serializePlayResult(compactedResult);
|
|
2833
|
+
emitWorkerLog(
|
|
2834
|
+
`[perf] result serialize elapsed_ms=${Date.now() - serializeStartedAt}`,
|
|
2835
|
+
options,
|
|
2836
|
+
);
|
|
2837
|
+
if (!totalRows) {
|
|
2838
|
+
totalRows = outputRowCount;
|
|
2839
|
+
}
|
|
2840
|
+
const resultTableNamespace = extractResultTableNamespace(serializedResult);
|
|
2841
|
+
ctx.recordReturn(outputRowCount);
|
|
2842
|
+
|
|
2843
|
+
await releaseRuntimeLeasesOnAppRunnerSettlement({
|
|
2844
|
+
config,
|
|
2845
|
+
runtimeSheetAttempts,
|
|
2846
|
+
runnerRunAttempt,
|
|
2847
|
+
preloadedDbSessions: runtimePreloadedDbSessions,
|
|
2848
|
+
postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
|
|
2849
|
+
emitWorkerLog: (message) => emitWorkerLog(message, options),
|
|
2850
|
+
reason: 'completed',
|
|
2851
|
+
});
|
|
2852
|
+
|
|
2853
|
+
const logs = ctx.getLogs();
|
|
2854
|
+
const runtimeDiagnostics = Array.from(
|
|
2855
|
+
new Map(
|
|
2856
|
+
logs
|
|
2857
|
+
.map((line) => parseCtxFetchHttpFailureDiagnostic(line))
|
|
2858
|
+
.filter((diagnostic): diagnostic is CtxFetchHttpFailureDiagnostic =>
|
|
2859
|
+
Boolean(diagnostic),
|
|
2860
|
+
)
|
|
2861
|
+
.map((diagnostic) => [
|
|
2862
|
+
`${diagnostic.key}\u0000${diagnostic.method}\u0000${diagnostic.url}\u0000${diagnostic.http_status}`,
|
|
2863
|
+
diagnostic,
|
|
2864
|
+
]),
|
|
2865
|
+
).values(),
|
|
2866
|
+
).slice(0, 16);
|
|
2867
|
+
|
|
2868
|
+
return {
|
|
2869
|
+
status: 'completed' as const,
|
|
2870
|
+
output: serializedResult,
|
|
2871
|
+
...(projectedResult.warnings.length > 0
|
|
2872
|
+
? { outputWarnings: projectedResult.warnings }
|
|
2873
|
+
: {}),
|
|
2874
|
+
outputRowCount,
|
|
2875
|
+
logs,
|
|
2876
|
+
...(runtimeDiagnostics.length > 0 ? { runtimeDiagnostics } : {}),
|
|
2877
|
+
stats: ctx.getStats(),
|
|
2878
|
+
steps: ctx.getSteps(),
|
|
2879
|
+
checkpoint: ctx.getCheckpoint(),
|
|
2880
|
+
tableNamespace: resultTableNamespace,
|
|
2881
|
+
totalRows,
|
|
2882
|
+
inserted,
|
|
2883
|
+
skipped,
|
|
2884
|
+
};
|
|
2885
|
+
} catch (caughtError) {
|
|
2886
|
+
// `ctx.sleep()` deliberately unwinds with an exception. Customer code may
|
|
2887
|
+
// catch it and throw a different exception instead, but the invocation's
|
|
2888
|
+
// sticky handoff is still authoritative until this runner returns it to
|
|
2889
|
+
// the scheduler. Resolve that precedence before classifying the terminal
|
|
2890
|
+
// result so a replacement error cannot turn a suspended run into failed.
|
|
2891
|
+
let error: unknown = caughtError;
|
|
2892
|
+
try {
|
|
2893
|
+
ctx.assertNoPendingDurableSuspension();
|
|
2894
|
+
} catch (suspension) {
|
|
2895
|
+
error = suspension;
|
|
2896
|
+
}
|
|
2897
|
+
try {
|
|
2898
|
+
await receiptStoreAdapter?.close();
|
|
2899
|
+
} catch (closeError) {
|
|
2900
|
+
emitWorkerLog(
|
|
2901
|
+
`[warn] runtime receipt writer close failed during teardown: ${
|
|
2902
|
+
closeError instanceof Error ? closeError.message : String(closeError)
|
|
2903
|
+
}`,
|
|
2904
|
+
options,
|
|
2905
|
+
);
|
|
2906
|
+
}
|
|
2907
|
+
if (isPlayExecutionSuspendedError(error)) {
|
|
2908
|
+
// Suspension is a controlled handoff to the scheduler. Promise.all siblings may
|
|
2909
|
+
// still hold runtime receipt leases when the first branch suspends; release
|
|
2910
|
+
// this attempt's leases so the resumed task can reclaim and reconcile
|
|
2911
|
+
// already-terminal child work instead of waiting for stale running receipts.
|
|
2912
|
+
await releaseRuntimeLeasesOnAppRunnerSettlement({
|
|
2913
|
+
config,
|
|
2914
|
+
runtimeSheetAttempts,
|
|
2915
|
+
runnerRunAttempt,
|
|
2916
|
+
preloadedDbSessions: runtimePreloadedDbSessions,
|
|
2917
|
+
postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
|
|
2918
|
+
emitWorkerLog: (message) => emitWorkerLog(message, options),
|
|
2919
|
+
reason: 'suspension',
|
|
2920
|
+
});
|
|
2921
|
+
return {
|
|
2922
|
+
status: 'suspended',
|
|
2923
|
+
suspension: error.suspension,
|
|
2924
|
+
logs: ctx.getLogs(),
|
|
2925
|
+
stats: ctx.getStats(),
|
|
2926
|
+
steps: ctx.getSteps(),
|
|
2927
|
+
checkpoint: ctx.getCheckpoint(),
|
|
2928
|
+
tableNamespace: null,
|
|
2929
|
+
totalRows,
|
|
2930
|
+
inserted,
|
|
2931
|
+
skipped,
|
|
2932
|
+
};
|
|
2933
|
+
}
|
|
2934
|
+
|
|
2935
|
+
// Controlled run-fatal teardown: release the runtime-sheet row leases and
|
|
2936
|
+
// work-receipt leases this attempt still holds so an immediate rerun is not
|
|
2937
|
+
// blocked for the full lease TTL. Best-effort but LOUD — a release failure
|
|
2938
|
+
// is logged and never masks the original run-fatal error. TTL expiry stays
|
|
2939
|
+
// the recovery path for true crashes where this teardown never runs.
|
|
2940
|
+
await releaseRuntimeLeasesOnAppRunnerSettlement({
|
|
2941
|
+
config,
|
|
2942
|
+
runtimeSheetAttempts,
|
|
2943
|
+
runnerRunAttempt,
|
|
2944
|
+
preloadedDbSessions: runtimePreloadedDbSessions,
|
|
2945
|
+
postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
|
|
2946
|
+
emitWorkerLog: (message) => emitWorkerLog(message, options),
|
|
2947
|
+
reason: 'run-fatal',
|
|
2948
|
+
});
|
|
2949
|
+
|
|
2950
|
+
ensureSourceMapSupport();
|
|
2951
|
+
const failure = normalizePlayRunFailure(error);
|
|
2952
|
+
return {
|
|
2953
|
+
status: 'failed',
|
|
2954
|
+
error: formatRunnerFailure(error),
|
|
2955
|
+
errors: [failure],
|
|
2956
|
+
logs: ctx.getLogs(),
|
|
2957
|
+
stats: ctx.getStats(),
|
|
2958
|
+
steps: ctx.getSteps(),
|
|
2959
|
+
checkpoint: ctx.getCheckpoint(),
|
|
2960
|
+
tableNamespace: null,
|
|
2961
|
+
totalRows,
|
|
2962
|
+
inserted,
|
|
2963
|
+
skipped,
|
|
2964
|
+
};
|
|
2965
|
+
}
|
|
2966
|
+
}
|
|
2967
|
+
|
|
2968
|
+
export async function executePlayRunner(
|
|
2969
|
+
config: PlayRunnerExecutionConfig,
|
|
2970
|
+
options?: ExecutePlayRunnerOptions,
|
|
2971
|
+
): Promise<PlayRunnerResult> {
|
|
2972
|
+
const result = await run(config, options);
|
|
2973
|
+
// Inline-executor mirror of the child terminal drain: the caller's finalize
|
|
2974
|
+
// reads the observations table right after this returns, so land in-flight
|
|
2975
|
+
// observation posts first (bounded, best-effort, docflow-gated by emptiness).
|
|
2976
|
+
await drainPendingDocflowObservationPosts(options);
|
|
2977
|
+
return result;
|
|
2978
|
+
}
|
|
2979
|
+
|
|
2980
|
+
function normalizeOutputRows(result: unknown): Record<string, unknown>[] {
|
|
2981
|
+
if (result && typeof result === 'object') {
|
|
2982
|
+
const record = result as Record<string, unknown>;
|
|
2983
|
+
if (isPlayDataset<Record<string, unknown>>(record.rows)) {
|
|
2984
|
+
return [];
|
|
2985
|
+
}
|
|
2986
|
+
if (isPlayDataset<Record<string, unknown>>(record.results)) {
|
|
2987
|
+
return [];
|
|
2988
|
+
}
|
|
2989
|
+
if (isPlayDataset<Record<string, unknown>>(record.leads)) {
|
|
2990
|
+
return [];
|
|
2991
|
+
}
|
|
2992
|
+
if (isPlayDataset<Record<string, unknown>>(record.contacts)) {
|
|
2993
|
+
return [];
|
|
2994
|
+
}
|
|
2995
|
+
if (Array.isArray(record.leads)) {
|
|
2996
|
+
return record.leads as Record<string, unknown>[];
|
|
2997
|
+
}
|
|
2998
|
+
if (Array.isArray(record.contacts)) {
|
|
2999
|
+
return record.contacts as Record<string, unknown>[];
|
|
3000
|
+
}
|
|
3001
|
+
if (Array.isArray(record.rows)) {
|
|
3002
|
+
return record.rows as Record<string, unknown>[];
|
|
3003
|
+
}
|
|
3004
|
+
if (Array.isArray(record.results)) {
|
|
3005
|
+
return record.results as Record<string, unknown>[];
|
|
3006
|
+
}
|
|
3007
|
+
return [record];
|
|
3008
|
+
}
|
|
3009
|
+
|
|
3010
|
+
return [];
|
|
3011
|
+
}
|
|
3012
|
+
|
|
3013
|
+
type PlayResultTransportProjection = {
|
|
3014
|
+
output: Record<string, unknown>;
|
|
3015
|
+
warnings: PlayRunOutputWarning[];
|
|
3016
|
+
};
|
|
3017
|
+
|
|
3018
|
+
function normalizeTopLevelPlayReturn(value: unknown): Record<string, unknown> {
|
|
3019
|
+
if (value === null || value === undefined) {
|
|
3020
|
+
return {};
|
|
3021
|
+
}
|
|
3022
|
+
if (Array.isArray(value)) {
|
|
3023
|
+
return { rows: value };
|
|
3024
|
+
}
|
|
3025
|
+
if (isPlayDataset(value) || isSerializedPlayDataset(value)) {
|
|
3026
|
+
return { rows: value };
|
|
3027
|
+
}
|
|
3028
|
+
if (typeof value !== 'object') {
|
|
3029
|
+
return { value };
|
|
3030
|
+
}
|
|
3031
|
+
|
|
3032
|
+
const record = { ...(value as Record<string, unknown>) };
|
|
3033
|
+
if (Object.prototype.hasOwnProperty.call(record, '_metadata')) {
|
|
3034
|
+
record.metadata = record._metadata;
|
|
3035
|
+
delete record._metadata;
|
|
3036
|
+
}
|
|
3037
|
+
return record;
|
|
3038
|
+
}
|
|
3039
|
+
|
|
3040
|
+
export function compactPlayResultForTransport(value: unknown): unknown {
|
|
3041
|
+
return projectPlayResultForTransport(value).output;
|
|
3042
|
+
}
|
|
3043
|
+
|
|
3044
|
+
export function projectPlayResultForTransport(
|
|
3045
|
+
value: unknown,
|
|
3046
|
+
): PlayResultTransportProjection {
|
|
3047
|
+
const record = normalizeTopLevelPlayReturn(value);
|
|
3048
|
+
const output: Record<string, unknown> = {};
|
|
3049
|
+
|
|
3050
|
+
for (const [key, entry] of Object.entries(record)) {
|
|
3051
|
+
if (isPlayDataset(entry)) {
|
|
3052
|
+
output[key] = entry.toJSON();
|
|
3053
|
+
continue;
|
|
3054
|
+
}
|
|
3055
|
+
if (isSerializedPlayDataset(entry)) {
|
|
3056
|
+
output[key] = entry;
|
|
3057
|
+
continue;
|
|
3058
|
+
}
|
|
3059
|
+
output[key] = entry;
|
|
3060
|
+
}
|
|
3061
|
+
|
|
3062
|
+
assertCustomerPlayReturnWithinLimit({
|
|
3063
|
+
value: output,
|
|
3064
|
+
});
|
|
3065
|
+
return { output, warnings: [] };
|
|
3066
|
+
}
|
|
3067
|
+
|
|
3068
|
+
function serializePlayResult<T>(value: T): T {
|
|
3069
|
+
return JSON.parse(JSON.stringify(value)) as T;
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3072
|
+
export function deriveOutputRowCount(
|
|
3073
|
+
result: unknown,
|
|
3074
|
+
normalizedRows: Record<string, unknown>[],
|
|
3075
|
+
): number {
|
|
3076
|
+
if (result && typeof result === 'object' && !Array.isArray(result)) {
|
|
3077
|
+
const record = result as Record<string, unknown>;
|
|
3078
|
+
if (isPlayDataset(record.rows)) {
|
|
3079
|
+
return record.rows.toJSON().count;
|
|
3080
|
+
}
|
|
3081
|
+
if (isPlayDataset(record.results)) {
|
|
3082
|
+
return record.results.toJSON().count;
|
|
3083
|
+
}
|
|
3084
|
+
if (isPlayDataset(record.leads)) {
|
|
3085
|
+
return record.leads.toJSON().count;
|
|
3086
|
+
}
|
|
3087
|
+
if (isPlayDataset(record.contacts)) {
|
|
3088
|
+
return record.contacts.toJSON().count;
|
|
3089
|
+
}
|
|
3090
|
+
if (isSerializedPlayDataset(record.rows)) {
|
|
3091
|
+
return record.rows.count;
|
|
3092
|
+
}
|
|
3093
|
+
if (isSerializedPlayDataset(record.results)) {
|
|
3094
|
+
return record.results.count;
|
|
3095
|
+
}
|
|
3096
|
+
if (isSerializedPlayDataset(record.leads)) {
|
|
3097
|
+
return record.leads.count;
|
|
3098
|
+
}
|
|
3099
|
+
if (isSerializedPlayDataset(record.contacts)) {
|
|
3100
|
+
return record.contacts.count;
|
|
3101
|
+
}
|
|
3102
|
+
}
|
|
3103
|
+
return normalizedRows.length;
|
|
3104
|
+
}
|
|
3105
|
+
|
|
3106
|
+
export function extractResultTableNamespace(result: unknown): string | null {
|
|
3107
|
+
if (result && typeof result === 'object' && !Array.isArray(result)) {
|
|
3108
|
+
const record = result as Record<string, unknown>;
|
|
3109
|
+
for (const candidate of [
|
|
3110
|
+
record.rows,
|
|
3111
|
+
record.results,
|
|
3112
|
+
record.leads,
|
|
3113
|
+
record.contacts,
|
|
3114
|
+
]) {
|
|
3115
|
+
if (isPlayDataset(candidate)) {
|
|
3116
|
+
return candidate.toJSON().tableNamespace ?? null;
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
for (const candidate of [
|
|
3120
|
+
record.rows,
|
|
3121
|
+
record.results,
|
|
3122
|
+
record.leads,
|
|
3123
|
+
record.contacts,
|
|
3124
|
+
]) {
|
|
3125
|
+
if (isSerializedPlayDataset(candidate)) {
|
|
3126
|
+
return candidate.tableNamespace ?? null;
|
|
3127
|
+
}
|
|
3128
|
+
}
|
|
3129
|
+
}
|
|
3130
|
+
return null;
|
|
3131
|
+
}
|
|
3132
|
+
|
|
3133
|
+
async function waitForConfigFile(configPath: string): Promise<void> {
|
|
3134
|
+
const startedAt = Date.now();
|
|
3135
|
+
for (;;) {
|
|
3136
|
+
try {
|
|
3137
|
+
await readFile(configPath);
|
|
3138
|
+
return;
|
|
3139
|
+
} catch (error) {
|
|
3140
|
+
const code = (error as { code?: unknown }).code;
|
|
3141
|
+
if (code !== 'ENOENT') {
|
|
3142
|
+
throw error;
|
|
3143
|
+
}
|
|
3144
|
+
if (Date.now() - startedAt > WAIT_CONFIG_TIMEOUT_MS) {
|
|
3145
|
+
throw new Error(
|
|
3146
|
+
`Timed out waiting for play runner config at ${configPath}.`,
|
|
3147
|
+
);
|
|
3148
|
+
}
|
|
3149
|
+
await new Promise((resolve) => setTimeout(resolve, WAIT_CONFIG_POLL_MS));
|
|
3150
|
+
}
|
|
3151
|
+
}
|
|
3152
|
+
}
|
|
3153
|
+
|
|
3154
|
+
export async function materializeTransportedArtifact(
|
|
3155
|
+
config: PlayRunnerExecutionConfig,
|
|
3156
|
+
): Promise<PlayRunnerExecutionConfig> {
|
|
3157
|
+
const bundledCodePath = config.artifactTransport?.bundledCodePath?.trim();
|
|
3158
|
+
if (!bundledCodePath) {
|
|
3159
|
+
if (!config.artifact.bundledCode) {
|
|
3160
|
+
throw new Error('Play runner config is missing artifact bundledCode.');
|
|
3161
|
+
}
|
|
3162
|
+
return config;
|
|
3163
|
+
}
|
|
3164
|
+
|
|
3165
|
+
const artifactBuffer = await readFile(bundledCodePath);
|
|
3166
|
+
const bundledCode =
|
|
3167
|
+
config.artifactTransport?.bundledCodeEncoding === 'gzip'
|
|
3168
|
+
? gunzipSync(artifactBuffer).toString('utf-8')
|
|
3169
|
+
: artifactBuffer.toString('utf-8');
|
|
3170
|
+
if (!bundledCode) {
|
|
3171
|
+
throw new Error(
|
|
3172
|
+
`Play runner artifact file ${bundledCodePath} did not contain bundledCode.`,
|
|
3173
|
+
);
|
|
3174
|
+
}
|
|
3175
|
+
const sourceMapPath = config.artifactTransport?.sourceMapPath?.trim();
|
|
3176
|
+
const sourceMap = sourceMapPath
|
|
3177
|
+
? await readFile(sourceMapPath, 'utf-8')
|
|
3178
|
+
: config.artifact.sourceMap;
|
|
3179
|
+
|
|
3180
|
+
return {
|
|
3181
|
+
...config,
|
|
3182
|
+
artifact: {
|
|
3183
|
+
...config.artifact,
|
|
3184
|
+
bundledCode,
|
|
3185
|
+
sourceMap,
|
|
3186
|
+
},
|
|
3187
|
+
};
|
|
3188
|
+
}
|
|
3189
|
+
|
|
3190
|
+
async function main() {
|
|
3191
|
+
emitWorkerLog(
|
|
3192
|
+
`[perf] process main elapsed_ms=${Math.round(performance.now())}`,
|
|
3193
|
+
);
|
|
3194
|
+
const waitForConfig = process.argv[2] === '--wait-config';
|
|
3195
|
+
const configPath = waitForConfig ? process.argv[3] : process.argv[2];
|
|
3196
|
+
if (!configPath) {
|
|
3197
|
+
throw new Error(
|
|
3198
|
+
'Usage: node play-runner.cjs [--wait-config] <config.json|config.json.gz>',
|
|
3199
|
+
);
|
|
3200
|
+
}
|
|
3201
|
+
if (waitForConfig) {
|
|
3202
|
+
await waitForConfigFile(configPath);
|
|
3203
|
+
}
|
|
3204
|
+
const configLoadStartedAt = Date.now();
|
|
3205
|
+
const configBuffer = await readFile(configPath);
|
|
3206
|
+
const configJson = configPath.endsWith('.gz')
|
|
3207
|
+
? gunzipSync(configBuffer).toString('utf-8')
|
|
3208
|
+
: configBuffer.toString('utf-8');
|
|
3209
|
+
const parsedConfig = JSON.parse(configJson) as PlayRunnerExecutionConfig;
|
|
3210
|
+
writeRunnerStartupDiagnostic(
|
|
3211
|
+
process.env.DEEPLINE_PLAY_RUNNER_STARTUP_DIAGNOSTIC_PATH,
|
|
3212
|
+
{ phase: 'booted' },
|
|
3213
|
+
);
|
|
3214
|
+
emitWorkerLog(
|
|
3215
|
+
`[perf] config load elapsed_ms=${Date.now() - configLoadStartedAt}`,
|
|
3216
|
+
);
|
|
3217
|
+
|
|
3218
|
+
// Set terminal push wiring before any startup work that can fail. A failed
|
|
3219
|
+
// artifact materialization can then report its terminal whenever the gateway
|
|
3220
|
+
// is reachable instead of relying only on the detached ceiling salvage.
|
|
3221
|
+
runnerTerminalPushContext = terminalPushContextFromConfig(parsedConfig);
|
|
3222
|
+
const executionController = new AbortController();
|
|
3223
|
+
const heartbeat = startRunnerPushHeartbeatFromConfig(parsedConfig, () => {
|
|
3224
|
+
executionController.abort(
|
|
3225
|
+
new Error('Play runner stopped by the runtime heartbeat channel.'),
|
|
3226
|
+
);
|
|
3227
|
+
});
|
|
3228
|
+
try {
|
|
3229
|
+
const artifactMaterializeStartedAt = Date.now();
|
|
3230
|
+
const config = await materializeTransportedArtifactWhileRunnerParks(
|
|
3231
|
+
parsedConfig,
|
|
3232
|
+
heartbeat,
|
|
3233
|
+
);
|
|
3234
|
+
emitWorkerLog(
|
|
3235
|
+
`[perf] artifact materialize elapsed_ms=${Date.now() - artifactMaterializeStartedAt}`,
|
|
3236
|
+
);
|
|
3237
|
+
|
|
3238
|
+
runnerProgressPusher = startRunnerProgressPusherFromConfig(config);
|
|
3239
|
+
const runtimeDeadline = startConfiguredPlayRunnerRuntimeDeadline({
|
|
3240
|
+
config,
|
|
3241
|
+
onDeadline: async (runtimeLimitSeconds) => {
|
|
3242
|
+
executionController.abort(
|
|
3243
|
+
new Error(
|
|
3244
|
+
`Play runner reached its ${runtimeLimitSeconds}-second runtime limit.`,
|
|
3245
|
+
),
|
|
3246
|
+
);
|
|
3247
|
+
heartbeat?.stop();
|
|
3248
|
+
await terminateRunnerForRuntimeDeadline(runtimeLimitSeconds);
|
|
3249
|
+
},
|
|
3250
|
+
onError: (error) => {
|
|
3251
|
+
console.error('[play-runner.runtime-deadline] terminal_failed', {
|
|
3252
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3253
|
+
});
|
|
3254
|
+
process.exit(1);
|
|
3255
|
+
},
|
|
3256
|
+
});
|
|
3257
|
+
writeRunnerRuntimeStartedMarker(
|
|
3258
|
+
process.env.DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH,
|
|
3259
|
+
);
|
|
3260
|
+
let result: PlayRunnerResult | null = null;
|
|
3261
|
+
let executionFailed = false;
|
|
3262
|
+
let executionError: unknown = undefined;
|
|
3263
|
+
try {
|
|
3264
|
+
result = await executePlayRunner(config, {
|
|
3265
|
+
signal: executionController.signal,
|
|
3266
|
+
});
|
|
3267
|
+
} catch (error) {
|
|
3268
|
+
executionFailed = true;
|
|
3269
|
+
executionError = error;
|
|
3270
|
+
}
|
|
3271
|
+
if (!runtimeDeadline.claimCompletion()) {
|
|
3272
|
+
await runtimeDeadline.deadlineSettled;
|
|
3273
|
+
throw new Error(
|
|
3274
|
+
'Runtime deadline handler returned without terminating the runner.',
|
|
3275
|
+
);
|
|
3276
|
+
}
|
|
3277
|
+
// claimCompletion() is the exact customer-execution boundary. Publish the
|
|
3278
|
+
// external fence immediately, before formatting failures or serializing the
|
|
3279
|
+
// terminal payload, so a run that finishes at the deadline receives the
|
|
3280
|
+
// terminal-settlement grace instead of being reclassified as a timeout.
|
|
3281
|
+
const workCompletedAtMs = Date.now();
|
|
3282
|
+
writeRunnerRuntimeCompletedMarker(
|
|
3283
|
+
process.env.DEEPLINE_PLAY_RUNNER_RUNTIME_COMPLETED_PATH,
|
|
3284
|
+
{ at: workCompletedAtMs },
|
|
3285
|
+
);
|
|
3286
|
+
if (executionFailed) {
|
|
3287
|
+
ensureSourceMapSupport();
|
|
3288
|
+
const message = formatRunnerFailure(executionError);
|
|
3289
|
+
result = {
|
|
3290
|
+
status: 'failed',
|
|
3291
|
+
error: message,
|
|
3292
|
+
errors: [normalizePlayRunFailure(executionError)],
|
|
3293
|
+
logs: [],
|
|
3294
|
+
stats: {},
|
|
3295
|
+
steps: [],
|
|
3296
|
+
tableNamespace: null,
|
|
3297
|
+
};
|
|
3298
|
+
}
|
|
3299
|
+
if (!result) {
|
|
3300
|
+
throw new Error('Play runner completed without a result.');
|
|
3301
|
+
}
|
|
3302
|
+
(
|
|
3303
|
+
result as PlayRunnerResult & { workCompletedAtMs?: number }
|
|
3304
|
+
).workCompletedAtMs = workCompletedAtMs;
|
|
3305
|
+
const terminalResult = runnerTerminalPushContext
|
|
3306
|
+
? runnerTerminalResultForTransport(result)
|
|
3307
|
+
: null;
|
|
3308
|
+
heartbeat?.stop();
|
|
3309
|
+
// Ordering guarantee for the durable trace (ADR 0016): the pushed terminal
|
|
3310
|
+
// wakes the parked worker whose finalize reads the observations table, so
|
|
3311
|
+
// in-flight observation posts must land first. Bounded — never fails or
|
|
3312
|
+
// delays the terminal beyond the cap; no-op for undiagrammed plays.
|
|
3313
|
+
// Runner progress joins the same terminal handoff immediately below.
|
|
3314
|
+
await drainPendingDocflowObservationPosts();
|
|
3315
|
+
if (terminalResult) {
|
|
3316
|
+
// Persist the exact terminal line locally before any network settlement.
|
|
3317
|
+
// If the gateway remains unavailable through the grace period, the shell
|
|
3318
|
+
// epilogue can still salvage this authored result instead of inventing a
|
|
3319
|
+
// runtime-limit failure after customer code already finished.
|
|
3320
|
+
await writeRunnerEventToStdout({
|
|
3321
|
+
type: 'result',
|
|
3322
|
+
result: terminalResult,
|
|
3323
|
+
});
|
|
3324
|
+
}
|
|
3325
|
+
// Push execution (B2-final): the pushed terminal is what WAKES the parked
|
|
3326
|
+
// worker attempt. Push before exiting; on exhaustion the captured output
|
|
3327
|
+
// file remains the ceiling-timeout salvage source.
|
|
3328
|
+
if (runnerTerminalPushContext && terminalResult) {
|
|
3329
|
+
// Every terminal boundary can wake a successor attempt. Transfer all
|
|
3330
|
+
// unacknowledged observations into that same ordered handoff before the
|
|
3331
|
+
// wake is visible; draining a suspended/failed attempt afterward lets
|
|
3332
|
+
// the successor claim the log channel and fence its predecessor's tail.
|
|
3333
|
+
const finalProgress = runnerProgressPusher?.sealForTerminal() ?? {
|
|
3334
|
+
events: [],
|
|
3335
|
+
liveLogOffset: 0,
|
|
3336
|
+
};
|
|
3337
|
+
runnerProgressPusher = null;
|
|
3338
|
+
await pushRunnerTerminalIfConfigured(terminalResult, {
|
|
3339
|
+
warn: createPostTerminalDiagnosticWriter(),
|
|
3340
|
+
finalProgressLogOffset: finalProgress.liveLogOffset,
|
|
3341
|
+
...(finalProgress.events.length
|
|
3342
|
+
? {
|
|
3343
|
+
finalProgressEvents: finalProgress.events,
|
|
3344
|
+
}
|
|
3345
|
+
: {}),
|
|
3346
|
+
});
|
|
3347
|
+
return exitRunner(0);
|
|
3348
|
+
}
|
|
3349
|
+
await runnerProgressPusher?.stop();
|
|
3350
|
+
runnerProgressPusher = null;
|
|
3351
|
+
// The parent process owns play status from the structured result event.
|
|
3352
|
+
// Keep the process exit clean so remote command transports do not drop the
|
|
3353
|
+
// final JSON line on non-zero exits.
|
|
3354
|
+
await emitAndExit(
|
|
3355
|
+
{
|
|
3356
|
+
type: 'result',
|
|
3357
|
+
result,
|
|
3358
|
+
},
|
|
3359
|
+
0,
|
|
3360
|
+
);
|
|
3361
|
+
} finally {
|
|
3362
|
+
heartbeat?.stop();
|
|
3363
|
+
const pusher = runnerProgressPusher;
|
|
3364
|
+
runnerProgressPusher = null;
|
|
3365
|
+
await pusher?.stop().catch(() => undefined);
|
|
3366
|
+
}
|
|
3367
|
+
}
|
|
3368
|
+
|
|
3369
|
+
export async function waitForRunnerExecutionAcknowledgement(
|
|
3370
|
+
heartbeat: ReturnType<typeof startRunnerPushHeartbeatFromConfig>,
|
|
3371
|
+
timeoutMs = RUNNER_EXECUTION_ACK_TIMEOUT_MS,
|
|
3372
|
+
): Promise<void> {
|
|
3373
|
+
if (!heartbeat) return;
|
|
3374
|
+
const deadline = Date.now() + Math.max(1, timeoutMs);
|
|
3375
|
+
let retryDelayMs = 100;
|
|
3376
|
+
while (Date.now() < deadline) {
|
|
3377
|
+
heartbeat.requestHeartbeat();
|
|
3378
|
+
const remainingMs = Math.max(1, deadline - Date.now());
|
|
3379
|
+
const acknowledged = await Promise.race([
|
|
3380
|
+
heartbeat.executionAcknowledged.then(() => true),
|
|
3381
|
+
new Promise<false>((resolve) =>
|
|
3382
|
+
setTimeout(resolve, Math.min(retryDelayMs, remainingMs), false),
|
|
3383
|
+
),
|
|
3384
|
+
]);
|
|
3385
|
+
if (acknowledged) return;
|
|
3386
|
+
retryDelayMs = Math.min(1_000, retryDelayMs * 2);
|
|
3387
|
+
}
|
|
3388
|
+
throw new Error(
|
|
3389
|
+
`RUNTIME_SANDBOX_START_FAILED: scheduler did not acknowledge the detached runner park within ${Math.max(1, timeoutMs)}ms. Customer play code was not started.`,
|
|
3390
|
+
);
|
|
3391
|
+
}
|
|
3392
|
+
|
|
3393
|
+
/**
|
|
3394
|
+
* Begins local artifact readback as soon as the runner has parsed its config,
|
|
3395
|
+
* while preserving the durable runner-park fence before customer code receives
|
|
3396
|
+
* that artifact. The artifact contains source text only; materializing it does
|
|
3397
|
+
* not evaluate user code or make any runtime-state transition.
|
|
3398
|
+
*
|
|
3399
|
+
* Keep a handled materialization result while waiting for the acknowledgement:
|
|
3400
|
+
* a failed read must still wait for the initial heartbeat so the configured
|
|
3401
|
+
* terminal push can report the authored failure through the normal path.
|
|
3402
|
+
*/
|
|
3403
|
+
export async function materializeTransportedArtifactWhileRunnerParks(
|
|
3404
|
+
parsedConfig: PlayRunnerExecutionConfig,
|
|
3405
|
+
heartbeat: ReturnType<typeof startRunnerPushHeartbeatFromConfig>,
|
|
3406
|
+
materialize: (
|
|
3407
|
+
config: PlayRunnerExecutionConfig,
|
|
3408
|
+
) => Promise<PlayRunnerExecutionConfig> = materializeTransportedArtifact,
|
|
3409
|
+
): Promise<PlayRunnerExecutionConfig> {
|
|
3410
|
+
const materialization = materialize(parsedConfig).then(
|
|
3411
|
+
(config) => ({ ok: true as const, config }),
|
|
3412
|
+
(error) => ({ ok: false as const, error }),
|
|
3413
|
+
);
|
|
3414
|
+
|
|
3415
|
+
// The first gateway heartbeat is the durable readiness fence. The worker
|
|
3416
|
+
// observes that same Postgres fact directly. It still gates every call to
|
|
3417
|
+
// executePlayRunner; only file I/O overlaps the handoff.
|
|
3418
|
+
await heartbeat?.firstHeartbeat;
|
|
3419
|
+
await waitForRunnerExecutionAcknowledgement(heartbeat);
|
|
3420
|
+
|
|
3421
|
+
const outcome = await materialization;
|
|
3422
|
+
if ('config' in outcome) {
|
|
3423
|
+
return outcome.config;
|
|
3424
|
+
}
|
|
3425
|
+
throw outcome.error;
|
|
3426
|
+
}
|
|
3427
|
+
|
|
3428
|
+
export function writeRunnerRuntimeStartedMarker(
|
|
3429
|
+
path: string | null | undefined,
|
|
3430
|
+
detail: { pid?: number; at?: number } = {},
|
|
3431
|
+
): void {
|
|
3432
|
+
const runtimeStartedPath = path?.trim();
|
|
3433
|
+
if (!runtimeStartedPath) return;
|
|
3434
|
+
writeFileSync(
|
|
3435
|
+
runtimeStartedPath,
|
|
3436
|
+
JSON.stringify({
|
|
3437
|
+
started: true,
|
|
3438
|
+
pid: detail.pid ?? process.pid,
|
|
3439
|
+
at: detail.at ?? Date.now(),
|
|
3440
|
+
}),
|
|
3441
|
+
'utf-8',
|
|
3442
|
+
);
|
|
3443
|
+
}
|
|
3444
|
+
|
|
3445
|
+
export function writeRunnerRuntimeCompletedMarker(
|
|
3446
|
+
path: string | null | undefined,
|
|
3447
|
+
detail: { pid?: number; at?: number } = {},
|
|
3448
|
+
): void {
|
|
3449
|
+
const runtimeCompletedPath = path?.trim();
|
|
3450
|
+
if (!runtimeCompletedPath) return;
|
|
3451
|
+
writeFileSync(
|
|
3452
|
+
runtimeCompletedPath,
|
|
3453
|
+
JSON.stringify({
|
|
3454
|
+
completed: true,
|
|
3455
|
+
pid: detail.pid ?? process.pid,
|
|
3456
|
+
at: detail.at ?? Date.now(),
|
|
3457
|
+
}),
|
|
3458
|
+
'utf-8',
|
|
3459
|
+
);
|
|
3460
|
+
}
|
|
3461
|
+
|
|
3462
|
+
/**
|
|
3463
|
+
* A tiny, strictly non-secret handoff record for detached Daytona startup.
|
|
3464
|
+
* The worker reads it only when the scheduler did not see the first heartbeat;
|
|
3465
|
+
* stdout is deliberately not inspected because it may contain customer data.
|
|
3466
|
+
*/
|
|
3467
|
+
export function writeRunnerStartupDiagnostic(
|
|
3468
|
+
path: string | null | undefined,
|
|
3469
|
+
detail:
|
|
3470
|
+
| { phase: 'booted' }
|
|
3471
|
+
| { phase: 'heartbeat_transport'; event: RunnerGatewayTransportEvent },
|
|
3472
|
+
): void {
|
|
3473
|
+
const diagnosticPath = path?.trim();
|
|
3474
|
+
if (!diagnosticPath) return;
|
|
3475
|
+
try {
|
|
3476
|
+
writeFileSync(
|
|
3477
|
+
diagnosticPath,
|
|
3478
|
+
JSON.stringify({ schemaVersion: 1, at: Date.now(), ...detail }),
|
|
3479
|
+
'utf-8',
|
|
3480
|
+
);
|
|
3481
|
+
} catch {
|
|
3482
|
+
// Diagnostics must never affect the runner's durable control path.
|
|
3483
|
+
}
|
|
3484
|
+
}
|
|
3485
|
+
|
|
3486
|
+
/** Terminal push wiring (set once in main after the config loads). */
|
|
3487
|
+
let runnerTerminalPushContext: {
|
|
3488
|
+
gatewayBaseUrl: string;
|
|
3489
|
+
executorToken: string;
|
|
3490
|
+
runId: string;
|
|
3491
|
+
} | null = null;
|
|
3492
|
+
|
|
3493
|
+
function terminalPushContextFromConfig(
|
|
3494
|
+
config: PlayRunnerExecutionConfig,
|
|
3495
|
+
): typeof runnerTerminalPushContext {
|
|
3496
|
+
const push = config.context.runnerPushExecution;
|
|
3497
|
+
const gatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
|
|
3498
|
+
const executorToken = config.context.executorToken?.trim();
|
|
3499
|
+
if (!push || !gatewayBaseUrl || !executorToken) {
|
|
3500
|
+
return null;
|
|
3501
|
+
}
|
|
3502
|
+
return { gatewayBaseUrl, executorToken, runId: push.runId };
|
|
3503
|
+
}
|
|
3504
|
+
|
|
3505
|
+
async function pushRunnerTerminalIfConfigured(
|
|
3506
|
+
result: PlayRunnerResult | PlayRunnerTerminalTransportResult,
|
|
3507
|
+
options?: Pick<
|
|
3508
|
+
PushRunnerTerminalOptions,
|
|
3509
|
+
'warn' | 'finalProgressEvents' | 'finalProgressLogOffset'
|
|
3510
|
+
>,
|
|
3511
|
+
): Promise<void> {
|
|
3512
|
+
const context = runnerTerminalPushContext;
|
|
3513
|
+
if (!context) return;
|
|
3514
|
+
await pushRunnerTerminal({
|
|
3515
|
+
gatewayBaseUrl: context.gatewayBaseUrl,
|
|
3516
|
+
executorToken: context.executorToken,
|
|
3517
|
+
runId: context.runId,
|
|
3518
|
+
result,
|
|
3519
|
+
...options,
|
|
3520
|
+
});
|
|
3521
|
+
}
|
|
3522
|
+
|
|
3523
|
+
/**
|
|
3524
|
+
* Start the push-execution live progress pusher when the launch enabled push
|
|
3525
|
+
* execution (Daytona sandbox runner). Returns null for worker-babysat backends,
|
|
3526
|
+
* which keep their sidecar-file progress path.
|
|
3527
|
+
*/
|
|
3528
|
+
function startRunnerProgressPusherFromConfig(
|
|
3529
|
+
config: PlayRunnerExecutionConfig,
|
|
3530
|
+
): RunnerProgressPusher | null {
|
|
3531
|
+
const push = config.context.runnerPushExecution;
|
|
3532
|
+
const gatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
|
|
3533
|
+
const executorToken = config.context.executorToken?.trim();
|
|
3534
|
+
const playName = config.context.playName?.trim();
|
|
3535
|
+
if (!push || !gatewayBaseUrl || !executorToken || !playName) {
|
|
3536
|
+
return null;
|
|
3537
|
+
}
|
|
3538
|
+
return createRunnerProgressPusher({
|
|
3539
|
+
gatewayBaseUrl,
|
|
3540
|
+
executorToken,
|
|
3541
|
+
runId: push.runId,
|
|
3542
|
+
playName,
|
|
3543
|
+
});
|
|
3544
|
+
}
|
|
3545
|
+
|
|
3546
|
+
/**
|
|
3547
|
+
* Start the push-execution lease heartbeat when the launch enabled it (Daytona
|
|
3548
|
+
* sandbox runner). Returns null for worker-babysat backends. On the gateway's
|
|
3549
|
+
* terminal cancellation channel — or a gateway that stays unreachable past the
|
|
3550
|
+
* lease window — the loop calls back here to terminate the runner loudly, the
|
|
3551
|
+
* same way the worker's CancelledTask path aborts the attempt today.
|
|
3552
|
+
*/
|
|
3553
|
+
function startRunnerPushHeartbeatFromConfig(
|
|
3554
|
+
config: PlayRunnerExecutionConfig,
|
|
3555
|
+
onSelfTerminate?: (stop: RunnerHeartbeatStop) => void,
|
|
3556
|
+
): ReturnType<typeof startRunnerHeartbeatLoop> | null {
|
|
3557
|
+
const push = config.context.runnerPushExecution;
|
|
3558
|
+
const gatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
|
|
3559
|
+
const executorToken = config.context.executorToken?.trim();
|
|
3560
|
+
if (!push || !gatewayBaseUrl || !executorToken) {
|
|
3561
|
+
return null;
|
|
3562
|
+
}
|
|
3563
|
+
return startRunnerHeartbeatLoop({
|
|
3564
|
+
gatewayBaseUrl,
|
|
3565
|
+
executorToken,
|
|
3566
|
+
runId: push.runId,
|
|
3567
|
+
leaseSeconds: push.leaseSeconds,
|
|
3568
|
+
onSelfTerminate: (stop) => {
|
|
3569
|
+
onSelfTerminate?.(stop);
|
|
3570
|
+
void terminateRunnerForHeartbeatStop(stop);
|
|
3571
|
+
},
|
|
3572
|
+
onTransport: (event) => {
|
|
3573
|
+
writeRunnerStartupDiagnostic(
|
|
3574
|
+
process.env.DEEPLINE_PLAY_RUNNER_STARTUP_DIAGNOSTIC_PATH,
|
|
3575
|
+
{ phase: 'heartbeat_transport', event },
|
|
3576
|
+
);
|
|
3577
|
+
},
|
|
3578
|
+
});
|
|
3579
|
+
}
|
|
3580
|
+
|
|
3581
|
+
function terminateRunnerForHeartbeatStop(
|
|
3582
|
+
stop: RunnerHeartbeatStop,
|
|
3583
|
+
): Promise<never> {
|
|
3584
|
+
const error =
|
|
3585
|
+
stop.reason === 'terminal'
|
|
3586
|
+
? stop.terminal === 'cancelled'
|
|
3587
|
+
? 'Play run was cancelled; sandbox runner self-terminated on the receipt gateway heartbeat cancellation channel.'
|
|
3588
|
+
: 'Play run already failed in the scheduler; sandbox runner self-terminated on the receipt gateway heartbeat cancellation channel.'
|
|
3589
|
+
: `Sandbox runner could not reach the receipt gateway for ${stop.reason === 'unreachable' ? stop.sinceMs : 0}ms (past the lease window); self-terminating — the parked attempt's ceiling timeout salvages this result from the captured output. ${formatRunnerGatewayTransportSummary(stop.reason === 'unreachable' ? stop.gatewayTransport : undefined)}`;
|
|
3590
|
+
const failedResult = runnerTerminalResultForTransport({
|
|
3591
|
+
status: 'failed',
|
|
3592
|
+
error,
|
|
3593
|
+
logs: [],
|
|
3594
|
+
stats: {},
|
|
3595
|
+
steps: [],
|
|
3596
|
+
tableNamespace: null,
|
|
3597
|
+
});
|
|
3598
|
+
return emitAndExit(
|
|
3599
|
+
{
|
|
3600
|
+
type: 'result',
|
|
3601
|
+
result: failedResult,
|
|
3602
|
+
},
|
|
3603
|
+
stop.reason === 'terminal' && stop.terminal === 'cancelled' ? 0 : 1,
|
|
3604
|
+
);
|
|
3605
|
+
}
|
|
3606
|
+
|
|
3607
|
+
async function terminateRunnerForRuntimeDeadline(
|
|
3608
|
+
runtimeLimitSeconds: number,
|
|
3609
|
+
): Promise<never> {
|
|
3610
|
+
const error = runtimeLimitExceededCause(runtimeLimitSeconds);
|
|
3611
|
+
const failedResult = runnerTerminalResultForTransport({
|
|
3612
|
+
status: 'failed',
|
|
3613
|
+
error,
|
|
3614
|
+
errors: [runtimeLimitExceededFailure(runtimeLimitSeconds, error)],
|
|
3615
|
+
logs: [],
|
|
3616
|
+
stats: {},
|
|
3617
|
+
steps: [],
|
|
3618
|
+
tableNamespace: null,
|
|
3619
|
+
});
|
|
3620
|
+
// Stop customer code immediately. The structured stdout event is captured by
|
|
3621
|
+
// the Daytona shell wrapper, whose crash epilogue owns the bounded gateway
|
|
3622
|
+
// push during the outer grace period. Awaiting network/progress work here
|
|
3623
|
+
// would let customer code continue dispatching after its deadline.
|
|
3624
|
+
return emitAndExit(
|
|
3625
|
+
{
|
|
3626
|
+
type: 'result',
|
|
3627
|
+
result: failedResult,
|
|
3628
|
+
},
|
|
3629
|
+
0,
|
|
3630
|
+
);
|
|
3631
|
+
}
|
|
3632
|
+
|
|
3633
|
+
export function runPlayRunnerMain(): void {
|
|
3634
|
+
main().catch(async (error) => {
|
|
3635
|
+
ensureSourceMapSupport();
|
|
3636
|
+
const message = formatRunnerFailure(error);
|
|
3637
|
+
const failedResult: PlayRunnerResult = {
|
|
3638
|
+
status: 'failed',
|
|
3639
|
+
error: message,
|
|
3640
|
+
errors: [normalizePlayRunFailure(error)],
|
|
3641
|
+
logs: [],
|
|
3642
|
+
stats: {},
|
|
3643
|
+
steps: [],
|
|
3644
|
+
tableNamespace: null,
|
|
3645
|
+
};
|
|
3646
|
+
const terminalResult = runnerTerminalResultForTransport(failedResult);
|
|
3647
|
+
// Write the salvageable terminal before its gateway push. Startup failures
|
|
3648
|
+
// and unexpected settlement failures must retain their real diagnosis even
|
|
3649
|
+
// if the push itself stalls or exhausts retries.
|
|
3650
|
+
await writeRunnerEventToStdout({
|
|
3651
|
+
type: 'result',
|
|
3652
|
+
result: terminalResult,
|
|
3653
|
+
}).catch(() => undefined);
|
|
3654
|
+
// Push execution: wake the parked worker attempt NOW instead of making it
|
|
3655
|
+
// wait for the ceiling-timeout salvage. Best-effort — the output file
|
|
3656
|
+
// still carries this result for the salvage path.
|
|
3657
|
+
await pushRunnerTerminalIfConfigured(terminalResult, {
|
|
3658
|
+
warn: createPostTerminalDiagnosticWriter(),
|
|
3659
|
+
}).catch(() => undefined);
|
|
3660
|
+
void exitRunner(0);
|
|
3661
|
+
});
|
|
3662
|
+
}
|