deepline 0.3.89 → 0.3.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (267) hide show
  1. package/contracts/install-commands.json +29 -0
  2. package/dist/bundling-sources/sdk/src/client.ts +221 -43
  3. package/dist/bundling-sources/sdk/src/index.ts +4 -0
  4. package/dist/bundling-sources/sdk/src/monitor-fleet-contract.ts +2 -556
  5. package/dist/bundling-sources/sdk/src/monitor-fleets.ts +2 -2
  6. package/dist/bundling-sources/sdk/src/pagination.ts +25 -0
  7. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
  8. package/dist/bundling-sources/sdk/src/release.ts +11 -9
  9. package/dist/bundling-sources/sdk/src/types.ts +29 -1
  10. package/dist/bundling-sources/shared_libs/integration-contracts/monitor-fleet-contract.ts +549 -0
  11. package/dist/bundling-sources/shared_libs/monitors/analytics-public.ts +10 -0
  12. package/dist/bundling-sources/shared_libs/monitors/fleets/errors.ts +96 -0
  13. package/dist/bundling-sources/shared_libs/monitors/fleets/limits.ts +8 -0
  14. package/dist/bundling-sources/shared_libs/monitors/fleets/plan.ts +320 -0
  15. package/dist/bundling-sources/shared_libs/monitors/fleets/verdict.ts +106 -0
  16. package/dist/bundling-sources/shared_libs/monitors/http-error-response.ts +8 -1
  17. package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +4 -4
  18. package/dist/bundling-sources/shared_libs/monitors/inventory-reconciliation.ts +59 -0
  19. package/dist/bundling-sources/shared_libs/monitors/lifecycle-notifications.ts +7 -9
  20. package/dist/bundling-sources/shared_libs/monitors/monitor-fleet-limits.ts +2 -12
  21. package/dist/bundling-sources/shared_libs/monitors/package.json +113 -4
  22. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +56 -9
  23. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-consumer-health.ts +2 -2
  24. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-identity-claims.ts +2 -2
  25. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-public-aliases.ts +1 -1
  26. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts +245 -0
  27. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-message.ts +38 -0
  28. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-ack-probe.ts +94 -0
  29. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-delivery.ts +60 -0
  30. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue.ts +60 -0
  31. package/dist/bundling-sources/shared_libs/monitors/snitcher-install-validation.ts +1 -1
  32. package/dist/bundling-sources/shared_libs/monitors/snitcher-ip2company.ts +2 -2
  33. package/dist/bundling-sources/shared_libs/monitors/snitcher-radar.ts +1 -1
  34. package/dist/bundling-sources/shared_libs/monitors/tracking-provider-integrations.ts +1 -1
  35. package/dist/bundling-sources/shared_libs/observability/build-context.ts +13 -0
  36. package/dist/bundling-sources/shared_libs/observability/log-metadata.ts +160 -0
  37. package/dist/bundling-sources/shared_libs/observability/request-context.ts +68 -0
  38. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +690 -0
  39. package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +35 -0
  40. package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +5 -0
  41. package/dist/bundling-sources/shared_libs/play-data-plane/neon-constants.ts +9 -0
  42. package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +272 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +46 -1
  44. package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +66 -1
  45. package/dist/bundling-sources/shared_libs/play-runtime/async-play-invocation.ts +0 -1
  46. package/dist/bundling-sources/shared_libs/play-runtime/bounceban-batching.ts +137 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +779 -140
  48. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +46 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/daytona-payload-contract.ts +68 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +103 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +725 -0
  52. package/dist/bundling-sources/shared_libs/play-runtime/db-session-lease-store.ts +578 -0
  53. package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +521 -0
  54. package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-receipt-gateway.ts +1378 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/runtime-topology.ts +1024 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-gateway-route-probe.ts +193 -0
  57. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machine-lifecycle.ts +104 -0
  58. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +1844 -0
  59. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +21 -0
  60. package/dist/bundling-sources/shared_libs/play-runtime/deployment/process-tree.ts +40 -0
  61. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +262 -0
  62. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-surface.ts +98 -0
  63. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +303 -0
  64. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-tuple-publication-guardian.ts +70 -0
  65. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-tuple-recovery.ts +519 -0
  66. package/dist/bundling-sources/shared_libs/play-runtime/deployment/vercel-fly-rollback-alignment.ts +280 -0
  67. package/dist/bundling-sources/shared_libs/play-runtime/deployment/vercel-production-gate.ts +159 -0
  68. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +29 -0
  69. package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +23 -0
  70. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +442 -0
  71. package/dist/bundling-sources/shared_libs/play-runtime/execution-protocol.ts +3 -3
  72. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +18 -0
  73. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +9 -0
  74. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +1 -1
  75. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +1 -1
  76. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +66 -0
  77. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +145 -0
  78. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/memory-store.ts +114 -0
  79. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/ports.ts +21 -0
  80. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/postgres-store.ts +170 -0
  81. package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +1322 -0
  82. package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +2 -2
  83. package/dist/bundling-sources/shared_libs/play-runtime/lane-retirement-gate.ts +1 -1
  84. package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +112 -0
  85. package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +8 -0
  86. package/dist/bundling-sources/shared_libs/play-runtime/perf-trace.ts +127 -0
  87. package/dist/bundling-sources/shared_libs/play-runtime/play-artifact-runtime-contract.ts +5 -0
  88. package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +24 -1
  89. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +56 -0
  90. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +13 -0
  91. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/job.ts +41 -0
  92. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/ports.ts +17 -0
  93. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +54 -0
  94. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/contracts.ts +114 -0
  95. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/engine-suspension.ts +108 -0
  96. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/index.ts +36 -0
  97. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/job.ts +97 -0
  98. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/ports.ts +104 -0
  99. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/postgres-fence-adapter.ts +263 -0
  100. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +64 -0
  101. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/contracts.ts +29 -0
  102. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +56 -0
  103. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/index.ts +15 -0
  104. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/job.ts +65 -0
  105. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/ports.ts +17 -0
  106. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts +42 -0
  107. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/index.ts +17 -0
  108. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +142 -0
  109. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/ports.ts +42 -0
  110. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/postgres-adapter.ts +102 -0
  111. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +60 -0
  112. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/index.ts +16 -0
  113. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +78 -0
  114. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +54 -0
  115. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +71 -0
  116. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +4 -0
  117. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +2 -0
  118. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +5 -8
  119. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +3 -5
  120. package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +168 -0
  121. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +44 -5
  122. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/child-play-resolver.ts +26 -0
  123. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +3662 -0
  124. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/progress-sidecar.ts +47 -0
  125. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +84 -0
  126. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-gateway-transport.ts +91 -0
  127. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-heartbeat.ts +424 -0
  128. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +321 -0
  129. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-sheet-persistence-retry.ts +110 -0
  130. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +625 -0
  131. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +72 -9
  132. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +6 -12
  133. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +19 -0
  134. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +0 -1
  135. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +14 -44
  136. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +103 -208
  137. package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts +201 -0
  138. package/dist/bundling-sources/shared_libs/play-runtime/runtime-env.ts +6 -0
  139. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +67 -0
  140. package/dist/bundling-sources/shared_libs/play-runtime/runtime-release-environment.ts +2 -0
  141. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-contract.ts +68 -0
  142. package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +14 -2
  143. package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +108 -0
  144. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +4 -0
  145. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +3 -0
  146. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
  147. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts +88 -0
  148. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +124 -0
  149. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-run-task.ts +68 -0
  150. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts +193 -0
  151. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts +269 -0
  152. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-suffix.ts +73 -0
  153. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +6642 -0
  154. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +897 -0
  155. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-metrics.ts +120 -0
  156. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +76 -0
  157. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +476 -0
  158. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +47 -0
  159. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +147 -0
  160. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/index.ts +27 -0
  161. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/lazy.ts +50 -0
  162. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/play-run-recovery.ts +98 -0
  163. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts +260 -0
  164. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +1416 -0
  165. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +1255 -0
  166. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +942 -0
  167. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +14636 -0
  168. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +888 -0
  169. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +355 -0
  170. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +46 -0
  171. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runner-push.ts +60 -0
  172. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-api-origin.ts +33 -0
  173. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-gateway-health.ts +64 -0
  174. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +835 -0
  175. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts +1218 -0
  176. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-operation.ts +207 -0
  177. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-scheduler-schema.ts +10 -0
  178. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts +123 -0
  179. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-launch-read-observability.ts +78 -0
  180. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +169 -0
  181. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/terminal-contract.ts +20 -0
  182. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-queue-metrics.ts +34 -0
  183. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +249 -0
  184. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +7 -0
  185. package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +296 -0
  186. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +3 -3
  187. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +20 -1
  188. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +60 -15
  189. package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +20 -0
  190. package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +12 -0
  191. package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +14 -1
  192. package/dist/bundling-sources/shared_libs/play-runtime/webhook-receipt-retry.ts +50 -0
  193. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +165 -0
  194. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +15 -23
  195. package/dist/bundling-sources/shared_libs/play-runtime/workflow-executor-auth.ts +169 -0
  196. package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +122 -0
  197. package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +2 -0
  198. package/dist/bundling-sources/shared_libs/plays/bundling/repository-adapter.ts +24 -0
  199. package/dist/bundling-sources/shared_libs/plays/child-run-projection.ts +174 -0
  200. package/dist/bundling-sources/shared_libs/plays/contracts.ts +2 -0
  201. package/dist/bundling-sources/shared_libs/plays/deploy/api-client.ts +267 -0
  202. package/dist/bundling-sources/shared_libs/plays/deploy/control-plane-types.ts +73 -0
  203. package/dist/bundling-sources/shared_libs/plays/deploy/manifest.ts +162 -0
  204. package/dist/bundling-sources/shared_libs/plays/deploy/project-serializer.ts +87 -0
  205. package/dist/bundling-sources/shared_libs/plays/deploy/source-file-path.ts +26 -0
  206. package/dist/bundling-sources/shared_libs/plays/deploy/target-paths.ts +36 -0
  207. package/dist/bundling-sources/shared_libs/plays/deployment-status.ts +1 -1
  208. package/dist/bundling-sources/shared_libs/plays/docflow.ts +1 -1
  209. package/dist/bundling-sources/shared_libs/plays/launch/input-contract.ts +400 -0
  210. package/dist/bundling-sources/shared_libs/plays/launch/prepare.ts +96 -0
  211. package/dist/bundling-sources/shared_libs/plays/local-file-discovery.ts +385 -0
  212. package/dist/bundling-sources/shared_libs/plays/play-grep-search.ts +112 -0
  213. package/dist/bundling-sources/shared_libs/plays/play-reference.ts +117 -0
  214. package/dist/bundling-sources/shared_libs/plays/run-input-summary.ts +260 -0
  215. package/dist/bundling-sources/shared_libs/plays/run-summary-types.ts +49 -0
  216. package/dist/bundling-sources/shared_libs/plays/secrets/errors.ts +19 -0
  217. package/dist/bundling-sources/shared_libs/plays/share/copy.ts +271 -0
  218. package/dist/bundling-sources/shared_libs/plays/share/metrics.ts +140 -0
  219. package/dist/bundling-sources/shared_libs/plays/share/public-read-model.ts +86 -0
  220. package/dist/bundling-sources/shared_libs/plays/share/seo-quality.ts +181 -0
  221. package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts +658 -0
  222. package/dist/bundling-sources/shared_libs/plays/share/types.ts +197 -0
  223. package/dist/bundling-sources/shared_libs/plays/sheet-filter.ts +347 -0
  224. package/dist/bundling-sources/shared_libs/plays/slack-hitl-timeout-error.ts +37 -0
  225. package/dist/bundling-sources/shared_libs/plays/standard-webhooks.ts +117 -0
  226. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +6 -1
  227. package/dist/bundling-sources/shared_libs/plays/status-result.ts +388 -0
  228. package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +1 -1
  229. package/dist/bundling-sources/shared_libs/plays/transform-factor-structure.ts +377 -0
  230. package/dist/bundling-sources/shared_libs/plays/webhook-payload.ts +113 -0
  231. package/dist/bundling-sources/shared_libs/product-notifications/configuration.ts +7 -1
  232. package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +3 -2
  233. package/dist/bundling-sources/shared_libs/product-notifications/overflow.ts +18 -0
  234. package/dist/bundling-sources/shared_libs/product-notifications/rolling-limit.ts +24 -0
  235. package/dist/bundling-sources/shared_libs/product-notifications/slack-delivery.ts +624 -0
  236. package/dist/bundling-sources/shared_libs/product-notifications/slack-payload.ts +140 -0
  237. package/dist/bundling-sources/shared_libs/product-notifications/slack-transport.ts +75 -0
  238. package/dist/bundling-sources/shared_libs/runtime-env.ts +6 -0
  239. package/dist/bundling-sources/shared_libs/security/email-blacklist.ts +42 -0
  240. package/dist/bundling-sources/shared_libs/security/header-safe-env.ts +10 -3
  241. package/dist/bundling-sources/shared_libs/security/slack-sanitize.ts +133 -0
  242. package/dist/cli/getting-started.d.mts +52 -0
  243. package/dist/cli/getting-started.d.ts +52 -0
  244. package/dist/cli/getting-started.js +65 -0
  245. package/dist/cli/getting-started.mjs +36 -0
  246. package/dist/cli/index.d.mts +9 -0
  247. package/dist/cli/index.d.ts +9 -0
  248. package/dist/cli/index.js +895 -407
  249. package/dist/cli/index.mjs +893 -414
  250. package/dist/{compiler-manifest-DsLbQaew.d.mts → compiler-manifest-x14KL83E.d.mts} +4 -2
  251. package/dist/{compiler-manifest-DsLbQaew.d.ts → compiler-manifest-x14KL83E.d.ts} +4 -2
  252. package/dist/index.d.mts +124 -154
  253. package/dist/index.d.ts +124 -154
  254. package/dist/index.js +132 -28
  255. package/dist/index.mjs +132 -28
  256. package/dist/install-integrity.json +182 -2
  257. package/dist/monitor-fleet-contract.d.mts +131 -0
  258. package/dist/monitor-fleet-contract.d.ts +131 -0
  259. package/dist/monitor-fleet-contract.js +406 -0
  260. package/dist/monitor-fleet-contract.mjs +367 -0
  261. package/dist/plays/bundle-play-file.d.mts +2 -2
  262. package/dist/plays/bundle-play-file.d.ts +2 -2
  263. package/dist/release.d.mts +255 -0
  264. package/dist/release.d.ts +255 -0
  265. package/dist/release.js +210 -0
  266. package/dist/release.mjs +183 -0
  267. package/package.json +21 -4
@@ -0,0 +1,3662 @@
1
+ import { appendFileSync, createReadStream, writeFileSync } from 'node:fs';
2
+ import { readFile } from 'node:fs/promises';
3
+ import { dirname, isAbsolute, resolve } from 'node:path';
4
+ import { createRequire } from 'node:module';
5
+ import { createHash } from 'node:crypto';
6
+ import { formatWithOptions } from 'node:util';
7
+ import { compileFunction } from 'node:vm';
8
+ import { gunzipSync } from 'node:zlib';
9
+ import { Readable } from 'node:stream';
10
+ import { parse } from 'csv-parse';
11
+ import { install as installSourceMapSupport } from 'source-map-support';
12
+ import {
13
+ createDeferredPlayDataset,
14
+ isPlayDataset,
15
+ isSerializedPlayDataset,
16
+ iteratePlayDatasetInputPages,
17
+ type PlayDataset,
18
+ } from '../../plays/dataset';
19
+ import type { PlayQueueHint } from '../governor/rate-state-backend';
20
+ import { AppRuntimeRateStateBackend } from '../governor/app-runtime-rate-state-backend';
21
+ import { AppRuntimeBudgetStateBackend } from '../governor/app-runtime-budget-state-backend';
22
+ import {
23
+ parseCtxFetchHttpFailureDiagnostic,
24
+ type CtxFetchHttpFailureDiagnostic,
25
+ } from '../log-provenance';
26
+ import type {
27
+ PlayRunnerEvent,
28
+ PlayRunnerExecutionConfig,
29
+ PlayRunOutputWarning,
30
+ PlayRunnerResult,
31
+ } from '../protocol';
32
+ import {
33
+ DURABLE_INVOCATION_FENCE_ATTESTATION_HEADER,
34
+ DURABLE_INVOCATION_FENCE_ATTESTATION_VALUE,
35
+ } from '../durable-invocation-fence';
36
+ import type { PlayRunInputPayload } from '../play-input';
37
+ import { playRunInputObjectView } from '../play-input';
38
+ import {
39
+ normalizePlayRunFailure,
40
+ runtimeLimitExceededCause,
41
+ runtimeLimitExceededFailure,
42
+ } from '../run-failure';
43
+ import { createPlayContext } from '../context';
44
+ import {
45
+ shouldEmitProgressSidecarEvent,
46
+ shouldEmitStdoutEvent,
47
+ } from './progress-sidecar';
48
+ import {
49
+ startRunnerHeartbeatLoop,
50
+ type RunnerHeartbeatStop,
51
+ } from './runtime-heartbeat';
52
+ import {
53
+ formatRunnerGatewayTransportSummary,
54
+ type RunnerGatewayTransportEvent,
55
+ } from './runtime-gateway-transport';
56
+ import { startConfiguredPlayRunnerRuntimeDeadline } from './runtime-deadline';
57
+ import {
58
+ createRunnerProgressPusher,
59
+ type RunnerProgressPusher,
60
+ } from './runtime-progress-push';
61
+ import {
62
+ pushRunnerTerminal,
63
+ runnerTerminalResultForTransport,
64
+ type PushRunnerTerminalOptions,
65
+ type PlayRunnerTerminalTransportResult,
66
+ } from './runtime-terminal-push';
67
+ import { vercelProtectionBypassHeader } from '../vercel-protection';
68
+ import { getPlayRuntimeBatchStrategy } from '../play-runtime-batching-registry';
69
+ import type { ToolResultMetadataInput } from '../tool-result';
70
+ import type {
71
+ ContextOptions,
72
+ IntegrationEventWaitBoundary,
73
+ IntegrationEventWaitHandler,
74
+ IntegrationEventWaitRuntimeContext,
75
+ PlayRowUpdate,
76
+ ResolvedPlayExecution,
77
+ RuntimeStepReceipt,
78
+ PlayExecutionEvent,
79
+ } from '../ctx-types';
80
+ import {
81
+ applyCsvRenameProjection,
82
+ type CsvRenameOptions,
83
+ } from '../csv-rename';
84
+ import { isPlayExecutionSuspendedError } from '../suspension';
85
+ import { assertNoSecretTaint } from '../secret-capability';
86
+ import { PLAY_ARTIFACT_KINDS } from '../backend';
87
+ import { assertNodeRunnableArtifact } from '../../plays/artifact-kind-guard';
88
+ import { createRuntimeDatasetId } from '../dataset-id';
89
+ import {
90
+ formatDatasetAvailabilityLog,
91
+ formatDatasetRegistrationLog,
92
+ } from '../dataset-lifecycle-log';
93
+ import {
94
+ assertCustomerPlayReturnWithinLimit,
95
+ RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES,
96
+ } from '../output-size-limits';
97
+ import {
98
+ resolveRuntimeReferencedPlay,
99
+ createCompleteRuntimeMapRowsOperation,
100
+ readRuntimeSheetDatasetRowKeys,
101
+ readRuntimeSheetDatasetRows,
102
+ releaseRuntimeWorkReceipts,
103
+ startRuntimeSheetDataset,
104
+ } from '../runtime-api';
105
+ import { installPgRuntimePoolDriver } from '../runtime-pg-driver-pg';
106
+ import { createRuntimeToolAuthScopeDigestResolver } from '../auth-scope-resolver';
107
+ import { resolveMapRowOutcomeKey } from '../map-row-outcome';
108
+ import { createRunScopedChildPlayResolver } from './child-play-resolver';
109
+ import { normalizePlayContractCompatibility } from '../../plays/contracts';
110
+ import {
111
+ SLACK_HITL_TIMEOUT_ERROR_CODE,
112
+ SlackHitlTimeoutError,
113
+ } from '../../plays/slack-hitl-timeout-error';
114
+ import { resolvePlayArtifactRuntimeContract } from '../play-artifact-runtime-contract';
115
+ import { validateOptionalPlayAuthoringField } from '../../plays/authoring-contract';
116
+ import { retryRuntimeSheetPersistence } from './runtime-sheet-persistence-retry';
117
+ import {
118
+ resolveMaxConcurrentExternalCalls,
119
+ resolveMaxConcurrentRows,
120
+ } from '../governor/policy';
121
+ import {
122
+ STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
123
+ validatePlaySandboxRuntimeLimits,
124
+ } from '../sandbox-runtime-limits';
125
+ import {
126
+ runtimeSheetPageTailHoldCompletedEvent,
127
+ runtimeSheetPageTailWriteMarkerEvent,
128
+ } from '../test-runtime-seams';
129
+
130
+ // Node-runtime play-runner uses the `pg` library for direct Postgres writes
131
+ // against per-run scoped Neon roles. Register the driver eagerly at module
132
+ // load so the first call into runtime-api.ts has a pool factory available.
133
+ installPgRuntimePoolDriver();
134
+
135
+ type SafeOutboundFetchModule =
136
+ typeof import('../../security/safe-outbound-fetch');
137
+ let safeOutboundFetchModule: Promise<SafeOutboundFetchModule> | null = null;
138
+
139
+ function loadSafeOutboundFetch(): Promise<SafeOutboundFetchModule> {
140
+ safeOutboundFetchModule ??= import('../../security/safe-outbound-fetch');
141
+ return safeOutboundFetchModule;
142
+ }
143
+
144
+ async function safeNodeOutboundFetch(
145
+ input: string | URL,
146
+ init?: RequestInit,
147
+ ): Promise<Response> {
148
+ const { safeOutboundFetch } = await loadSafeOutboundFetch();
149
+ return safeOutboundFetch(input, init);
150
+ }
151
+
152
+ async function safeNodeStreamingOutboundFetch(
153
+ input: string | URL,
154
+ init?: RequestInit,
155
+ ): Promise<Response> {
156
+ const { safeOutboundFetch } = await loadSafeOutboundFetch();
157
+ return safeOutboundFetch(input, init, { streamResponseBody: true });
158
+ }
159
+
160
+ import {
161
+ resolveSheetContractForTableNamespace,
162
+ sqlSafePlayColumnName,
163
+ type PlaySheetContract,
164
+ } from '../../plays/static-pipeline';
165
+ import { isIntegrationEventWaitToolId } from '../event-wait-tools';
166
+ import { type PlayBundleArtifact } from '../../plays/artifact-types';
167
+ import {
168
+ RuntimeReceiptStoreAdapter,
169
+ type RuntimeReceiptStoreHandlers,
170
+ } from '../runtime-receipt-store-adapter';
171
+ import {
172
+ applyRowUpdatesViaAppRuntime,
173
+ launchPlayAsyncViaAppRuntime,
174
+ observeDocflowNodeViaAppRuntime,
175
+ } from '../app-runtime-api';
176
+ import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
177
+ import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
178
+
179
+ const playRuntimeRequire = createRequire(`${process.cwd()}/package.json`);
180
+ const moduleRuntime = playRuntimeRequire('module') as {
181
+ Module: {
182
+ new (
183
+ id: string,
184
+ parent?: unknown,
185
+ ): {
186
+ exports: unknown;
187
+ filename: string;
188
+ paths: string[];
189
+ require: (id: string) => unknown;
190
+ };
191
+ _nodeModulePaths: (from: string) => string[];
192
+ };
193
+ };
194
+ let sourceMapSupportInstalled = false;
195
+ const runtimeSourceMaps = new Map<string, string>();
196
+ const runtimeSourceFiles = new Map<string, string>();
197
+ const runtimeArtifactReferences = new Map<
198
+ string,
199
+ { artifactHash: string; sourceMapHash: string; count: number }
200
+ >();
201
+ const CSV_NAMESPACE_PREFIX = 'csv';
202
+
203
+ function assertBundledRuntimeRequire(id: string): void {
204
+ if (
205
+ id === '.' ||
206
+ id.startsWith('./') ||
207
+ id.startsWith('../') ||
208
+ isAbsolute(id) ||
209
+ id.startsWith('file:')
210
+ ) {
211
+ throw new Error(
212
+ `Runtime require("${id}") cannot load a local file. Local play dependencies must be bundled before execution.`,
213
+ );
214
+ }
215
+ }
216
+
217
+ function createPlayRuntimeRequire(): ReturnType<typeof createRequire> {
218
+ const runtimeRequire = ((id: string) => {
219
+ assertBundledRuntimeRequire(id);
220
+ return playRuntimeRequire(id);
221
+ }) as ReturnType<typeof createRequire>;
222
+ runtimeRequire.resolve = Object.assign(
223
+ (id: string, options?: { paths?: string[] }) => {
224
+ assertBundledRuntimeRequire(id);
225
+ return playRuntimeRequire.resolve(id, options);
226
+ },
227
+ {
228
+ paths: (id: string) => {
229
+ assertBundledRuntimeRequire(id);
230
+ return playRuntimeRequire.resolve.paths(id);
231
+ },
232
+ },
233
+ );
234
+ runtimeRequire.main = playRuntimeRequire.main;
235
+ runtimeRequire.extensions = playRuntimeRequire.extensions;
236
+ runtimeRequire.cache = playRuntimeRequire.cache;
237
+ return runtimeRequire;
238
+ }
239
+ const WAIT_CONFIG_POLL_MS = 25;
240
+ const WAIT_CONFIG_TIMEOUT_MS = 120_000;
241
+ export const RUNNER_EXECUTION_ACK_TIMEOUT_MS = 75_000;
242
+ const RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS =
243
+ RUNTIME_RELIABILITY_POLICY.gateway.runnerControlMaxAttempts;
244
+ const RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS = [100, 250] as const;
245
+ export const RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS =
246
+ RUNTIME_RELIABILITY_POLICY.gateway.runnerControlRequestTimeoutMs;
247
+ const RUNTIME_RECEIPT_GATEWAY_BATCH_SIZE = 200;
248
+ const RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_WAIT_MS = 15;
249
+ const RUNTIME_RECEIPT_GATEWAY_REQUEST_TIMEOUT_MS =
250
+ RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs;
251
+ const RETRY_SAFE_EVENT_WAIT_ARM_TOOLS = new Set(['test_wait_for_event']);
252
+
253
+ // The one receipt backend: one Play Durability Store Adapter bound to the
254
+ // co-located receipt gateway. There is no host-injected or app-runtime path.
255
+ type RuntimeStepReceiptHandlers = RuntimeReceiptStoreHandlers;
256
+
257
+ type RuntimeToolMetadata = {
258
+ provider?: string | null;
259
+ operation?: string | null;
260
+ extractors?: ToolResultMetadataInput['extractors'];
261
+ actionCacheVersion: string;
262
+ actionCacheReadVersions?: string[];
263
+ queueHints?: unknown;
264
+ retry?: {
265
+ retrySafeTransientHttp?: boolean;
266
+ requiresExecutionFence?: boolean;
267
+ };
268
+ targetGetters?: Record<string, string[]>;
269
+ listExtractorPaths?: string[];
270
+ listIdentityGetters?: Record<string, string[]>;
271
+ eventWait?: boolean;
272
+ asyncOperation?:
273
+ | import('../async-operation').AsyncOperationContractWire
274
+ | null;
275
+ asyncOperationPoll?:
276
+ | import('../async-operation').AsyncOperationPollExecution
277
+ | null;
278
+ };
279
+
280
+ type RuntimeToolAuthScope = {
281
+ digest?: unknown;
282
+ rateScope?: {
283
+ bucketId?: unknown;
284
+ provider?: unknown;
285
+ token?: unknown;
286
+ };
287
+ };
288
+
289
+ export function assertRuntimeSheetAttemptPersistedTerminalRows(input: {
290
+ tableNamespace: string;
291
+ runId: string;
292
+ attemptId?: string | null;
293
+ updatedRows: number;
294
+ expectedRows: number;
295
+ persistedTerminalRows?: number | null;
296
+ fencedKeys?: string[];
297
+ }): void {
298
+ // Kept as a temporary source-compatible seam for callers compiled against
299
+ // the lease model. A partial update now means a newer scheduled write won;
300
+ // it is an observable stale drop, not a run-fatal ownership error.
301
+ void input;
302
+ }
303
+
304
+ export function deterministicRuntimeSheetAttemptId(input: {
305
+ playName: string;
306
+ runId: string;
307
+ tableNamespace: string;
308
+ attemptSeq: number;
309
+ }): string {
310
+ const digest = createHash('sha256')
311
+ .update(input.playName)
312
+ .update('\0')
313
+ .update(input.runId)
314
+ .update('\0')
315
+ .update(input.tableNamespace)
316
+ .update('\0')
317
+ .update(String(input.attemptSeq))
318
+ .digest('hex')
319
+ .slice(0, 32);
320
+ return `sheet-attempt:${digest}`;
321
+ }
322
+
323
+ async function vercelBypassHeaders(
324
+ config: PlayRunnerExecutionConfig,
325
+ ): Promise<Record<string, string>> {
326
+ return vercelProtectionBypassHeader(
327
+ config.context.vercelProtectionBypassToken,
328
+ );
329
+ }
330
+
331
+ function isRetryableRuntimeControlStatus(status: number): boolean {
332
+ return (
333
+ status === 408 ||
334
+ status === 429 ||
335
+ (status >= 520 && status <= 524) ||
336
+ status === 502 ||
337
+ status === 503 ||
338
+ status === 504
339
+ );
340
+ }
341
+
342
+ async function sleepRunnerRuntimeControlRetry(ms: number): Promise<void> {
343
+ await new Promise<void>((resolve) => setTimeout(resolve, ms));
344
+ }
345
+
346
+ function runnerRuntimeControlUrlHost(url: string): string {
347
+ try {
348
+ return new URL(url).host;
349
+ } catch {
350
+ return '<unparseable-url>';
351
+ }
352
+ }
353
+
354
+ /**
355
+ * Thrown when a sandbox -> receipt-gateway runtime-control call is redirected
356
+ * (any 3xx). This is a deterministic wiring failure, never a transient one, so
357
+ * it must escape the retry loop immediately without being re-wrapped as a
358
+ * transport error.
359
+ */
360
+ export class RunnerRuntimeControlRedirectError extends Error {
361
+ constructor(message: string) {
362
+ super(message);
363
+ this.name = 'RunnerRuntimeControlRedirectError';
364
+ }
365
+ }
366
+
367
+ export async function fetchRunnerRuntimeControl(input: {
368
+ url: string;
369
+ init: RequestInit;
370
+ description: string;
371
+ retrySafe: boolean;
372
+ /** Injectable fetch for unit tests; defaults to the global fetch. */
373
+ fetchImpl?: typeof fetch;
374
+ /** Injectable retry sleep for unit tests; defaults to a real timer. */
375
+ sleepImpl?: (ms: number) => Promise<void>;
376
+ /** Per-attempt transport deadline; injectable only to keep timeout tests fast. */
377
+ timeoutMs?: number;
378
+ }): Promise<Response> {
379
+ const fetchImpl = input.fetchImpl ?? fetch;
380
+ const sleepImpl = input.sleepImpl ?? sleepRunnerRuntimeControlRetry;
381
+ for (
382
+ let attempt = 1;
383
+ attempt <= RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS;
384
+ attempt += 1
385
+ ) {
386
+ const timeoutMs =
387
+ input.timeoutMs ?? RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS;
388
+ const timeoutSignal = AbortSignal.timeout(timeoutMs);
389
+ const signal = input.init.signal
390
+ ? AbortSignal.any([input.init.signal, timeoutSignal])
391
+ : timeoutSignal;
392
+ try {
393
+ // Never follow redirects on the sandbox -> receipt-gateway control plane.
394
+ // Every runtime-control call from the Daytona sandbox targets the gateway
395
+ // (context.baseUrl is rewritten to the gateway URL, and the sandbox holds
396
+ // NO Vercel protection-bypass credential). A 3xx therefore means the
397
+ // request escaped the gateway and hit Vercel Deployment Protection bare,
398
+ // whose SSO redirect drops our Authorization/bypass headers. Following it
399
+ // would swallow the failure into an opaque "307 Redirecting..." body from
400
+ // the wrong host. Surface it loudly with the target host instead.
401
+ const response = await fetchImpl(input.url, {
402
+ ...input.init,
403
+ redirect: 'manual',
404
+ signal,
405
+ });
406
+ if (response.status >= 300 && response.status < 400) {
407
+ const location = response.headers.get('location') ?? '<none>';
408
+ await response.text().catch(() => '');
409
+ throw new RunnerRuntimeControlRedirectError(
410
+ `${input.description} was redirected (${response.status}) by host ` +
411
+ `${runnerRuntimeControlUrlHost(input.url)} -> ${location}. Sandbox ` +
412
+ `runtime-control calls must terminate at the receipt gateway; a ` +
413
+ `redirect means the request reached Vercel Deployment Protection ` +
414
+ `without a bypass credential. Confirm context.baseUrl points at the ` +
415
+ `gateway and the launch carries vercelProtectionBypassToken.`,
416
+ );
417
+ }
418
+ if (
419
+ input.retrySafe &&
420
+ attempt < RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS &&
421
+ isRetryableRuntimeControlStatus(response.status)
422
+ ) {
423
+ await response.text().catch(() => '');
424
+ await sleepImpl(
425
+ RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS[attempt - 1] ?? 250,
426
+ );
427
+ continue;
428
+ }
429
+ // `fetch()` resolves once headers arrive. Leaving the body for callers
430
+ // means this attempt's timeout signal can fire during `response.json()`
431
+ // outside this retry loop. That turns a recoverable, partially delivered
432
+ // gateway response into a terminal child-run failure under a large HITL
433
+ // fanout. Buffer the deliberately small runtime-control response before
434
+ // returning it so one attempt owns both headers and body.
435
+ const body = await response.arrayBuffer();
436
+ return new Response(body, {
437
+ status: response.status,
438
+ statusText: response.statusText,
439
+ headers: response.headers,
440
+ });
441
+ } catch (error) {
442
+ // A redirect is a deterministic wiring failure, not a transient transport
443
+ // fault. Surface it immediately with its host-named message intact.
444
+ if (error instanceof RunnerRuntimeControlRedirectError) {
445
+ throw error;
446
+ }
447
+ const message = error instanceof Error ? error.message : String(error);
448
+ const transportMessage = timeoutSignal.aborted
449
+ ? `timed out after ${timeoutMs}ms`
450
+ : message;
451
+ if (!input.retrySafe || attempt >= RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS) {
452
+ throw new Error(
453
+ `${input.description} transport failed: ${transportMessage}`,
454
+ );
455
+ }
456
+ await sleepImpl(
457
+ RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS[attempt - 1] ?? 250,
458
+ );
459
+ }
460
+ }
461
+ throw new Error(`${input.description} transport failed.`);
462
+ }
463
+
464
+ function createRuntimeToolMetadataResolver(config: PlayRunnerExecutionConfig) {
465
+ const cached = new Map<string, Promise<RuntimeToolMetadata>>();
466
+
467
+ return async (toolId: string): Promise<RuntimeToolMetadata> => {
468
+ const normalizedToolId = String(toolId || '').trim();
469
+ if (
470
+ !normalizedToolId ||
471
+ !config.context.executorToken ||
472
+ !config.context.baseUrl
473
+ ) {
474
+ throw new Error(
475
+ `Runtime metadata lookup for tool ${normalizedToolId || '<empty>'} requires executorToken and baseUrl.`,
476
+ );
477
+ }
478
+
479
+ const cachedMetadata = cached.get(normalizedToolId);
480
+ if (cachedMetadata) {
481
+ return await cachedMetadata;
482
+ }
483
+
484
+ const promise = (async () => {
485
+ const url = `${config.context.baseUrl}/api/v2/plays/runtime-tools/${encodeURIComponent(normalizedToolId)}`;
486
+ const response = await fetchRunnerRuntimeControl({
487
+ url,
488
+ description: `Runtime metadata lookup for tool ${normalizedToolId}`,
489
+ retrySafe: true,
490
+ init: {
491
+ method: 'GET',
492
+ headers: {
493
+ Authorization: `Bearer ${config.context.executorToken}`,
494
+ ...(await vercelBypassHeaders(config)),
495
+ },
496
+ },
497
+ });
498
+ if (!response.ok) {
499
+ throw new Error(
500
+ `Runtime metadata lookup for tool ${normalizedToolId} failed (${response.status}): ${await response.text()}`,
501
+ );
502
+ }
503
+ return (await response.json()) as RuntimeToolMetadata;
504
+ })();
505
+
506
+ cached.set(normalizedToolId, promise);
507
+ return await promise;
508
+ };
509
+ }
510
+
511
+ function createAppRuntimeToolAuthScopeDigestResolver(
512
+ config: PlayRunnerExecutionConfig,
513
+ ) {
514
+ const rateScopes = new Map<
515
+ string,
516
+ { bucketId: string; provider: string; token: string }
517
+ >();
518
+ const digestResolver = createRuntimeToolAuthScopeDigestResolver({
519
+ missingToolIdMessage: 'Runtime auth scope lookup requires tool id.',
520
+ retryDelaysMs: [],
521
+ fetchDigest: async (normalizedToolId) => {
522
+ if (!config.context.executorToken || !config.context.baseUrl) {
523
+ throw new Error(
524
+ `Runtime auth scope lookup for tool ${normalizedToolId || '<empty>'} requires executorToken and baseUrl.`,
525
+ );
526
+ }
527
+ const url = `${config.context.baseUrl}/api/v2/plays/runtime-tools/${encodeURIComponent(normalizedToolId)}/auth-scope`;
528
+ const response = await fetchRunnerRuntimeControl({
529
+ url,
530
+ description: `Runtime auth scope lookup for tool ${normalizedToolId}`,
531
+ retrySafe: true,
532
+ init: {
533
+ method: 'GET',
534
+ headers: {
535
+ Authorization: `Bearer ${config.context.executorToken}`,
536
+ ...(await vercelBypassHeaders(config)),
537
+ },
538
+ },
539
+ });
540
+ if (!response.ok) {
541
+ throw new Error(
542
+ `Runtime auth scope lookup for tool ${normalizedToolId} failed (${response.status}): ${await response.text()}`,
543
+ );
544
+ }
545
+ const body = (await response.json()) as RuntimeToolAuthScope;
546
+ const digest = typeof body.digest === 'string' ? body.digest.trim() : '';
547
+ if (!digest) {
548
+ throw new Error(
549
+ `Runtime auth scope lookup for tool ${normalizedToolId} returned no digest.`,
550
+ );
551
+ }
552
+ const rateScope = body.rateScope;
553
+ const bucketId =
554
+ typeof rateScope?.bucketId === 'string'
555
+ ? rateScope.bucketId.trim()
556
+ : '';
557
+ const provider =
558
+ typeof rateScope?.provider === 'string'
559
+ ? rateScope.provider.trim()
560
+ : '';
561
+ const token =
562
+ typeof rateScope?.token === 'string' ? rateScope.token.trim() : '';
563
+ if (!bucketId || !provider || !token) {
564
+ throw new Error(
565
+ `Runtime auth scope lookup for tool ${normalizedToolId} returned no rate scope.`,
566
+ );
567
+ }
568
+ rateScopes.set(normalizedToolId, { bucketId, provider, token });
569
+ return digest;
570
+ },
571
+ });
572
+ const invalidateDigest = digestResolver.invalidate;
573
+ return Object.assign(digestResolver, {
574
+ invalidate: (toolId?: string | null) => {
575
+ invalidateDigest(toolId);
576
+ const normalized = String(toolId || '').trim();
577
+ if (normalized) rateScopes.delete(normalized);
578
+ else rateScopes.clear();
579
+ },
580
+ rateScope: async (toolId: string, expectedProvider: string) => {
581
+ await digestResolver(toolId);
582
+ const scope = rateScopes.get(toolId.trim());
583
+ if (!scope || scope.provider !== expectedProvider) {
584
+ throw new Error(
585
+ `Runtime rate scope provider mismatch for tool ${toolId}.`,
586
+ );
587
+ }
588
+ return { bucketId: scope.bucketId, token: scope.token };
589
+ },
590
+ });
591
+ }
592
+
593
+ function createRateStateBackendForConfig(
594
+ config: PlayRunnerExecutionConfig,
595
+ ): Pick<
596
+ ContextOptions,
597
+ 'rateState' | 'budgetState' | 'requireSharedRateState'
598
+ > {
599
+ const backend = config.context.rateStateBackend;
600
+ if (!backend || backend.kind === 'in_memory') return {};
601
+ const baseUrl = config.context.baseUrl?.trim();
602
+ const executorToken = config.context.executorToken?.trim();
603
+ if (!baseUrl || !executorToken) {
604
+ throw new Error(
605
+ 'Node play runtime requires baseUrl and executorToken for shared rate state.',
606
+ );
607
+ }
608
+ const runtimeApi = {
609
+ baseUrl,
610
+ executorToken,
611
+ vercelProtectionBypassToken:
612
+ config.context.vercelProtectionBypassToken ?? null,
613
+ integrationMode: config.context.integrationMode,
614
+ };
615
+ return {
616
+ requireSharedRateState: true,
617
+ rateState: new AppRuntimeRateStateBackend(runtimeApi, {
618
+ schedulerSchema: backend.schedulerSchema ?? null,
619
+ }),
620
+ budgetState: new AppRuntimeBudgetStateBackend(
621
+ runtimeApi,
622
+ backend.schedulerSchema ?? null,
623
+ ),
624
+ };
625
+ }
626
+
627
+ export function createRuntimeEventWaitHandler(
628
+ config: PlayRunnerExecutionConfig,
629
+ toolId: string,
630
+ ): IntegrationEventWaitHandler | null {
631
+ const normalizedToolId = String(toolId || '').trim();
632
+ if (
633
+ !normalizedToolId ||
634
+ !config.context.executorToken ||
635
+ !config.context.baseUrl
636
+ ) {
637
+ return null;
638
+ }
639
+
640
+ async function callRuntimeEventWaitEndpoint(
641
+ input:
642
+ | {
643
+ phase: 'prepare';
644
+ payload: Record<string, unknown>;
645
+ context: IntegrationEventWaitRuntimeContext;
646
+ }
647
+ | {
648
+ phase: 'arm';
649
+ payload: Record<string, unknown>;
650
+ context: IntegrationEventWaitRuntimeContext;
651
+ boundary: IntegrationEventWaitBoundary;
652
+ }
653
+ | {
654
+ phase: 'timeout';
655
+ payload: Record<string, unknown>;
656
+ context: IntegrationEventWaitRuntimeContext;
657
+ boundary: IntegrationEventWaitBoundary;
658
+ },
659
+ ): Promise<IntegrationEventWaitBoundary> {
660
+ const url = `${config.context.baseUrl}/api/v2/plays/runtime-tools/${encodeURIComponent(normalizedToolId)}/event-wait`;
661
+ const response = await fetchRunnerRuntimeControl({
662
+ url,
663
+ description: `Event wait runtime call for tool ${normalizedToolId}`,
664
+ retrySafe:
665
+ input.phase === 'prepare' ||
666
+ RETRY_SAFE_EVENT_WAIT_ARM_TOOLS.has(normalizedToolId),
667
+ init: {
668
+ method: 'POST',
669
+ headers: {
670
+ 'Content-Type': 'application/json',
671
+ Authorization: `Bearer ${config.context.executorToken}`,
672
+ ...(await vercelBypassHeaders(config)),
673
+ },
674
+ body: JSON.stringify(input),
675
+ },
676
+ });
677
+ if (response.status === 404) {
678
+ throw new Error(`No event wait handler for tool ${normalizedToolId}.`);
679
+ }
680
+ if (!response.ok) {
681
+ const bodyText = await response.text();
682
+ const timeoutError = runtimeControlSlackHitlTimeoutError({
683
+ toolId: normalizedToolId,
684
+ phase: input.phase,
685
+ status: response.status,
686
+ bodyText,
687
+ });
688
+ if (timeoutError) throw timeoutError;
689
+ throw new Error(
690
+ formatRuntimeControlFailure({
691
+ description: `Event wait runtime call for tool ${normalizedToolId}`,
692
+ status: response.status,
693
+ bodyText,
694
+ }),
695
+ );
696
+ }
697
+ return (await response.json()) as IntegrationEventWaitBoundary;
698
+ }
699
+
700
+ const handler: IntegrationEventWaitHandler = {
701
+ provider: 'runtime',
702
+ toolId: normalizedToolId,
703
+ prepare: async ({ payload, context }) => {
704
+ const boundary = await callRuntimeEventWaitEndpoint({
705
+ phase: 'prepare',
706
+ payload,
707
+ context,
708
+ });
709
+ // The app owns provider-specific timeout semantics. An older app omits
710
+ // this additive field, retaining its legacy replay behavior safely.
711
+ handler.suspendAfterTimeoutMs =
712
+ boundary.timeoutPolicy?.suspendAfterTimeoutMs;
713
+ return boundary;
714
+ },
715
+ arm: async ({ payload, context, boundary }) =>
716
+ await callRuntimeEventWaitEndpoint({
717
+ phase: 'arm',
718
+ payload,
719
+ context,
720
+ boundary,
721
+ }),
722
+ onTimeout: async ({ payload, context, boundary }) => {
723
+ // Only a newer app advertises this phase. Do not send a new request to an
724
+ // older app during the rollout-skew window.
725
+ if (boundary.timeoutPolicy?.finalizeOnTimeout !== true) return;
726
+ await callRuntimeEventWaitEndpoint({
727
+ phase: 'timeout',
728
+ payload,
729
+ context,
730
+ boundary,
731
+ });
732
+ },
733
+ };
734
+ return handler;
735
+ }
736
+
737
+ export function formatRuntimeControlFailure(input: {
738
+ description: string;
739
+ status: number;
740
+ bodyText: string;
741
+ }): string {
742
+ let message = input.bodyText;
743
+ try {
744
+ const parsed = JSON.parse(input.bodyText) as Record<string, unknown>;
745
+ const code =
746
+ typeof parsed.code === 'string' && parsed.code.trim()
747
+ ? parsed.code.trim()
748
+ : null;
749
+ const error =
750
+ typeof parsed.error === 'string' && parsed.error.trim()
751
+ ? parsed.error.trim()
752
+ : typeof parsed.message === 'string' && parsed.message.trim()
753
+ ? parsed.message.trim()
754
+ : null;
755
+ const operatorHint =
756
+ typeof parsed.operator_hint === 'string' && parsed.operator_hint.trim()
757
+ ? parsed.operator_hint.trim()
758
+ : null;
759
+ if (code && error) {
760
+ message = [code, error, operatorHint].filter(Boolean).join(': ');
761
+ }
762
+ } catch {
763
+ // Preserve the original runtime-control response for unstructured errors.
764
+ }
765
+ return `${input.description} failed (${input.status}): ${message}`;
766
+ }
767
+
768
+ function runtimeControlSlackHitlTimeoutError(input: {
769
+ toolId: string;
770
+ phase: 'prepare' | 'arm' | 'timeout';
771
+ status: number;
772
+ bodyText: string;
773
+ }): SlackHitlTimeoutError | null {
774
+ if (
775
+ input.toolId !== 'slack_message_with_hitl' ||
776
+ input.phase !== 'timeout' ||
777
+ input.status !== 408
778
+ ) {
779
+ return null;
780
+ }
781
+ try {
782
+ const body = JSON.parse(input.bodyText) as Record<string, unknown>;
783
+ if (body.code !== SLACK_HITL_TIMEOUT_ERROR_CODE) return null;
784
+ const message =
785
+ typeof body.message === 'string' && body.message.trim()
786
+ ? body.message.trim()
787
+ : typeof body.error === 'string' && body.error.trim()
788
+ ? body.error.trim()
789
+ : undefined;
790
+ return new SlackHitlTimeoutError(
791
+ message?.replace(
792
+ new RegExp(`^${SLACK_HITL_TIMEOUT_ERROR_CODE}:\\s*`),
793
+ '',
794
+ ),
795
+ );
796
+ } catch {
797
+ return null;
798
+ }
799
+ }
800
+
801
+ function inferCsvSheetContract(input: {
802
+ tableNamespace: string;
803
+ rows: readonly Record<string, unknown>[];
804
+ }): PlaySheetContract {
805
+ const seen = new Set<string>();
806
+ const columns = input.rows.flatMap((row) =>
807
+ Object.keys(row).flatMap((field) => {
808
+ const normalizedField = field.trim();
809
+ if (!normalizedField || seen.has(normalizedField)) {
810
+ return [];
811
+ }
812
+ seen.add(normalizedField);
813
+ return [
814
+ {
815
+ id: normalizedField,
816
+ sqlName: sqlSafePlayColumnName(normalizedField),
817
+ source: 'input' as const,
818
+ field: normalizedField,
819
+ },
820
+ ];
821
+ }),
822
+ );
823
+ return {
824
+ tableNamespace: input.tableNamespace,
825
+ columns,
826
+ };
827
+ }
828
+
829
+ type ReplayableCsvSource = {
830
+ iterate: () => AsyncIterable<Record<string, unknown>>;
831
+ };
832
+
833
+ async function createStreamingCsvDataset(input: {
834
+ playName: string;
835
+ tableNamespace: string;
836
+ source: ReplayableCsvSource;
837
+ sourceLabel: string;
838
+ options?: CsvRenameOptions;
839
+ }): Promise<PlayDataset<Record<string, unknown>>> {
840
+ let cachedCount: number | null = null;
841
+ const iterateProjected = () =>
842
+ ({
843
+ async *[Symbol.asyncIterator]() {
844
+ for await (const row of input.source.iterate()) {
845
+ yield applyCsvRenameProjection([row], input.options)[0]!;
846
+ }
847
+ },
848
+ }) as AsyncIterable<Record<string, unknown>>;
849
+ const collect = async (limit?: number) => {
850
+ const rows: Record<string, unknown>[] = [];
851
+ if (limit !== undefined && limit <= 0) return rows;
852
+ for await (const row of iterateProjected()) {
853
+ rows.push(row);
854
+ if (limit !== undefined && rows.length >= limit) break;
855
+ }
856
+ return rows;
857
+ };
858
+ const previewRows = await collect(10);
859
+ return createDeferredPlayDataset({
860
+ datasetKind: 'csv',
861
+ datasetId: createRuntimeDatasetId(input.playName, input.tableNamespace),
862
+ count: 0,
863
+ knownCount: null,
864
+ previewRows,
865
+ sourceLabel: input.sourceLabel,
866
+ tableNamespace: input.tableNamespace,
867
+ resolvers: {
868
+ count: async () => {
869
+ if (cachedCount !== null) return cachedCount;
870
+ let count = 0;
871
+ for await (const _row of iterateProjected()) count += 1;
872
+ cachedCount = count;
873
+ return count;
874
+ },
875
+ peek: async (limit) => await collect(Math.max(0, limit)),
876
+ materialize: collect,
877
+ iterate: iterateProjected,
878
+ },
879
+ });
880
+ }
881
+
882
+ export interface ExecutePlayRunnerOptions {
883
+ emitEvent?: (event: PlayRunnerEvent) => void;
884
+ signal?: AbortSignal;
885
+ }
886
+
887
+ function emitRunnerEvent(
888
+ event: PlayRunnerEvent,
889
+ options?: ExecutePlayRunnerOptions,
890
+ ): void {
891
+ if (options?.emitEvent) {
892
+ options.emitEvent(event);
893
+ return;
894
+ }
895
+ emit(event);
896
+ }
897
+
898
+ /**
899
+ * In-flight durable docflow observation posts (ADR 0016 rule 2). The wrapper
900
+ * fires them fire-and-forget, but a detached push run's terminal WAKES the
901
+ * parked worker, whose finalize immediately reads the observations table — so
902
+ * a post still in flight over the tunnel loses the race and the terminal
903
+ * ledger misses its nodeIo. Before the graceful terminal emit the runner
904
+ * drains these with a hard cap: the terminal is never failed or delayed
905
+ * beyond the cap, and hard-kill paths (runtime deadline, heartbeat stop,
906
+ * crash epilogue) never wait at all. Empty for undiagrammed plays.
907
+ */
908
+ const pendingDocflowObservationPosts = new Set<Promise<unknown>>();
909
+ const DOCFLOW_OBSERVATION_TERMINAL_DRAIN_CAP_MS = 2_000;
910
+
911
+ async function drainPendingDocflowObservationPosts(
912
+ options?: ExecutePlayRunnerOptions,
913
+ ): Promise<void> {
914
+ if (pendingDocflowObservationPosts.size === 0) {
915
+ return;
916
+ }
917
+ const pendingCount = pendingDocflowObservationPosts.size;
918
+ let capTimer: ReturnType<typeof setTimeout> | null = null;
919
+ const capped = await Promise.race([
920
+ Promise.allSettled([...pendingDocflowObservationPosts]).then(
921
+ () => 'drained' as const,
922
+ ),
923
+ new Promise<'capped'>((resolve) => {
924
+ capTimer = setTimeout(
925
+ () => resolve('capped'),
926
+ DOCFLOW_OBSERVATION_TERMINAL_DRAIN_CAP_MS,
927
+ );
928
+ }),
929
+ ]).catch(() => 'drained' as const);
930
+ if (capTimer) clearTimeout(capTimer);
931
+ if (capped === 'capped') {
932
+ emitWorkerLog(
933
+ `[warn] docflow observation drain hit the ${DOCFLOW_OBSERVATION_TERMINAL_DRAIN_CAP_MS}ms terminal cap with ${pendingDocflowObservationPosts.size}/${pendingCount} posts still in flight; the worker-side retry fold covers the stragglers`,
934
+ options,
935
+ );
936
+ }
937
+ }
938
+
939
+ function emitWorkerLog(line: string, options?: ExecutePlayRunnerOptions): void {
940
+ emitRunnerEvent(
941
+ {
942
+ type: 'log',
943
+ at: new Date().toISOString(),
944
+ source: 'worker',
945
+ line,
946
+ },
947
+ options,
948
+ );
949
+ }
950
+
951
+ function emitSystemLog(line: string, options?: ExecutePlayRunnerOptions): void {
952
+ emitRunnerEvent(
953
+ {
954
+ type: 'log',
955
+ at: new Date().toISOString(),
956
+ source: 'system',
957
+ line,
958
+ },
959
+ options,
960
+ );
961
+ }
962
+
963
+ export function playRunnerContextCachePolicy(
964
+ context: PlayRunnerExecutionConfig['context'],
965
+ ): ContextOptions['cachePolicy'] | undefined {
966
+ if (context.forceToolRefresh) {
967
+ return {
968
+ ...(context.force ? { forceStepRefresh: true } : {}),
969
+ forceToolRefresh: true,
970
+ forceFailedToolRefresh: true,
971
+ };
972
+ }
973
+ if (context.force) {
974
+ return { forceStepRefresh: true, forceFailedToolRefresh: true };
975
+ }
976
+ return undefined;
977
+ }
978
+
979
+ export function playRunnerContextRuntimeSheetForce(
980
+ context: PlayRunnerExecutionConfig['context'],
981
+ ): boolean {
982
+ return context.force === true || context.forceToolRefresh === true;
983
+ }
984
+
985
+ export function playRunnerInvocationFenceFlags(
986
+ context: PlayRunnerExecutionConfig['context'],
987
+ ): Pick<
988
+ ContextOptions,
989
+ 'requestDurableInvocationFence' | 'durableInvocationFence'
990
+ > {
991
+ return {
992
+ requestDurableInvocationFence:
993
+ context.requestDurableInvocationFence === true ||
994
+ (Boolean(context.receiptGatewayBaseUrl?.trim()) &&
995
+ context.baseUrl?.replace(/\/$/, '') ===
996
+ context.receiptGatewayBaseUrl?.trim().replace(/\/$/, '')),
997
+ durableInvocationFence: context.durableInvocationFence === true,
998
+ };
999
+ }
1000
+
1001
+ export function playRunnerRuntimeReceiptsEnabled(
1002
+ context: PlayRunnerExecutionConfig['context'],
1003
+ ): boolean {
1004
+ return Boolean(
1005
+ context.orgId &&
1006
+ context.playName &&
1007
+ context.baseUrl &&
1008
+ context.executorToken &&
1009
+ context.baseUrl.trim(),
1010
+ );
1011
+ }
1012
+
1013
+ /**
1014
+ * Runtime Sheets belong to an organization and Play, not to the human who
1015
+ * initiated the run. Triggered runs intentionally have no actor email.
1016
+ */
1017
+ export function assertPlayRunnerRuntimeSheetPersistenceContext(
1018
+ context: PlayRunnerExecutionConfig['context'],
1019
+ ): asserts context is PlayRunnerExecutionConfig['context'] & {
1020
+ orgId: string;
1021
+ playName: string;
1022
+ baseUrl: string;
1023
+ executorToken: string;
1024
+ } {
1025
+ const missing = [
1026
+ !context.orgId?.trim() ? 'organization id' : null,
1027
+ !context.playName?.trim() ? 'Play name' : null,
1028
+ !(context.runId?.trim() || context.workflowId?.trim()) ? 'run id' : null,
1029
+ !context.baseUrl?.trim() ? 'runtime API origin' : null,
1030
+ !context.executorToken?.trim() ? 'executor token' : null,
1031
+ ].filter((field): field is string => field !== null);
1032
+ if (missing.length > 0) {
1033
+ throw new Error(
1034
+ `Runtime Sheet persistence is required for ctx.dataset() and ctx.csv(); run launch is missing ${missing.join(', ')}.`,
1035
+ );
1036
+ }
1037
+ }
1038
+
1039
+ function collectReturnedDatasets(value: unknown): unknown[] {
1040
+ const datasets: unknown[] = [];
1041
+ const seen = new Set<unknown>();
1042
+
1043
+ const visit = (entry: unknown) => {
1044
+ if (!entry || typeof entry !== 'object' || seen.has(entry)) {
1045
+ return;
1046
+ }
1047
+ seen.add(entry);
1048
+ if (isPlayDataset(entry)) {
1049
+ datasets.push(entry);
1050
+ return;
1051
+ }
1052
+ if (Array.isArray(entry)) {
1053
+ for (const item of entry) {
1054
+ visit(item);
1055
+ }
1056
+ return;
1057
+ }
1058
+ for (const child of Object.values(entry as Record<string, unknown>)) {
1059
+ visit(child);
1060
+ }
1061
+ };
1062
+
1063
+ visit(value);
1064
+ return datasets;
1065
+ }
1066
+
1067
+ // Absorb ENOBUFS / EPIPE on stdout — the parent reads stdout chunks anyway,
1068
+ // and crashing the runner process on backpressure loses the entire run.
1069
+ process.stdout.on('error', (err: NodeJS.ErrnoException) => {
1070
+ if (err.code === 'ENOBUFS' || err.code === 'EPIPE') return;
1071
+ throw err;
1072
+ });
1073
+
1074
+ // Push-execution live progress channel. When the launch enables push execution
1075
+ // (Daytona sandbox runner), progress events are PUSHED to the receipt gateway
1076
+ // instead of polled out of a sidecar file by the worker. Set once in main().
1077
+ let runnerProgressPusher: RunnerProgressPusher | null = null;
1078
+
1079
+ function emit(event: PlayRunnerEvent): void {
1080
+ emitProgressSidecarEvent(event);
1081
+ if (runnerProgressPusher && shouldEmitProgressSidecarEvent(event)) {
1082
+ runnerProgressPusher.enqueue(event);
1083
+ }
1084
+ if (!shouldEmitStdoutEvent(event)) {
1085
+ return;
1086
+ }
1087
+ try {
1088
+ process.stdout.write(`${JSON.stringify(event)}\n`);
1089
+ } catch (err: unknown) {
1090
+ const code = (err as NodeJS.ErrnoException).code;
1091
+ if (code === 'ENOBUFS' || code === 'EPIPE') return;
1092
+ throw err;
1093
+ }
1094
+ }
1095
+
1096
+ const PROGRESS_SIDECAR_SEGMENT_BYTES = 256 * 1024;
1097
+ let progressSidecarState: {
1098
+ basePath: string;
1099
+ segment: number;
1100
+ segmentBytes: number;
1101
+ } | null = null;
1102
+
1103
+ function emitProgressSidecarEvent(event: PlayRunnerEvent): void {
1104
+ if (!shouldEmitProgressSidecarEvent(event)) {
1105
+ return;
1106
+ }
1107
+ const progressPath =
1108
+ process.env.DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH?.trim();
1109
+ if (!progressPath) {
1110
+ return;
1111
+ }
1112
+ try {
1113
+ const line = `${JSON.stringify(event)}\n`;
1114
+ const lineBytes = Buffer.byteLength(line);
1115
+ if (
1116
+ !progressSidecarState ||
1117
+ progressSidecarState.basePath !== progressPath
1118
+ ) {
1119
+ progressSidecarState = {
1120
+ basePath: progressPath,
1121
+ segment: 0,
1122
+ segmentBytes: 0,
1123
+ };
1124
+ writeFileSync(progressPath, '0', 'utf-8');
1125
+ }
1126
+ if (
1127
+ progressSidecarState.segmentBytes > 0 &&
1128
+ progressSidecarState.segmentBytes + lineBytes >
1129
+ PROGRESS_SIDECAR_SEGMENT_BYTES
1130
+ ) {
1131
+ progressSidecarState.segment += 1;
1132
+ progressSidecarState.segmentBytes = 0;
1133
+ writeFileSync(
1134
+ progressPath,
1135
+ String(progressSidecarState.segment),
1136
+ 'utf-8',
1137
+ );
1138
+ }
1139
+ const segmentPath = `${progressPath}.${String(
1140
+ progressSidecarState.segment,
1141
+ ).padStart(6, '0')}`;
1142
+ appendFileSync(segmentPath, line, 'utf-8');
1143
+ progressSidecarState.segmentBytes += lineBytes;
1144
+ } catch {
1145
+ // Best-effort live observability. The normal stdout result stream remains
1146
+ // authoritative, so a sidecar write failure must not fail the play.
1147
+ }
1148
+ }
1149
+
1150
+ async function writeRunnerEventToStdout(
1151
+ event:
1152
+ | PlayRunnerEvent
1153
+ | { type: 'result'; result: PlayRunnerTerminalTransportResult },
1154
+ ): Promise<void> {
1155
+ await new Promise<void>((resolve, reject) => {
1156
+ process.stdout.write(`${JSON.stringify(event)}\n`, (error) => {
1157
+ if (error) {
1158
+ reject(error);
1159
+ return;
1160
+ }
1161
+ resolve();
1162
+ });
1163
+ });
1164
+ }
1165
+
1166
+ function truncateUtf8Prefix(value: string, maxBytes: number): string {
1167
+ if (Buffer.byteLength(value, 'utf8') <= maxBytes) return value;
1168
+ const encoded = Buffer.from(value, 'utf8');
1169
+ for (let end = Math.min(encoded.length, maxBytes); end > 0; end -= 1) {
1170
+ const candidate = encoded.subarray(0, end).toString('utf8');
1171
+ if (Buffer.byteLength(candidate, 'utf8') <= maxBytes) return candidate;
1172
+ }
1173
+ return '';
1174
+ }
1175
+
1176
+ /**
1177
+ * The Daytona crash pusher reads a bounded tail of merged stdout/stderr. Once
1178
+ * its authoritative `result` event is written, no runner-owned diagnostic may
1179
+ * push that event out of the tail. The normal progress pusher is stopped before
1180
+ * the result; this bounds only terminal-push retry diagnostics that follow it.
1181
+ */
1182
+ function createPostTerminalDiagnosticWriter(): NonNullable<
1183
+ PushRunnerTerminalOptions['warn']
1184
+ > {
1185
+ let remaining = RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES;
1186
+ return (message, detail) => {
1187
+ if (remaining <= 1) return;
1188
+ const prefix = `[play-runner.terminal-push] ${message} `;
1189
+ const prefixBytes = Buffer.byteLength(prefix, 'utf8');
1190
+ if (remaining <= prefixBytes + 1) return;
1191
+ const formatted = formatWithOptions(
1192
+ {
1193
+ breakLength: Infinity,
1194
+ colors: false,
1195
+ compact: true,
1196
+ depth: 4,
1197
+ maxArrayLength: 20,
1198
+ maxStringLength: 4_096,
1199
+ },
1200
+ detail,
1201
+ );
1202
+ const suffix = truncateUtf8Prefix(formatted, remaining - prefixBytes - 1);
1203
+ const line = `${prefix}${suffix}\n`;
1204
+ const bytes = Buffer.byteLength(line, 'utf8');
1205
+ if (bytes > remaining) return;
1206
+ remaining -= bytes;
1207
+ try {
1208
+ process.stderr.write(line);
1209
+ } catch (error: unknown) {
1210
+ const code = (error as NodeJS.ErrnoException).code;
1211
+ if (code !== 'ENOBUFS' && code !== 'EPIPE') throw error;
1212
+ }
1213
+ };
1214
+ }
1215
+
1216
+ async function exitRunner(exitCode: number): Promise<never> {
1217
+ await Promise.race([
1218
+ Promise.resolve(),
1219
+ new Promise<void>((resolve) => setTimeout(resolve, 1_000)),
1220
+ ]);
1221
+
1222
+ process.exit(exitCode);
1223
+ }
1224
+
1225
+ async function emitAndExit(
1226
+ event:
1227
+ | PlayRunnerEvent
1228
+ | { type: 'result'; result: PlayRunnerTerminalTransportResult },
1229
+ exitCode: number,
1230
+ ): Promise<never> {
1231
+ await writeRunnerEventToStdout(event);
1232
+ return exitRunner(exitCode);
1233
+ }
1234
+
1235
+ function parseCsvStream(
1236
+ source: NodeJS.ReadableStream,
1237
+ ): AsyncIterable<Record<string, unknown>> {
1238
+ const parser = parse({ columns: true, skip_empty_lines: true, trim: true });
1239
+ // `Readable.pipe()` does not forward source errors to the destination. A
1240
+ // missing staged file therefore used to emit an unhandled `error` from
1241
+ // createReadStream and kill the Daytona process outside main().catch().
1242
+ // Route it through the parser's async iterator so normal runner failure
1243
+ // handling can push an actionable terminal instead.
1244
+ source.once('error', (error) => parser.destroy(error));
1245
+ return source.pipe(parser) as AsyncIterable<Record<string, unknown>>;
1246
+ }
1247
+
1248
+ function createCsvSource(input: {
1249
+ path?: string;
1250
+ url?: string;
1251
+ contentBase64?: string;
1252
+ }): ReplayableCsvSource {
1253
+ return {
1254
+ iterate: () => {
1255
+ if (input.contentBase64) {
1256
+ return parseCsvStream(
1257
+ Readable.from([Buffer.from(input.contentBase64, 'base64')]),
1258
+ );
1259
+ }
1260
+ if (input.url) {
1261
+ return {
1262
+ async *[Symbol.asyncIterator]() {
1263
+ const response = await fetch(input.url!);
1264
+ if (!response.ok || !response.body) {
1265
+ throw new Error(
1266
+ `CSV source URL failed with status ${response.status}.`,
1267
+ );
1268
+ }
1269
+ yield* parseCsvStream(
1270
+ Readable.fromWeb(
1271
+ response.body as Parameters<typeof Readable.fromWeb>[0],
1272
+ ),
1273
+ );
1274
+ },
1275
+ } as AsyncIterable<Record<string, unknown>>;
1276
+ }
1277
+ if (!input.path) throw new Error('CSV source path is required.');
1278
+ return parseCsvStream(createReadStream(input.path));
1279
+ },
1280
+ };
1281
+ }
1282
+
1283
+ /**
1284
+ * Serialize a runner failure with the message guaranteed present. Bun can
1285
+ * surface `error.stack` without the leading "Error: <message>" line (seen on
1286
+ * governor guard errors thrown deep in async chains), and the run's terminal
1287
+ * error text is what users and the guard e2e assertions read — so never trust
1288
+ * the stack alone to carry the message.
1289
+ */
1290
+ function formatRunnerFailure(error: unknown): string {
1291
+ if (!(error instanceof Error)) {
1292
+ return String(error);
1293
+ }
1294
+ const stack = error.stack ?? '';
1295
+ if (!error.message.trim()) {
1296
+ return stack || String(error);
1297
+ }
1298
+ if (stack.includes(error.message)) {
1299
+ return stack;
1300
+ }
1301
+ const headline = `${error.name || 'Error'}: ${error.message}`;
1302
+ if (!stack) {
1303
+ return headline;
1304
+ }
1305
+ const frames = stack.startsWith(`${error.name || 'Error'}\n`)
1306
+ ? stack.slice((error.name || 'Error').length + 1)
1307
+ : stack;
1308
+ return `${headline}\n${frames}`;
1309
+ }
1310
+
1311
+ function ensureSourceMapSupport(): void {
1312
+ if (sourceMapSupportInstalled) {
1313
+ return;
1314
+ }
1315
+ installSourceMapSupport({
1316
+ environment: 'node',
1317
+ handleUncaughtExceptions: false,
1318
+ // A runner can execute many Play revisions. `source-map-support` otherwise
1319
+ // caches forever by virtual filename and can map a later revision through
1320
+ // an older artifact's map. Stack formatting is synchronous, so clearing
1321
+ // this internal cache between operations is concurrency-safe.
1322
+ emptyCacheBetweenOperations: true,
1323
+ retrieveFile(source) {
1324
+ return runtimeSourceFiles.get(source) ?? '';
1325
+ },
1326
+ retrieveSourceMap(source) {
1327
+ const map = runtimeSourceMaps.get(source);
1328
+ if (!map) {
1329
+ return null;
1330
+ }
1331
+ return {
1332
+ url: source,
1333
+ map,
1334
+ };
1335
+ },
1336
+ });
1337
+ sourceMapSupportInstalled = true;
1338
+ }
1339
+
1340
+ function retainRuntimeArtifact(artifact: PlayBundleArtifact): () => void {
1341
+ const existing = runtimeArtifactReferences.get(artifact.virtualFilename);
1342
+ if (
1343
+ existing &&
1344
+ (existing.artifactHash !== artifact.artifactHash ||
1345
+ existing.sourceMapHash !== artifact.sourceMapHash)
1346
+ ) {
1347
+ throw new Error(
1348
+ `Concurrent Play artifacts cannot share virtual filename ${artifact.virtualFilename} with different contents.`,
1349
+ );
1350
+ }
1351
+ runtimeSourceFiles.set(artifact.virtualFilename, artifact.bundledCode);
1352
+ runtimeSourceMaps.set(artifact.virtualFilename, artifact.sourceMap);
1353
+ runtimeArtifactReferences.set(artifact.virtualFilename, {
1354
+ artifactHash: artifact.artifactHash,
1355
+ sourceMapHash: artifact.sourceMapHash,
1356
+ count: (existing?.count ?? 0) + 1,
1357
+ });
1358
+
1359
+ let released = false;
1360
+ return () => {
1361
+ if (released) return;
1362
+ released = true;
1363
+ const current = runtimeArtifactReferences.get(artifact.virtualFilename);
1364
+ if (
1365
+ !current ||
1366
+ current.artifactHash !== artifact.artifactHash ||
1367
+ current.sourceMapHash !== artifact.sourceMapHash
1368
+ ) {
1369
+ return;
1370
+ }
1371
+ if (current.count > 1) {
1372
+ runtimeArtifactReferences.set(artifact.virtualFilename, {
1373
+ ...current,
1374
+ count: current.count - 1,
1375
+ });
1376
+ return;
1377
+ }
1378
+ runtimeArtifactReferences.delete(artifact.virtualFilename);
1379
+ runtimeSourceFiles.delete(artifact.virtualFilename);
1380
+ runtimeSourceMaps.delete(artifact.virtualFilename);
1381
+ };
1382
+ }
1383
+
1384
+ async function executeRuntimePlayCode(input: {
1385
+ artifact: PlayBundleArtifact;
1386
+ ctx: ReturnType<typeof createPlayContext>;
1387
+ runtimeInput: PlayRunInputPayload;
1388
+ options?: ExecutePlayRunnerOptions;
1389
+ }): Promise<unknown> {
1390
+ const compileStartedAt = Date.now();
1391
+ // Defense-in-depth at the load seam: this runner compiles play code with
1392
+ // Module._compile, which cannot resolve `cloudflare:workers` scheme imports.
1393
+ // If a historical or malformed bundle reaches here, fail loudly with a
1394
+ // directional artifact mismatch instead of the opaque scheme error thrown at
1395
+ // compile time.
1396
+ assertNodeRunnableArtifact({
1397
+ artifact: input.artifact,
1398
+ expectedKind: PLAY_ARTIFACT_KINDS.cjsNode20,
1399
+ context: `play-runner ${input.artifact.virtualFilename}`,
1400
+ });
1401
+ ensureSourceMapSupport();
1402
+ const releaseRuntimeArtifact = retainRuntimeArtifact(input.artifact);
1403
+ try {
1404
+ const runtimeModule = new moduleRuntime.Module(
1405
+ input.artifact.virtualFilename,
1406
+ );
1407
+ runtimeModule.filename = input.artifact.virtualFilename;
1408
+ runtimeModule.paths = moduleRuntime.Module._nodeModulePaths(process.cwd());
1409
+ // Bun's `Module` compatibility object does not install `module.require`.
1410
+ // Install a fresh package-only wrapper for both CommonJS entry points. Local
1411
+ // files belong in the bundle; resolving them from the host repo would cross
1412
+ // the artifact boundary, and sharing one function would leak mutations.
1413
+ const runtimeRequire = createPlayRuntimeRequire();
1414
+ runtimeModule.require = runtimeRequire;
1415
+ // Resolve free `console` references through a per-run context extension.
1416
+ // This keeps directive prologues and customer-declared top-level `console`
1417
+ // bindings intact, while avoiding a process-global console swap that could
1418
+ // mix concurrent Absurd claims.
1419
+ const compiled = compileFunction(
1420
+ input.artifact.bundledCode,
1421
+ ['exports', 'require', 'module', '__filename', '__dirname'],
1422
+ {
1423
+ filename: input.artifact.virtualFilename,
1424
+ contextExtensions: [
1425
+ { console: createCapturedCustomerConsole(input.ctx) },
1426
+ ],
1427
+ },
1428
+ );
1429
+ compiled.call(
1430
+ runtimeModule.exports,
1431
+ runtimeModule.exports,
1432
+ runtimeRequire,
1433
+ runtimeModule,
1434
+ input.artifact.virtualFilename,
1435
+ dirname(input.artifact.virtualFilename),
1436
+ );
1437
+ emitWorkerLog(
1438
+ `[perf] artifact compile elapsed_ms=${Date.now() - compileStartedAt}`,
1439
+ input.options,
1440
+ );
1441
+
1442
+ const candidate =
1443
+ typeof runtimeModule.exports === 'function'
1444
+ ? runtimeModule.exports
1445
+ : (runtimeModule.exports as { default?: unknown }).default;
1446
+
1447
+ if (typeof candidate !== 'function') {
1448
+ throw new Error(
1449
+ 'Bundled play module must default export an async function.',
1450
+ );
1451
+ }
1452
+
1453
+ const userFunctionStartedAt = Date.now();
1454
+ const result = await (
1455
+ candidate as (
1456
+ ctx: unknown,
1457
+ runtimeInput: PlayRunInputPayload,
1458
+ ) => Promise<unknown>
1459
+ )(input.ctx, input.runtimeInput);
1460
+ // A Play may catch an individual tool failure, but a runner-owned fatal
1461
+ // tool failure still cannot settle as a normal result. Today this covers
1462
+ // Deepline's own insufficient-credit denial and a Slack HITL deadline
1463
+ // whose Slack-card replacement was confirmed. Provider capacity remains a
1464
+ // normal typed waterfall miss.
1465
+ input.ctx.assertNoRunFatalToolFailure();
1466
+ emitWorkerLog(
1467
+ `[perf] user function returned elapsed_ms=${Date.now() - userFunctionStartedAt}`,
1468
+ input.options,
1469
+ );
1470
+ return result;
1471
+ } catch (error) {
1472
+ // Error.stack is lazy in V8. Materialize it while this artifact's source
1473
+ // map is still registered; the mapped string remains on the Error after
1474
+ // the in-memory artifact is released and crosses the runner boundary.
1475
+ if (error instanceof Error) void error.stack;
1476
+ throw error;
1477
+ } finally {
1478
+ releaseRuntimeArtifact();
1479
+ }
1480
+ }
1481
+
1482
+ const CAPTURED_CONSOLE_MAX_CHARS = 2_000;
1483
+
1484
+ function logLevelForCapturedConsoleMethod(
1485
+ method: string,
1486
+ ): 'debug' | 'info' | 'warn' | 'error' {
1487
+ if (method === 'debug' || method === 'trace') return 'debug';
1488
+ if (method === 'warn') return 'warn';
1489
+ if (method === 'error' || method === 'assert') return 'error';
1490
+ return 'info';
1491
+ }
1492
+
1493
+ function createCapturedCustomerConsole(
1494
+ ctx: ReturnType<typeof createPlayContext>,
1495
+ ): Console {
1496
+ const counters = new Map<string, number>();
1497
+ const timers = new Map<string, number>();
1498
+ const capture =
1499
+ (method: string) =>
1500
+ (...args: unknown[]) => {
1501
+ assertNoSecretTaint(args, `console.${method}`);
1502
+ const formatted = formatWithOptions(
1503
+ {
1504
+ breakLength: Infinity,
1505
+ colors: false,
1506
+ compact: true,
1507
+ depth: 4,
1508
+ maxArrayLength: 20,
1509
+ maxStringLength: 2_000,
1510
+ },
1511
+ ...args,
1512
+ );
1513
+ const message =
1514
+ formatted.length > CAPTURED_CONSOLE_MAX_CHARS
1515
+ ? `${formatted.slice(0, CAPTURED_CONSOLE_MAX_CHARS - 1)}…`
1516
+ : formatted;
1517
+ ctx.log(`[console.${method}] ${message}`, {
1518
+ level: logLevelForCapturedConsoleMethod(method),
1519
+ });
1520
+ };
1521
+
1522
+ const captured = {
1523
+ assert(condition?: boolean, ...args: unknown[]) {
1524
+ if (!condition) {
1525
+ capture('assert')('Assertion failed', ...args);
1526
+ }
1527
+ },
1528
+ clear() {},
1529
+ count(label = 'default') {
1530
+ const next = (counters.get(label) ?? 0) + 1;
1531
+ counters.set(label, next);
1532
+ capture('count')(`${label}: ${next}`);
1533
+ },
1534
+ countReset(label = 'default') {
1535
+ counters.delete(label);
1536
+ },
1537
+ debug: capture('debug'),
1538
+ dir: capture('dir'),
1539
+ dirxml: capture('dirxml'),
1540
+ error: capture('error'),
1541
+ group: capture('group'),
1542
+ groupCollapsed: capture('groupCollapsed'),
1543
+ groupEnd() {},
1544
+ info: capture('info'),
1545
+ log: capture('log'),
1546
+ profile() {},
1547
+ profileEnd() {},
1548
+ table: capture('table'),
1549
+ time(label = 'default') {
1550
+ timers.set(label, Date.now());
1551
+ },
1552
+ timeEnd(label = 'default') {
1553
+ const startedAt = timers.get(label);
1554
+ timers.delete(label);
1555
+ capture('timeEnd')(
1556
+ `${label}: ${startedAt === undefined ? 0 : Date.now() - startedAt}ms`,
1557
+ );
1558
+ },
1559
+ timeLog(label = 'default', ...args: unknown[]) {
1560
+ const startedAt = timers.get(label);
1561
+ capture('timeLog')(
1562
+ `${label}: ${startedAt === undefined ? 0 : Date.now() - startedAt}ms`,
1563
+ ...args,
1564
+ );
1565
+ },
1566
+ timeStamp() {},
1567
+ trace: capture('trace'),
1568
+ warn: capture('warn'),
1569
+ };
1570
+
1571
+ // Deliberately use no Console prototype. Any method inherited from Node's
1572
+ // process-global console can write to shared stdout/stderr, bypassing both
1573
+ // the run log and secret-taint validation.
1574
+ return Object.assign(Object.create(null), captured) as Console;
1575
+ }
1576
+
1577
+ /**
1578
+ * Release attempt-scoped leases when a run settles. Terminal runs no longer own
1579
+ * rows, so their sheet fences must not delay an immediate repair run. A
1580
+ * suspension keeps its sheet fences because the same run will resume.
1581
+ */
1582
+ async function releaseRuntimeLeasesOnAppRunnerSettlement(input: {
1583
+ config: PlayRunnerExecutionConfig;
1584
+ runtimeSheetAttempts: Map<
1585
+ string,
1586
+ {
1587
+ attemptId: string;
1588
+ attemptOwnerRunId: string;
1589
+ attemptExpiresAt: string;
1590
+ attemptSeq: number;
1591
+ writeVersion?: number;
1592
+ }
1593
+ >;
1594
+ runnerRunAttempt: number;
1595
+ preloadedDbSessions: PlayRunnerExecutionConfig['context']['preloadedDbSessions'];
1596
+ postgresSessionUnwrapKey: string | null;
1597
+ emitWorkerLog: (message: string) => void;
1598
+ reason: 'completed' | 'suspension' | 'run-fatal';
1599
+ }): Promise<void> {
1600
+ const { config } = input;
1601
+ const reasonLabel =
1602
+ input.reason === 'completed'
1603
+ ? 'completed settlement'
1604
+ : `${input.reason} teardown`;
1605
+ const orgId = config.context.orgId;
1606
+ const playName = config.context.playName;
1607
+ const runId =
1608
+ config.context.runId ?? config.context.workflowId ?? 'anonymous-run';
1609
+ if (!orgId || !playName) return;
1610
+ const runtimeContext = {
1611
+ executorToken: config.context.executorToken,
1612
+ baseUrl: config.context.baseUrl,
1613
+ ...playRunnerInvocationFenceFlags(config.context),
1614
+ runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
1615
+ vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
1616
+ orgId,
1617
+ playName,
1618
+ runId,
1619
+ userEmail: config.context.userEmail,
1620
+ dbSessionStrategy: config.context.dbSessionStrategy,
1621
+ preloadedDbSessions: input.preloadedDbSessions,
1622
+ postgresSessionUnwrapKey: input.postgresSessionUnwrapKey ?? undefined,
1623
+ };
1624
+ // Runtime Sheet rows use schedule-time write versions. They have no lease
1625
+ // to heartbeat or release; late writers are rejected by the write fence.
1626
+ void input.runtimeSheetAttempts;
1627
+ try {
1628
+ const released = await releaseRuntimeWorkReceipts(runtimeContext, {
1629
+ playName,
1630
+ runId,
1631
+ runAttempt: input.runnerRunAttempt,
1632
+ });
1633
+ if (released.released > 0) {
1634
+ input.emitWorkerLog(
1635
+ `[info] released ${released.released} completed-work lease(s) on ${reasonLabel}`,
1636
+ );
1637
+ }
1638
+ } catch (releaseError) {
1639
+ input.emitWorkerLog(
1640
+ `[warn] completed-work lease release failed on ${reasonLabel} (rerun falls back to lease-TTL expiry): ${
1641
+ releaseError instanceof Error
1642
+ ? releaseError.message
1643
+ : String(releaseError)
1644
+ }`,
1645
+ );
1646
+ }
1647
+ }
1648
+
1649
+ async function run(
1650
+ config: PlayRunnerExecutionConfig,
1651
+ options?: ExecutePlayRunnerOptions,
1652
+ ): Promise<PlayRunnerResult> {
1653
+ const runStartedAt = Date.now();
1654
+ const persistenceSignal =
1655
+ options?.signal ??
1656
+ AbortSignal.timeout(
1657
+ validatePlaySandboxRuntimeLimits(
1658
+ config.context.sandboxRuntimeLimits ??
1659
+ STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
1660
+ ).timeoutSeconds * 1_000,
1661
+ );
1662
+ const contextSetupStartedAt = Date.now();
1663
+ const getRuntimeToolMetadata = createRuntimeToolMetadataResolver(config);
1664
+ const getRuntimeToolAuthScopeDigest =
1665
+ createAppRuntimeToolAuthScopeDigestResolver(config);
1666
+ let totalRows = 0;
1667
+ let inserted = 0;
1668
+ let skipped = 0;
1669
+ const runtimePreloadedDbSessions = config.context.preloadedDbSessions;
1670
+ const runtimePostgresSessionUnwrapKey =
1671
+ // Preloaded sessions are sealed with the immutable submit-time token, which
1672
+ // may differ from the per-attempt `executorToken` durable schedulers
1673
+ // re-mint. Prefer the pinned unwrap key; fall back for legacy launches.
1674
+ config.context.postgresSessionUnwrapKey ??
1675
+ config.context.executorToken ??
1676
+ process.env.DEEPLINE_INTERNAL_TOKEN ??
1677
+ null;
1678
+ const runtimeCsvDatasets = new Map<
1679
+ string,
1680
+ {
1681
+ tableNamespace: string;
1682
+ dataset: PlayDataset<Record<string, unknown>>;
1683
+ sheetContract: PlaySheetContract;
1684
+ sourceLabel: string;
1685
+ }
1686
+ >();
1687
+ const appRuntimeApiContext = {
1688
+ baseUrl: config.context.baseUrl ?? '',
1689
+ executorToken: config.context.executorToken ?? '',
1690
+ vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
1691
+ runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
1692
+ };
1693
+ const receiptGatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
1694
+ // Receipt state has exactly two backends: handlers provided by the host
1695
+ // process (inline executor) or the co-located receipt gateway named by the
1696
+ // launch context. There is deliberately no fallback onto the app runtime
1697
+ // API and no env-var switch here.
1698
+ const runtimeReceiptApiContext = receiptGatewayBaseUrl
1699
+ ? {
1700
+ ...appRuntimeApiContext,
1701
+ baseUrl: receiptGatewayBaseUrl,
1702
+ boundary: 'receipt_gateway' as const,
1703
+ requestTimeoutMs: RUNTIME_RECEIPT_GATEWAY_REQUEST_TIMEOUT_MS,
1704
+ }
1705
+ : null;
1706
+ const canUseRuntimeStepReceipts = playRunnerRuntimeReceiptsEnabled(
1707
+ config.context,
1708
+ );
1709
+ const receiptStoreAdapter =
1710
+ runtimeReceiptApiContext &&
1711
+ canUseRuntimeStepReceipts &&
1712
+ config.context.playName &&
1713
+ config.context.orgId
1714
+ ? new RuntimeReceiptStoreAdapter({
1715
+ context: runtimeReceiptApiContext,
1716
+ playName: config.context.playName,
1717
+ runId:
1718
+ config.context.runId ??
1719
+ config.context.workflowId ??
1720
+ 'anonymous-run',
1721
+ maxBatchSize: RUNTIME_RECEIPT_GATEWAY_BATCH_SIZE,
1722
+ maxFlushMs: RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_WAIT_MS,
1723
+ onRetryTelemetry: (line) => emitSystemLog(line, options),
1724
+ })
1725
+ : null;
1726
+ const receiptGatewayHandlers: RuntimeStepReceiptHandlers | null =
1727
+ receiptStoreAdapter?.handlers ?? null;
1728
+ if (canUseRuntimeStepReceipts && !receiptGatewayHandlers) {
1729
+ throw new Error(
1730
+ 'Play runner has no receipt backend: the launch context must carry ' +
1731
+ 'receiptGatewayBaseUrl. Runtime receipts have exactly one path — ' +
1732
+ 'the receipt gateway — never the app runtime API.',
1733
+ );
1734
+ }
1735
+ const receiptHandlers = receiptGatewayHandlers;
1736
+ emitWorkerLog(
1737
+ `[perf] runtime state=${
1738
+ receiptGatewayHandlers ? 'durable' : 'disabled'
1739
+ } db_session_strategy=${config.context.dbSessionStrategy ?? 'unset'} preloaded_db_sessions=${runtimePreloadedDbSessions?.length ?? 0}`,
1740
+ options,
1741
+ );
1742
+ const runtimeSheetAttempts = new Map<
1743
+ string,
1744
+ {
1745
+ attemptId: string;
1746
+ attemptOwnerRunId: string;
1747
+ attemptExpiresAt: string;
1748
+ attemptSeq: number;
1749
+ writeVersion?: number;
1750
+ }
1751
+ >();
1752
+ const runtimeMapDatasetSettlements = new Map<
1753
+ string,
1754
+ {
1755
+ datasetId: string;
1756
+ path: string;
1757
+ tableNamespace: string;
1758
+ runId: string;
1759
+ persistedRows: number;
1760
+ succeededRows: number;
1761
+ failedRows: number;
1762
+ }
1763
+ >();
1764
+ const runtimeSheetAttemptKey = (input: {
1765
+ playName: string;
1766
+ runId: string;
1767
+ tableNamespace: string;
1768
+ }) => `${input.playName}\u0000${input.runId}\u0000${input.tableNamespace}`;
1769
+ const runnerRunAttempt =
1770
+ typeof config.context.runAttempt === 'number' &&
1771
+ Number.isFinite(config.context.runAttempt)
1772
+ ? Math.max(0, Math.floor(config.context.runAttempt))
1773
+ : 0;
1774
+ const cachePolicy = playRunnerContextCachePolicy(config.context);
1775
+ const rateStateBackend = createRateStateBackendForConfig(config);
1776
+ const runtimeInput = config.input ?? {};
1777
+ const runtimeInputObject: Record<string, unknown> =
1778
+ playRunInputObjectView(runtimeInput);
1779
+ const resolveChildPlay =
1780
+ createRunScopedChildPlayResolver<ResolvedPlayExecution>({
1781
+ resolvePlay: async (playRef) => {
1782
+ if (!config.context.orgId) {
1783
+ return null;
1784
+ }
1785
+ // Child plays always resolve through the server's CJS-only path.
1786
+ try {
1787
+ const resolved = await resolveRuntimeReferencedPlay(
1788
+ {
1789
+ executorToken: config.context.executorToken,
1790
+ baseUrl: config.context.baseUrl,
1791
+ vercelProtectionBypassToken:
1792
+ config.context.vercelProtectionBypassToken,
1793
+ playName: config.context.playName,
1794
+ userEmail: config.context.userEmail,
1795
+ },
1796
+ playRef,
1797
+ );
1798
+ if (resolved?.resolutionDiagnostics) {
1799
+ emitWorkerLog(
1800
+ `[runtime.resolve-play-contract] ${JSON.stringify({
1801
+ playRef,
1802
+ ...resolved.resolutionDiagnostics,
1803
+ runnerPipeline: Boolean(resolved.staticPipeline),
1804
+ })}`,
1805
+ options,
1806
+ );
1807
+ }
1808
+ return resolved as ResolvedPlayExecution | null;
1809
+ } catch (error) {
1810
+ const message =
1811
+ error instanceof Error ? error.message : String(error);
1812
+ throw new Error(
1813
+ `ctx.runPlay failed to resolve child play "${playRef}" via runtime API baseUrl=${config.context.baseUrl ?? '<missing>'}: ${message}`,
1814
+ );
1815
+ }
1816
+ },
1817
+ });
1818
+ const maxConcurrentExternalCalls = resolveMaxConcurrentExternalCalls(
1819
+ config.context.maxConcurrentExternalCalls,
1820
+ );
1821
+ const maxConcurrentRows = resolveMaxConcurrentRows(
1822
+ config.context.maxConcurrentRows,
1823
+ );
1824
+ emitWorkerLog(
1825
+ `[runtime.resource-policy] max_concurrent_external_calls=${maxConcurrentExternalCalls} max_concurrent_rows=${maxConcurrentRows ?? 'map_default'}`,
1826
+ options,
1827
+ );
1828
+ const artifactCompatibility = normalizePlayContractCompatibility(
1829
+ config.artifact.compatibility,
1830
+ );
1831
+ const artifactRuntimeContract = resolvePlayArtifactRuntimeContract({
1832
+ authoringContractEdition: artifactCompatibility.authoringContractEdition,
1833
+ slackHitlTimeoutOutcome: artifactCompatibility.slackHitlTimeoutOutcome,
1834
+ toolResponseContract: config.context.toolResponseContract,
1835
+ toolResponseReceiptRevision: config.context.toolResponseReceiptRevision,
1836
+ });
1837
+ const ctx = createPlayContext({
1838
+ artifactRuntimeContract,
1839
+ executorToken: config.context.executorToken,
1840
+ baseUrl: config.context.baseUrl,
1841
+ // Runtime control stays on baseUrl (the receipt gateway); provider calls
1842
+ // alone use this separate relay origin.
1843
+ executionGatewayBaseUrl: config.context.executionGatewayBaseUrl,
1844
+ ...playRunnerInvocationFenceFlags(config.context),
1845
+ runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
1846
+ integrationMode: config.context.integrationMode,
1847
+ docflowEnabled: config.context.docflowEnabled === true,
1848
+ fixtureBehavior: config.context.fixtureBehavior,
1849
+ maxConcurrentExternalCalls,
1850
+ maxConcurrentRows,
1851
+ verifyDurableInvocationFence: async (signal) => {
1852
+ const gatewayBaseUrl =
1853
+ config.context.executionGatewayBaseUrl?.trim().replace(/\/$/, '') ||
1854
+ config.context.receiptGatewayBaseUrl?.trim().replace(/\/$/, '') ||
1855
+ '';
1856
+ const hasExplicitFence =
1857
+ config.context.requestDurableInvocationFence === true;
1858
+ if (
1859
+ !gatewayBaseUrl ||
1860
+ (!hasExplicitFence &&
1861
+ config.context.baseUrl?.replace(/\/$/, '') !== gatewayBaseUrl)
1862
+ ) {
1863
+ return false;
1864
+ }
1865
+ try {
1866
+ const timeoutSignal = AbortSignal.timeout(
1867
+ RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS,
1868
+ );
1869
+ const response = await safeNodeOutboundFetch(
1870
+ `${gatewayBaseUrl}/healthz`,
1871
+ {
1872
+ redirect: 'manual',
1873
+ signal: signal
1874
+ ? AbortSignal.any([signal, timeoutSignal])
1875
+ : timeoutSignal,
1876
+ },
1877
+ );
1878
+ return (
1879
+ response.ok &&
1880
+ response.headers.get(DURABLE_INVOCATION_FENCE_ATTESTATION_HEADER) ===
1881
+ DURABLE_INVOCATION_FENCE_ATTESTATION_VALUE
1882
+ );
1883
+ } catch {
1884
+ return false;
1885
+ }
1886
+ },
1887
+ dbSessionStrategy: config.context.dbSessionStrategy,
1888
+ fetchImpl: safeNodeStreamingOutboundFetch,
1889
+ vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
1890
+ orgId: config.context.orgId,
1891
+ userEmail: config.context.userEmail,
1892
+ playName: config.context.playName,
1893
+ graphHash: config.artifact.graphHash,
1894
+ artifactHash: config.artifact.artifactHash,
1895
+ convexUrl: config.context.convexUrl,
1896
+ runtimeSchedulerSchema: config.context.runtimeSchedulerSchema ?? null,
1897
+ runtimeSheetBackedMapDatasets: true,
1898
+ runtimeReceiptClaimsEstablishExecutionFence:
1899
+ receiptStoreAdapter?.claimsEstablishExecutionFence === true,
1900
+ workflowId: config.context.workflowId,
1901
+ playId: config.context.playId ?? config.context.workflowId,
1902
+ runId: config.context.runId ?? config.context.workflowId,
1903
+ rootRunId:
1904
+ config.context.rootRunId ??
1905
+ config.context.runId ??
1906
+ config.context.workflowId,
1907
+ runAttempt: runnerRunAttempt,
1908
+ staticPipeline: config.context.staticPipeline,
1909
+ checkpoint: config.checkpoint ?? undefined,
1910
+ durableBoundaries: true,
1911
+ governance: config.context.governance ?? undefined,
1912
+ ...rateStateBackend,
1913
+ ...(cachePolicy ? { cachePolicy } : {}),
1914
+ ...(receiptHandlers ?? {}),
1915
+ resolvePlay: resolveChildPlay,
1916
+ launchPlayAsync: async (request) =>
1917
+ await launchPlayAsyncViaAppRuntime(
1918
+ {
1919
+ ...appRuntimeApiContext,
1920
+ requestTimeoutMs: RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS,
1921
+ },
1922
+ request,
1923
+ ),
1924
+ onBatchComplete: (checkpoint) => {
1925
+ emitRunnerEvent(
1926
+ {
1927
+ type: 'checkpoint',
1928
+ checkpoint,
1929
+ },
1930
+ options,
1931
+ );
1932
+ },
1933
+ getToolQueueHints: async (toolId) => {
1934
+ const metadata = await getRuntimeToolMetadata(toolId);
1935
+ return Array.isArray(metadata.queueHints)
1936
+ ? (metadata.queueHints as PlayQueueHint[])
1937
+ : [];
1938
+ },
1939
+ getToolProvider: async (toolId) => {
1940
+ const provider = (await getRuntimeToolMetadata(toolId)).provider;
1941
+ return typeof provider === 'string' && provider.trim()
1942
+ ? provider.trim()
1943
+ : null;
1944
+ },
1945
+ getToolOperation: async (toolId) => {
1946
+ const operation = (await getRuntimeToolMetadata(toolId)).operation;
1947
+ return typeof operation === 'string' && operation.trim()
1948
+ ? operation.trim()
1949
+ : null;
1950
+ },
1951
+ getToolRetryPolicy: async (toolId) => {
1952
+ const metadata = await getRuntimeToolMetadata(toolId);
1953
+ return {
1954
+ retrySafeTransientHttp: metadata.retry?.retrySafeTransientHttp === true,
1955
+ requiresExecutionFence: metadata.retry?.requiresExecutionFence === true,
1956
+ };
1957
+ },
1958
+ getToolActionCacheVersion: async (toolId) => {
1959
+ const metadata = await getRuntimeToolMetadata(toolId);
1960
+ const version = metadata.actionCacheVersion.trim();
1961
+ if (!version) {
1962
+ throw new Error(
1963
+ `Runtime metadata for ${toolId} is missing actionCacheVersion.`,
1964
+ );
1965
+ }
1966
+ return version;
1967
+ },
1968
+ getToolActionCacheReadVersions: async (toolId) => {
1969
+ const metadata = await getRuntimeToolMetadata(toolId);
1970
+ return [
1971
+ ...new Set(
1972
+ (metadata.actionCacheReadVersions ?? [])
1973
+ .map((version) => String(version || '').trim())
1974
+ .filter(Boolean),
1975
+ ),
1976
+ ];
1977
+ },
1978
+ getToolAuthScopeDigest: getRuntimeToolAuthScopeDigest,
1979
+ getToolRateScope: getRuntimeToolAuthScopeDigest.rateScope,
1980
+ invalidateToolAuthScopeDigest: (toolId) =>
1981
+ getRuntimeToolAuthScopeDigest.invalidate?.(toolId),
1982
+ getToolTargetGetters: async (toolId, output) => {
1983
+ const metadata = await getRuntimeToolMetadata(toolId);
1984
+ return metadata.targetGetters?.[output] ?? [];
1985
+ },
1986
+ getToolResultMetadata: async (toolId) => {
1987
+ const metadata = await getRuntimeToolMetadata(toolId);
1988
+ return {
1989
+ toolId,
1990
+ extractors: metadata.extractors ?? {},
1991
+ targetGetters: metadata.targetGetters ?? {},
1992
+ listExtractorPaths: metadata.listExtractorPaths ?? [],
1993
+ listIdentityGetters: metadata.listIdentityGetters ?? {},
1994
+ };
1995
+ },
1996
+ getToolAsyncOperation: async (toolId) =>
1997
+ (await getRuntimeToolMetadata(toolId)).asyncOperation ?? null,
1998
+ getToolAsyncOperationPoll: async (toolId) =>
1999
+ (await getRuntimeToolMetadata(toolId)).asyncOperationPoll ?? null,
2000
+ getBatchOperationStrategy: (toolId) => getPlayRuntimeBatchStrategy(toolId),
2001
+ getIntegrationEventWaitHandler: async (toolId) => {
2002
+ if (isIntegrationEventWaitToolId(toolId)) {
2003
+ return createRuntimeEventWaitHandler(config, toolId);
2004
+ }
2005
+ const metadata = await getRuntimeToolMetadata(toolId);
2006
+ return metadata.eventWait
2007
+ ? createRuntimeEventWaitHandler(config, toolId)
2008
+ : null;
2009
+ },
2010
+ onRowUpdate: (() => {
2011
+ let rowUpdateCount = 0;
2012
+ let lastEmitAt = 0;
2013
+ const ROW_UPDATE_EMIT_INTERVAL_MS = 200;
2014
+ const ROW_UPDATE_EMIT_SAMPLE_RATE = 100;
2015
+ const terminalCellStates = new Set([
2016
+ 'completed',
2017
+ 'failed',
2018
+ 'cached',
2019
+ 'missed',
2020
+ 'skipped',
2021
+ ]);
2022
+ return (update: PlayRowUpdate) => {
2023
+ rowUpdateCount++;
2024
+ const now = Date.now();
2025
+ const hasTerminalCellUpdate = Object.values(
2026
+ update.cellMetaPatch ?? {},
2027
+ ).some((cell) => terminalCellStates.has(cell.status));
2028
+ const isTerminalUpdate =
2029
+ update.status === 'completed' ||
2030
+ update.status === 'failed' ||
2031
+ hasTerminalCellUpdate;
2032
+ if (
2033
+ isTerminalUpdate ||
2034
+ rowUpdateCount === 1 ||
2035
+ rowUpdateCount % ROW_UPDATE_EMIT_SAMPLE_RATE === 0 ||
2036
+ now - lastEmitAt >= ROW_UPDATE_EMIT_INTERVAL_MS
2037
+ ) {
2038
+ lastEmitAt = now;
2039
+ emitRunnerEvent({ type: 'row_update', update }, options);
2040
+ }
2041
+ };
2042
+ })(),
2043
+ onExecutionEvent: (() => {
2044
+ // map.row.updated fires for every cell-state change of every row (~7x
2045
+ // per row in a multi-column map) with no terminal semantics — unsampled
2046
+ // it floods the event transport at 150k-row scale. Sample it like row
2047
+ // updates; every other event type passes through untouched.
2048
+ let mapRowUpdatedCount = 0;
2049
+ let lastMapRowUpdatedEmitAt = 0;
2050
+ const MAP_ROW_UPDATED_EMIT_INTERVAL_MS = 200;
2051
+ const MAP_ROW_UPDATED_EMIT_SAMPLE_RATE = 100;
2052
+ return (event: PlayExecutionEvent) => {
2053
+ if (event.type === 'map.row.updated') {
2054
+ mapRowUpdatedCount++;
2055
+ const now = Date.now();
2056
+ if (
2057
+ mapRowUpdatedCount !== 1 &&
2058
+ mapRowUpdatedCount % MAP_ROW_UPDATED_EMIT_SAMPLE_RATE !== 0 &&
2059
+ now - lastMapRowUpdatedEmitAt < MAP_ROW_UPDATED_EMIT_INTERVAL_MS
2060
+ ) {
2061
+ return;
2062
+ }
2063
+ lastMapRowUpdatedEmitAt = now;
2064
+ }
2065
+ emitRunnerEvent(
2066
+ {
2067
+ type: 'execution_event',
2068
+ event,
2069
+ },
2070
+ options,
2071
+ );
2072
+ };
2073
+ })(),
2074
+ // Durable docflow observation sink (ADR 0016 rule 2). Present only when the
2075
+ // co-located receipt gateway is wired (same durable path as receipts) and
2076
+ // the play carries an org/play scope. The compiler injects the observation
2077
+ // wrapper ONLY into instrumented (diagrammed) plays, so an undiagrammed play
2078
+ // never invokes this sink — the gating is at the wrapper. Fire-and-forget:
2079
+ // any transport failure is swallowed-and-logged so the play body is never
2080
+ // affected. The live docflow.node.* event stream is unchanged.
2081
+ onDocflowObservation:
2082
+ runtimeReceiptApiContext && config.context.playName
2083
+ ? (observation) => {
2084
+ // Tracked so the graceful terminal path can drain in-flight posts
2085
+ // (bounded) before the terminal wakes the parked worker's finalize
2086
+ // read. Still strictly fire-and-forget from the play body.
2087
+ const post = observeDocflowNodeViaAppRuntime(
2088
+ runtimeReceiptApiContext,
2089
+ {
2090
+ playName: config.context.playName!,
2091
+ runId:
2092
+ observation.runId ||
2093
+ config.context.runId ||
2094
+ config.context.workflowId ||
2095
+ 'anonymous-run',
2096
+ nodeId: observation.nodeId,
2097
+ attempt: observation.attempt,
2098
+ invocationSeq: observation.invocationSeq,
2099
+ invocationId: observation.invocationId ?? null,
2100
+ status: observation.status,
2101
+ inputs: observation.inputs ?? null,
2102
+ outputs: observation.outputs ?? null,
2103
+ inputsTruncated: observation.inputsTruncated === true,
2104
+ outputsTruncated: observation.outputsTruncated === true,
2105
+ error: observation.error ?? null,
2106
+ at: observation.at,
2107
+ },
2108
+ ).catch((error) => {
2109
+ emitWorkerLog(
2110
+ `[error] docflow observation write failed node=${observation.nodeId} status=${observation.status}: ${
2111
+ error instanceof Error ? error.message : String(error)
2112
+ }`,
2113
+ options,
2114
+ );
2115
+ });
2116
+ pendingDocflowObservationPosts.add(post);
2117
+ void post.finally(() => {
2118
+ pendingDocflowObservationPosts.delete(post);
2119
+ });
2120
+ }
2121
+ : undefined,
2122
+ onLog: (line) => {
2123
+ const timestampMatch = line.match(/^\[([^\]]+)\]/);
2124
+ const timestamp = timestampMatch?.[1];
2125
+ const at =
2126
+ timestamp && !Number.isNaN(new Date(timestamp).getTime())
2127
+ ? new Date(timestamp).toISOString()
2128
+ : new Date().toISOString();
2129
+ emitRunnerEvent(
2130
+ {
2131
+ type: 'log',
2132
+ at,
2133
+ source: 'play',
2134
+ line,
2135
+ },
2136
+ options,
2137
+ );
2138
+ },
2139
+ onMapStart: async (items, tableNamespace, mapContext) => {
2140
+ assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
2141
+ const { baseUrl, orgId } = config.context;
2142
+ const startedAt = Date.now();
2143
+ totalRows += items.length;
2144
+ const userEmail = config.context.userEmail;
2145
+ const playName = mapContext.playName ?? config.context.playName;
2146
+ const runId =
2147
+ mapContext.runId ??
2148
+ config.context.runId ??
2149
+ config.context.workflowId ??
2150
+ 'anonymous-run';
2151
+ const executorToken =
2152
+ mapContext.executorToken ?? config.context.executorToken;
2153
+ const rootRunId =
2154
+ config.context.runId ?? config.context.workflowId ?? 'anonymous-run';
2155
+ const canUseRootPreloadedDbSessions =
2156
+ executorToken === config.context.executorToken && runId === rootRunId;
2157
+
2158
+ const sheetContract = resolveSheetContractForTableNamespace(
2159
+ mapContext.staticPipeline ?? config.context.staticPipeline,
2160
+ tableNamespace,
2161
+ );
2162
+ if (!sheetContract) {
2163
+ throw new Error(
2164
+ `Missing sheet contract for play "${playName}" and table namespace "${tableNamespace}".`,
2165
+ );
2166
+ }
2167
+ const mapStartResult = await retryRuntimeSheetPersistence({
2168
+ description: `runtime map start table=${tableNamespace}`,
2169
+ log: (line) => emitWorkerLog(line, options),
2170
+ signal: persistenceSignal,
2171
+ run: () =>
2172
+ startRuntimeSheetDataset(
2173
+ {
2174
+ executorToken,
2175
+ baseUrl,
2176
+ vercelProtectionBypassToken:
2177
+ config.context.vercelProtectionBypassToken,
2178
+ orgId,
2179
+ playName,
2180
+ runId,
2181
+ userEmail,
2182
+ dbSessionStrategy: config.context.dbSessionStrategy,
2183
+ preloadedDbSessions: canUseRootPreloadedDbSessions
2184
+ ? runtimePreloadedDbSessions
2185
+ : undefined,
2186
+ postgresSessionUnwrapKey: canUseRootPreloadedDbSessions
2187
+ ? runtimePostgresSessionUnwrapKey
2188
+ : executorToken,
2189
+ abortSignal: persistenceSignal,
2190
+ },
2191
+ {
2192
+ playName,
2193
+ tableNamespace,
2194
+ playInput: runtimeInputObject,
2195
+ sheetContract,
2196
+ rows: items,
2197
+ inputOffset: mapContext.inputOffset,
2198
+ mode: mapContext.mode,
2199
+ runId,
2200
+ attemptId: deterministicRuntimeSheetAttemptId({
2201
+ playName,
2202
+ runId,
2203
+ tableNamespace,
2204
+ attemptSeq: runnerRunAttempt,
2205
+ }),
2206
+ attemptOwnerRunId: runId,
2207
+ attemptSeq: runnerRunAttempt,
2208
+ ...(playRunnerContextRuntimeSheetForce(config.context)
2209
+ ? { force: true }
2210
+ : {}),
2211
+ },
2212
+ ),
2213
+ });
2214
+ const datasetId = createRuntimeDatasetId(playName, tableNamespace);
2215
+ const datasetPath = `datasets.${tableNamespace}`;
2216
+ const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
2217
+ datasetId,
2218
+ path: datasetPath,
2219
+ tableNamespace,
2220
+ runId,
2221
+ persistedRows: 0,
2222
+ succeededRows: 0,
2223
+ failedRows: 0,
2224
+ };
2225
+ // Reused rows are already terminal for this run. No completion callback
2226
+ // follows for them, so include them in the eventual terminal settlement.
2227
+ const alreadyCompletedRows = mapStartResult.completedRows.length;
2228
+ settlement.persistedRows += alreadyCompletedRows;
2229
+ settlement.succeededRows += alreadyCompletedRows;
2230
+ runtimeMapDatasetSettlements.set(datasetId, settlement);
2231
+ emitRunnerEvent(
2232
+ {
2233
+ type: 'execution_event',
2234
+ event: {
2235
+ type: 'dataset.lifecycle',
2236
+ datasetId,
2237
+ path: datasetPath,
2238
+ tableNamespace,
2239
+ phase: 'registered',
2240
+ persistedRows: mapStartResult.completedRows.length,
2241
+ succeededRows: mapStartResult.completedRows.length,
2242
+ failedRows: 0,
2243
+ complete: false,
2244
+ // ADR 0019 dataset-grain row birth. These rows entered from
2245
+ // play-body code, so the only witnessed fact is how many arrived at
2246
+ // this table — never a per-row source mapping, which the runtime
2247
+ // does not see here. `rowCountIn` counts rows admitted through this
2248
+ // boundary so far, so a paged dataset converges upward instead of
2249
+ // reporting only its last page.
2250
+ bornFrom: {
2251
+ table: tableNamespace,
2252
+ rowCountIn: (mapContext.inputOffset ?? 0) + items.length,
2253
+ },
2254
+ at: Date.now(),
2255
+ },
2256
+ },
2257
+ options,
2258
+ );
2259
+ emitRunnerEvent(
2260
+ {
2261
+ type: 'log',
2262
+ at: new Date().toISOString(),
2263
+ source: 'play',
2264
+ line: formatDatasetRegistrationLog({
2265
+ dataset: { path: datasetPath, tableNamespace },
2266
+ rows: items.length,
2267
+ }),
2268
+ },
2269
+ options,
2270
+ );
2271
+ if (
2272
+ mapStartResult.attemptId &&
2273
+ mapStartResult.attemptOwnerRunId &&
2274
+ mapStartResult.attemptExpiresAt
2275
+ ) {
2276
+ runtimeSheetAttempts.set(
2277
+ runtimeSheetAttemptKey({ playName, runId, tableNamespace }),
2278
+ {
2279
+ attemptId: mapStartResult.attemptId,
2280
+ attemptOwnerRunId: mapStartResult.attemptOwnerRunId,
2281
+ attemptExpiresAt: mapStartResult.attemptExpiresAt,
2282
+ attemptSeq: mapStartResult.attemptSeq ?? runnerRunAttempt,
2283
+ writeVersion: mapStartResult.writeVersion,
2284
+ },
2285
+ );
2286
+ }
2287
+ emitWorkerLog(
2288
+ `[perf] runtime map setup table=${tableNamespace} rows=${items.length} elapsed_ms=${Date.now() - startedAt}`,
2289
+ options,
2290
+ );
2291
+ inserted += mapStartResult.inserted;
2292
+ skipped += mapStartResult.skipped;
2293
+ return mapStartResult;
2294
+ },
2295
+ onMapRowsCheckpoint: async (input) => {
2296
+ if (input.updates.length === 0) return;
2297
+ assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
2298
+ const playName = input.playName ?? config.context.playName;
2299
+ const userEmail = config.context.userEmail;
2300
+ const { runId, tableNamespace, updates } = input;
2301
+ const baseUrl = config.context.baseUrl;
2302
+ const executorToken = input.executorToken ?? config.context.executorToken;
2303
+ if (!baseUrl || !executorToken) {
2304
+ throw new Error(
2305
+ `Runtime map checkpoint for "${playName}" requires the runtime API origin and executor token.`,
2306
+ );
2307
+ }
2308
+ const sheetContract = resolveSheetContractForTableNamespace(
2309
+ input.staticPipeline ?? config.context.staticPipeline,
2310
+ tableNamespace,
2311
+ );
2312
+ if (!sheetContract) {
2313
+ throw new Error(
2314
+ `Missing sheet contract for play "${playName}" and table namespace "${tableNamespace}" while checkpointing partial rows.`,
2315
+ );
2316
+ }
2317
+ const attempt = runtimeSheetAttempts.get(
2318
+ runtimeSheetAttemptKey({
2319
+ playName,
2320
+ runId,
2321
+ tableNamespace,
2322
+ }),
2323
+ );
2324
+ const checkpointUpdates = updates.map((update) => ({
2325
+ ...update,
2326
+ attemptId: attempt?.attemptId ?? null,
2327
+ attemptOwnerRunId: attempt?.attemptOwnerRunId ?? null,
2328
+ attemptExpiresAt: attempt?.attemptExpiresAt ?? null,
2329
+ attemptSeq: attempt?.attemptSeq ?? runnerRunAttempt,
2330
+ writeVersion: attempt?.writeVersion ?? null,
2331
+ }));
2332
+ await retryRuntimeSheetPersistence({
2333
+ description: `runtime map checkpoint table=${tableNamespace}`,
2334
+ log: (line) => emitWorkerLog(line, options),
2335
+ signal: persistenceSignal,
2336
+ run: () =>
2337
+ applyRowUpdatesViaAppRuntime(
2338
+ {
2339
+ baseUrl,
2340
+ executorToken,
2341
+ vercelProtectionBypassToken:
2342
+ config.context.vercelProtectionBypassToken,
2343
+ signal: persistenceSignal,
2344
+ },
2345
+ {
2346
+ playName,
2347
+ tableNamespace,
2348
+ sheetContract,
2349
+ contractSnapshot: {
2350
+ staticPipeline:
2351
+ input.staticPipeline ?? config.context.staticPipeline ?? null,
2352
+ },
2353
+ runId,
2354
+ userEmail,
2355
+ updates: checkpointUpdates,
2356
+ },
2357
+ ),
2358
+ });
2359
+ },
2360
+ onMapRowsCompleted: async (input) => {
2361
+ // Persist executed rows (data + cell meta) into the tenant sheet —
2362
+ // the sheet is the source of truth for dataset state, and the
2363
+ // deterministic row keys make this write idempotent across retries
2364
+ // and reruns. An actor email is optional attribution, never durability
2365
+ // authority for a scheduler-backed Play run.
2366
+ assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
2367
+ const playName = input.playName ?? config.context.playName;
2368
+ const runId =
2369
+ input.runId ??
2370
+ config.context.runId ??
2371
+ config.context.workflowId ??
2372
+ 'anonymous-run';
2373
+ const executorToken = input.executorToken ?? config.context.executorToken;
2374
+ const rootRunId =
2375
+ config.context.runId ?? config.context.workflowId ?? 'anonymous-run';
2376
+ const canUseRootPreloadedDbSessions =
2377
+ executorToken === config.context.executorToken && runId === rootRunId;
2378
+ const sheetContract = resolveSheetContractForTableNamespace(
2379
+ input.staticPipeline ?? config.context.staticPipeline,
2380
+ input.tableNamespace,
2381
+ );
2382
+ if (!sheetContract) {
2383
+ throw new Error(
2384
+ `Missing sheet contract for play "${playName}" and table namespace "${input.tableNamespace}" while persisting executed rows.`,
2385
+ );
2386
+ }
2387
+ const persistStartedAt = Date.now();
2388
+ const attempt = runtimeSheetAttempts.get(
2389
+ runtimeSheetAttemptKey({
2390
+ playName,
2391
+ runId,
2392
+ tableNamespace: input.tableNamespace,
2393
+ }),
2394
+ );
2395
+ const runtimeSheetContext = {
2396
+ executorToken,
2397
+ baseUrl: config.context.baseUrl,
2398
+ vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
2399
+ orgId: config.context.orgId,
2400
+ playName,
2401
+ runId,
2402
+ userEmail: config.context.userEmail,
2403
+ dbSessionStrategy: config.context.dbSessionStrategy,
2404
+ runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
2405
+ preloadedDbSessions: canUseRootPreloadedDbSessions
2406
+ ? runtimePreloadedDbSessions
2407
+ : undefined,
2408
+ postgresSessionUnwrapKey: canUseRootPreloadedDbSessions
2409
+ ? runtimePostgresSessionUnwrapKey
2410
+ : executorToken,
2411
+ abortSignal: persistenceSignal,
2412
+ };
2413
+ const pageTailWriteMarker = runtimeSheetPageTailWriteMarkerEvent(
2414
+ config.context.runtimeTestFaultHeader,
2415
+ input.rows,
2416
+ input.tableNamespace,
2417
+ config.context.dbSessionStrategy,
2418
+ );
2419
+ if (pageTailWriteMarker) {
2420
+ // Authorized synthetic fault only. Source "play" is intentional: the
2421
+ // direct-worker reporter drops host diagnostics, while this one marker
2422
+ // is the durable proof that the page-tail terminal write was admitted.
2423
+ emitRunnerEvent(pageTailWriteMarker, options);
2424
+ }
2425
+ emitWorkerLog(
2426
+ `[perf] runtime map persist:start table=${input.tableNamespace} rows=${input.rows.length}`,
2427
+ options,
2428
+ );
2429
+ const completeRows = createCompleteRuntimeMapRowsOperation(
2430
+ runtimeSheetContext,
2431
+ {
2432
+ tableNamespace: input.tableNamespace,
2433
+ sheetContract,
2434
+ rows: input.rows,
2435
+ outputFields: input.outputFields,
2436
+ runId,
2437
+ attemptId: attempt?.attemptId ?? null,
2438
+ attemptOwnerRunId: attempt?.attemptOwnerRunId ?? null,
2439
+ attemptExpiresAt: attempt?.attemptExpiresAt ?? null,
2440
+ attemptSeq: attempt?.attemptSeq ?? runnerRunAttempt,
2441
+ writeVersion: attempt?.writeVersion ?? null,
2442
+ },
2443
+ );
2444
+ const result = await retryRuntimeSheetPersistence({
2445
+ description: `runtime map persist table=${input.tableNamespace}`,
2446
+ log: (line) => emitWorkerLog(line, options),
2447
+ signal: persistenceSignal,
2448
+ run: completeRows,
2449
+ });
2450
+ const pageTailHoldCompleted = runtimeSheetPageTailHoldCompletedEvent(
2451
+ result.runtimeTestPageTailHoldMs,
2452
+ input.tableNamespace,
2453
+ );
2454
+ if (pageTailHoldCompleted) {
2455
+ emitRunnerEvent(pageTailHoldCompleted, options);
2456
+ }
2457
+ emitWorkerLog(
2458
+ `[perf] runtime map persist:write_done table=${input.tableNamespace} rows=${input.rows.length} updated=${result.updated} elapsed_ms=${Date.now() - persistStartedAt}`,
2459
+ options,
2460
+ );
2461
+ const staleKeys = new Set(result.staleKeys);
2462
+ const staleDropped = staleKeys.size;
2463
+ let recoveredStaleKeys: string[] = [];
2464
+ if (staleDropped > 0) {
2465
+ // A terminal retry can lose the SQL mutation race to its own earlier
2466
+ // commit. The Context must not turn that already-durable row into an
2467
+ // empty successful dataset. Verify ownership through the runner's
2468
+ // preloaded session (never by minting a new database authority), then
2469
+ // return only the same-run terminal keys as idempotent settlements.
2470
+ const recovered = await retryRuntimeSheetPersistence({
2471
+ description: `runtime map stale settlement reconcile table=${input.tableNamespace}`,
2472
+ log: (line) => emitWorkerLog(line, options),
2473
+ signal: persistenceSignal,
2474
+ run: () =>
2475
+ readRuntimeSheetDatasetRowKeys(runtimeSheetContext, {
2476
+ tableNamespace: input.tableNamespace,
2477
+ runId,
2478
+ keys: [...staleKeys],
2479
+ rowMode: 'all',
2480
+ }),
2481
+ });
2482
+ recoveredStaleKeys = recovered.keys;
2483
+ for (const key of recoveredStaleKeys) staleKeys.delete(key);
2484
+ }
2485
+ if (staleDropped > 0) {
2486
+ emitWorkerLog(
2487
+ `[info] dropped ${staleKeys.size} stale runtime sheet write(s) for table=${input.tableNamespace}; ${recoveredStaleKeys.length} same-run terminal write(s) were accepted idempotently`,
2488
+ options,
2489
+ );
2490
+ }
2491
+ const failedRows = input.rows.filter(
2492
+ (row) =>
2493
+ row.status === 'failed' &&
2494
+ !staleKeys.has(resolveMapRowOutcomeKey(row) ?? String(row.key ?? '')),
2495
+ ).length;
2496
+ // The fenced commit response is the persistence barrier. An exact-version
2497
+ // replay may report zero physical updates while still being an accepted,
2498
+ // durable settlement, so derive accepted rows from submitted minus stale
2499
+ // keys instead of issuing a read-after-write visibility audit.
2500
+ const committedTerminalRows = result.committedKeys.length;
2501
+ const succeededRows = Math.max(0, committedTerminalRows - failedRows);
2502
+ const datasetPath = `datasets.${input.tableNamespace}`;
2503
+ const datasetId = createRuntimeDatasetId(playName, input.tableNamespace);
2504
+ const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
2505
+ datasetId,
2506
+ path: datasetPath,
2507
+ tableNamespace: input.tableNamespace,
2508
+ runId,
2509
+ persistedRows: 0,
2510
+ succeededRows: 0,
2511
+ failedRows: 0,
2512
+ };
2513
+ // Only exact-version accepted rows belong to this run's lifecycle
2514
+ // settlement. Stale rows belong to the newer writer and stay excluded.
2515
+ settlement.persistedRows += committedTerminalRows;
2516
+ settlement.succeededRows += succeededRows;
2517
+ settlement.failedRows += failedRows;
2518
+ runtimeMapDatasetSettlements.set(datasetId, settlement);
2519
+ emitWorkerLog(
2520
+ `[perf] runtime map persist table=${input.tableNamespace} rows=${input.rows.length} updated=${result.updated} accepted=${committedTerminalRows} stale=${staleKeys.size} elapsed_ms=${Date.now() - persistStartedAt}`,
2521
+ options,
2522
+ );
2523
+ return {
2524
+ updated: result.updated,
2525
+ staleDroppedKeys: [...staleKeys],
2526
+ ...(recoveredStaleKeys.length > 0 ? { recoveredStaleKeys } : {}),
2527
+ };
2528
+ },
2529
+ });
2530
+
2531
+ const csvSourceCache = new Map<string, ReplayableCsvSource>();
2532
+ const csvDatasetCache = new Map<
2533
+ string,
2534
+ import('../../plays/dataset').PlayDataset<Record<string, unknown>>
2535
+ >();
2536
+ const csvDatasetNamespaceByCacheKey = new Map<string, string>();
2537
+ const runtimeInputAlias = config.runtimeInputAlias ?? null;
2538
+ const csvSourcePath = config.csvSourcePath ?? null;
2539
+ const csvSourceUrl = config.csvSourceUrl ?? null;
2540
+ const csvSourceContentBase64 = config.csvSourceContentBase64 ?? null;
2541
+
2542
+ (
2543
+ ctx as unknown as {
2544
+ csv: (
2545
+ requestedPath: string,
2546
+ options?: CsvRenameOptions,
2547
+ ) => Promise<
2548
+ import('../../plays/dataset').PlayDataset<Record<string, unknown>>
2549
+ >;
2550
+ }
2551
+ ).csv = async (requestedPath: string, csvOptions?: CsvRenameOptions) => {
2552
+ if (csvOptions) {
2553
+ for (const [field, value] of [
2554
+ ['ctx.csv.options.columns', csvOptions.columns],
2555
+ ['ctx.csv.options.rename', csvOptions.rename],
2556
+ ['ctx.csv.options.required', csvOptions.required],
2557
+ ] as const) {
2558
+ validateOptionalPlayAuthoringField(field, value);
2559
+ }
2560
+ }
2561
+ const resolvedRequestedPath =
2562
+ typeof requestedPath === 'string' && requestedPath.trim().length > 0
2563
+ ? requestedPath
2564
+ : runtimeInputAlias;
2565
+
2566
+ if (!resolvedRequestedPath) {
2567
+ throw new Error(
2568
+ 'ctx.csv(...) requires a path, or the play run must provide an input file.',
2569
+ );
2570
+ }
2571
+
2572
+ const shouldUseInlineCsvSource =
2573
+ Boolean(csvSourceContentBase64) &&
2574
+ (!runtimeInputAlias || resolvedRequestedPath === runtimeInputAlias);
2575
+ const shouldUseCsvSourceUrl =
2576
+ Boolean(csvSourceUrl) &&
2577
+ (!runtimeInputAlias || resolvedRequestedPath === runtimeInputAlias);
2578
+ const targetPath = shouldUseInlineCsvSource
2579
+ ? `inline:${resolvedRequestedPath}`
2580
+ : shouldUseCsvSourceUrl
2581
+ ? csvSourceUrl!
2582
+ : (config.materializedFiles[resolvedRequestedPath] ??
2583
+ config.materializedFiles[
2584
+ resolvedRequestedPath.replace(/^\.\//, '')
2585
+ ] ??
2586
+ resolve(config.workspaceRoot, resolvedRequestedPath));
2587
+
2588
+ if (
2589
+ !shouldUseInlineCsvSource &&
2590
+ !shouldUseCsvSourceUrl &&
2591
+ !targetPath.startsWith(config.workspaceRoot)
2592
+ ) {
2593
+ throw new Error(
2594
+ `ctx.csv("${resolvedRequestedPath}") resolved outside the packaged play workspace.`,
2595
+ );
2596
+ }
2597
+
2598
+ if (!csvSourceCache.has(targetPath)) {
2599
+ try {
2600
+ csvSourceCache.set(
2601
+ targetPath,
2602
+ createCsvSource(
2603
+ shouldUseInlineCsvSource
2604
+ ? { contentBase64: csvSourceContentBase64! }
2605
+ : shouldUseCsvSourceUrl
2606
+ ? { url: targetPath }
2607
+ : { path: targetPath },
2608
+ ),
2609
+ );
2610
+ } catch {
2611
+ const knownFiles = Object.keys(config.materializedFiles)
2612
+ .filter((key) => !key.includes('/'))
2613
+ .sort()
2614
+ .join(', ');
2615
+ throw new Error(
2616
+ `ctx.csv("${resolvedRequestedPath}") could not be resolved in the packaged play workspace. ` +
2617
+ `Known packaged files: ${knownFiles || 'none'}.`,
2618
+ );
2619
+ }
2620
+ }
2621
+
2622
+ const csvProjectionIdentity = `${JSON.stringify(csvOptions?.columns ?? {})}:${JSON.stringify(csvOptions?.rename ?? {})}:${JSON.stringify(csvOptions?.required ?? [])}`;
2623
+ const datasetCacheKey = `${resolvedRequestedPath}:${targetPath}:${csvProjectionIdentity}`;
2624
+ let tableNamespace = csvDatasetNamespaceByCacheKey.get(datasetCacheKey);
2625
+ if (!tableNamespace) {
2626
+ const baseTableNamespace = `${CSV_NAMESPACE_PREFIX}_${createHash('sha1').update(resolvedRequestedPath).digest('hex').slice(0, 12)}`;
2627
+ const baseAlreadyAssigned = [
2628
+ ...csvDatasetNamespaceByCacheKey.values(),
2629
+ ].some((assigned) => assigned === baseTableNamespace);
2630
+ tableNamespace = baseAlreadyAssigned
2631
+ ? `${baseTableNamespace}_${createHash('sha1').update(csvProjectionIdentity).digest('hex').slice(0, 12)}`
2632
+ : baseTableNamespace;
2633
+ csvDatasetNamespaceByCacheKey.set(datasetCacheKey, tableNamespace);
2634
+ }
2635
+
2636
+ assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
2637
+
2638
+ assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
2639
+ const cachedDataset = csvDatasetCache.get(datasetCacheKey);
2640
+ if (cachedDataset) {
2641
+ return cachedDataset;
2642
+ }
2643
+
2644
+ const playName = config.context.playName;
2645
+ const dataset = await createStreamingCsvDataset({
2646
+ playName,
2647
+ tableNamespace,
2648
+ source: csvSourceCache.get(targetPath)!,
2649
+ sourceLabel: resolvedRequestedPath,
2650
+ options: csvOptions,
2651
+ });
2652
+ const sheetContract = inferCsvSheetContract({
2653
+ tableNamespace,
2654
+ rows: await dataset.peek(10),
2655
+ });
2656
+ runtimeCsvDatasets.set(dataset.datasetId, {
2657
+ tableNamespace,
2658
+ dataset,
2659
+ sheetContract,
2660
+ sourceLabel: resolvedRequestedPath,
2661
+ });
2662
+ csvDatasetCache.set(datasetCacheKey, dataset);
2663
+ return dataset;
2664
+ };
2665
+ emitWorkerLog(
2666
+ `[perf] context setup elapsed_ms=${Date.now() - contextSetupStartedAt}`,
2667
+ options,
2668
+ );
2669
+
2670
+ const playInput: PlayRunInputPayload =
2671
+ runtimeInputAlias && !Array.isArray(runtimeInput)
2672
+ ? {
2673
+ ...runtimeInputObject,
2674
+ file: runtimeInputAlias,
2675
+ }
2676
+ : runtimeInput;
2677
+
2678
+ try {
2679
+ emitWorkerLog('Play started', options);
2680
+ const rawResult = await executeRuntimePlayCode({
2681
+ artifact: config.artifact,
2682
+ ctx,
2683
+ runtimeInput: playInput,
2684
+ options,
2685
+ });
2686
+ // A durable ctx boundary deliberately unwinds execution. Customer code can
2687
+ // catch that error, so the context keeps a sticky handoff that remains
2688
+ // authoritative until this invocation returns it to the scheduler.
2689
+ ctx.assertNoPendingDurableSuspension();
2690
+ emitWorkerLog(
2691
+ `[perf] play body returned elapsed_ms=${Date.now() - runStartedAt}`,
2692
+ options,
2693
+ );
2694
+ for (const settlement of runtimeMapDatasetSettlements.values()) {
2695
+ emitRunnerEvent(
2696
+ {
2697
+ type: 'execution_event',
2698
+ event: {
2699
+ type: 'dataset.lifecycle',
2700
+ datasetId: settlement.datasetId,
2701
+ path: settlement.path,
2702
+ tableNamespace: settlement.tableNamespace,
2703
+ phase: 'available',
2704
+ persistedRows: settlement.persistedRows,
2705
+ succeededRows: settlement.succeededRows,
2706
+ failedRows: settlement.failedRows,
2707
+ complete: true,
2708
+ at: Date.now(),
2709
+ },
2710
+ },
2711
+ options,
2712
+ );
2713
+ emitRunnerEvent(
2714
+ {
2715
+ type: 'log',
2716
+ at: new Date().toISOString(),
2717
+ source: 'play',
2718
+ line: formatDatasetAvailabilityLog({
2719
+ runId: settlement.runId,
2720
+ dataset: {
2721
+ path: settlement.path,
2722
+ tableNamespace: settlement.tableNamespace,
2723
+ },
2724
+ persistedRows: settlement.persistedRows,
2725
+ }),
2726
+ },
2727
+ options,
2728
+ );
2729
+ }
2730
+ await receiptStoreAdapter?.close();
2731
+ const result = normalizeTopLevelPlayReturn(rawResult);
2732
+ for (const returnedDataset of collectReturnedDatasets(result)) {
2733
+ if (!isPlayDataset<Record<string, unknown>>(returnedDataset)) {
2734
+ continue;
2735
+ }
2736
+ const csvDataset = runtimeCsvDatasets.get(returnedDataset.datasetId);
2737
+ if (!csvDataset) {
2738
+ continue;
2739
+ }
2740
+ const persistStartedAt = Date.now();
2741
+ const csvRunId =
2742
+ config.context.runId ?? config.context.workflowId ?? 'anonymous-run';
2743
+ const csvRuntimeContext = {
2744
+ executorToken: config.context.executorToken,
2745
+ baseUrl: config.context.baseUrl,
2746
+ vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
2747
+ orgId: config.context.orgId,
2748
+ playName: config.context.playName!,
2749
+ runId: csvRunId,
2750
+ userEmail: config.context.userEmail,
2751
+ dbSessionStrategy: config.context.dbSessionStrategy,
2752
+ preloadedDbSessions: runtimePreloadedDbSessions,
2753
+ postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
2754
+ abortSignal: persistenceSignal,
2755
+ };
2756
+ let csvRowsPersisted = 0;
2757
+ for await (const page of iteratePlayDatasetInputPages(
2758
+ csvDataset.dataset,
2759
+ )) {
2760
+ await retryRuntimeSheetPersistence({
2761
+ description: `runtime csv persist table=${csvDataset.tableNamespace}`,
2762
+ log: (line) => emitWorkerLog(line, options),
2763
+ signal: persistenceSignal,
2764
+ run: () =>
2765
+ startRuntimeSheetDataset(csvRuntimeContext, {
2766
+ playName: config.context.playName!,
2767
+ tableNamespace: csvDataset.tableNamespace,
2768
+ playInput: runtimeInputObject,
2769
+ sheetContract: csvDataset.sheetContract,
2770
+ rows: page.rows,
2771
+ runId: csvRunId,
2772
+ inputOffset: page.offset,
2773
+ }),
2774
+ });
2775
+ csvRowsPersisted += page.rows.length;
2776
+ page.rows.length = 0;
2777
+ }
2778
+ emitWorkerLog(
2779
+ `[perf] runtime csv persist table=${csvDataset.tableNamespace} rows=${csvRowsPersisted} elapsed_ms=${Date.now() - persistStartedAt}`,
2780
+ options,
2781
+ );
2782
+ const csvDatasetPath = `datasets.${csvDataset.tableNamespace}`;
2783
+ emitRunnerEvent(
2784
+ {
2785
+ type: 'execution_event',
2786
+ event: {
2787
+ type: 'dataset.lifecycle',
2788
+ datasetId: createRuntimeDatasetId(
2789
+ config.context.playName!,
2790
+ csvDataset.tableNamespace,
2791
+ ),
2792
+ path: csvDatasetPath,
2793
+ tableNamespace: csvDataset.tableNamespace,
2794
+ phase: 'available',
2795
+ persistedRows: csvRowsPersisted,
2796
+ succeededRows: csvRowsPersisted,
2797
+ failedRows: 0,
2798
+ complete: true,
2799
+ at: Date.now(),
2800
+ },
2801
+ },
2802
+ options,
2803
+ );
2804
+ emitRunnerEvent(
2805
+ {
2806
+ type: 'log',
2807
+ at: new Date().toISOString(),
2808
+ source: 'play',
2809
+ line: formatDatasetAvailabilityLog({
2810
+ runId: csvRunId,
2811
+ dataset: {
2812
+ path: csvDatasetPath,
2813
+ tableNamespace: csvDataset.tableNamespace,
2814
+ },
2815
+ persistedRows: csvRowsPersisted,
2816
+ }),
2817
+ },
2818
+ options,
2819
+ );
2820
+ }
2821
+ emitWorkerLog(`Play finished in ${Date.now() - runStartedAt}ms`, options);
2822
+ const normalizeStartedAt = Date.now();
2823
+ const outputRows = normalizeOutputRows(result);
2824
+ const outputRowCount = deriveOutputRowCount(result, outputRows);
2825
+ emitWorkerLog(
2826
+ `[perf] output normalize rows=${outputRowCount} elapsed_ms=${Date.now() - normalizeStartedAt}`,
2827
+ options,
2828
+ );
2829
+ const serializeStartedAt = Date.now();
2830
+ const projectedResult = projectPlayResultForTransport(result);
2831
+ const compactedResult = projectedResult.output;
2832
+ const serializedResult = serializePlayResult(compactedResult);
2833
+ emitWorkerLog(
2834
+ `[perf] result serialize elapsed_ms=${Date.now() - serializeStartedAt}`,
2835
+ options,
2836
+ );
2837
+ if (!totalRows) {
2838
+ totalRows = outputRowCount;
2839
+ }
2840
+ const resultTableNamespace = extractResultTableNamespace(serializedResult);
2841
+ ctx.recordReturn(outputRowCount);
2842
+
2843
+ await releaseRuntimeLeasesOnAppRunnerSettlement({
2844
+ config,
2845
+ runtimeSheetAttempts,
2846
+ runnerRunAttempt,
2847
+ preloadedDbSessions: runtimePreloadedDbSessions,
2848
+ postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
2849
+ emitWorkerLog: (message) => emitWorkerLog(message, options),
2850
+ reason: 'completed',
2851
+ });
2852
+
2853
+ const logs = ctx.getLogs();
2854
+ const runtimeDiagnostics = Array.from(
2855
+ new Map(
2856
+ logs
2857
+ .map((line) => parseCtxFetchHttpFailureDiagnostic(line))
2858
+ .filter((diagnostic): diagnostic is CtxFetchHttpFailureDiagnostic =>
2859
+ Boolean(diagnostic),
2860
+ )
2861
+ .map((diagnostic) => [
2862
+ `${diagnostic.key}\u0000${diagnostic.method}\u0000${diagnostic.url}\u0000${diagnostic.http_status}`,
2863
+ diagnostic,
2864
+ ]),
2865
+ ).values(),
2866
+ ).slice(0, 16);
2867
+
2868
+ return {
2869
+ status: 'completed' as const,
2870
+ output: serializedResult,
2871
+ ...(projectedResult.warnings.length > 0
2872
+ ? { outputWarnings: projectedResult.warnings }
2873
+ : {}),
2874
+ outputRowCount,
2875
+ logs,
2876
+ ...(runtimeDiagnostics.length > 0 ? { runtimeDiagnostics } : {}),
2877
+ stats: ctx.getStats(),
2878
+ steps: ctx.getSteps(),
2879
+ checkpoint: ctx.getCheckpoint(),
2880
+ tableNamespace: resultTableNamespace,
2881
+ totalRows,
2882
+ inserted,
2883
+ skipped,
2884
+ };
2885
+ } catch (caughtError) {
2886
+ // `ctx.sleep()` deliberately unwinds with an exception. Customer code may
2887
+ // catch it and throw a different exception instead, but the invocation's
2888
+ // sticky handoff is still authoritative until this runner returns it to
2889
+ // the scheduler. Resolve that precedence before classifying the terminal
2890
+ // result so a replacement error cannot turn a suspended run into failed.
2891
+ let error: unknown = caughtError;
2892
+ try {
2893
+ ctx.assertNoPendingDurableSuspension();
2894
+ } catch (suspension) {
2895
+ error = suspension;
2896
+ }
2897
+ try {
2898
+ await receiptStoreAdapter?.close();
2899
+ } catch (closeError) {
2900
+ emitWorkerLog(
2901
+ `[warn] runtime receipt writer close failed during teardown: ${
2902
+ closeError instanceof Error ? closeError.message : String(closeError)
2903
+ }`,
2904
+ options,
2905
+ );
2906
+ }
2907
+ if (isPlayExecutionSuspendedError(error)) {
2908
+ // Suspension is a controlled handoff to the scheduler. Promise.all siblings may
2909
+ // still hold runtime receipt leases when the first branch suspends; release
2910
+ // this attempt's leases so the resumed task can reclaim and reconcile
2911
+ // already-terminal child work instead of waiting for stale running receipts.
2912
+ await releaseRuntimeLeasesOnAppRunnerSettlement({
2913
+ config,
2914
+ runtimeSheetAttempts,
2915
+ runnerRunAttempt,
2916
+ preloadedDbSessions: runtimePreloadedDbSessions,
2917
+ postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
2918
+ emitWorkerLog: (message) => emitWorkerLog(message, options),
2919
+ reason: 'suspension',
2920
+ });
2921
+ return {
2922
+ status: 'suspended',
2923
+ suspension: error.suspension,
2924
+ logs: ctx.getLogs(),
2925
+ stats: ctx.getStats(),
2926
+ steps: ctx.getSteps(),
2927
+ checkpoint: ctx.getCheckpoint(),
2928
+ tableNamespace: null,
2929
+ totalRows,
2930
+ inserted,
2931
+ skipped,
2932
+ };
2933
+ }
2934
+
2935
+ // Controlled run-fatal teardown: release the runtime-sheet row leases and
2936
+ // work-receipt leases this attempt still holds so an immediate rerun is not
2937
+ // blocked for the full lease TTL. Best-effort but LOUD — a release failure
2938
+ // is logged and never masks the original run-fatal error. TTL expiry stays
2939
+ // the recovery path for true crashes where this teardown never runs.
2940
+ await releaseRuntimeLeasesOnAppRunnerSettlement({
2941
+ config,
2942
+ runtimeSheetAttempts,
2943
+ runnerRunAttempt,
2944
+ preloadedDbSessions: runtimePreloadedDbSessions,
2945
+ postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
2946
+ emitWorkerLog: (message) => emitWorkerLog(message, options),
2947
+ reason: 'run-fatal',
2948
+ });
2949
+
2950
+ ensureSourceMapSupport();
2951
+ const failure = normalizePlayRunFailure(error);
2952
+ return {
2953
+ status: 'failed',
2954
+ error: formatRunnerFailure(error),
2955
+ errors: [failure],
2956
+ logs: ctx.getLogs(),
2957
+ stats: ctx.getStats(),
2958
+ steps: ctx.getSteps(),
2959
+ checkpoint: ctx.getCheckpoint(),
2960
+ tableNamespace: null,
2961
+ totalRows,
2962
+ inserted,
2963
+ skipped,
2964
+ };
2965
+ }
2966
+ }
2967
+
2968
+ export async function executePlayRunner(
2969
+ config: PlayRunnerExecutionConfig,
2970
+ options?: ExecutePlayRunnerOptions,
2971
+ ): Promise<PlayRunnerResult> {
2972
+ const result = await run(config, options);
2973
+ // Inline-executor mirror of the child terminal drain: the caller's finalize
2974
+ // reads the observations table right after this returns, so land in-flight
2975
+ // observation posts first (bounded, best-effort, docflow-gated by emptiness).
2976
+ await drainPendingDocflowObservationPosts(options);
2977
+ return result;
2978
+ }
2979
+
2980
+ function normalizeOutputRows(result: unknown): Record<string, unknown>[] {
2981
+ if (result && typeof result === 'object') {
2982
+ const record = result as Record<string, unknown>;
2983
+ if (isPlayDataset<Record<string, unknown>>(record.rows)) {
2984
+ return [];
2985
+ }
2986
+ if (isPlayDataset<Record<string, unknown>>(record.results)) {
2987
+ return [];
2988
+ }
2989
+ if (isPlayDataset<Record<string, unknown>>(record.leads)) {
2990
+ return [];
2991
+ }
2992
+ if (isPlayDataset<Record<string, unknown>>(record.contacts)) {
2993
+ return [];
2994
+ }
2995
+ if (Array.isArray(record.leads)) {
2996
+ return record.leads as Record<string, unknown>[];
2997
+ }
2998
+ if (Array.isArray(record.contacts)) {
2999
+ return record.contacts as Record<string, unknown>[];
3000
+ }
3001
+ if (Array.isArray(record.rows)) {
3002
+ return record.rows as Record<string, unknown>[];
3003
+ }
3004
+ if (Array.isArray(record.results)) {
3005
+ return record.results as Record<string, unknown>[];
3006
+ }
3007
+ return [record];
3008
+ }
3009
+
3010
+ return [];
3011
+ }
3012
+
3013
+ type PlayResultTransportProjection = {
3014
+ output: Record<string, unknown>;
3015
+ warnings: PlayRunOutputWarning[];
3016
+ };
3017
+
3018
+ function normalizeTopLevelPlayReturn(value: unknown): Record<string, unknown> {
3019
+ if (value === null || value === undefined) {
3020
+ return {};
3021
+ }
3022
+ if (Array.isArray(value)) {
3023
+ return { rows: value };
3024
+ }
3025
+ if (isPlayDataset(value) || isSerializedPlayDataset(value)) {
3026
+ return { rows: value };
3027
+ }
3028
+ if (typeof value !== 'object') {
3029
+ return { value };
3030
+ }
3031
+
3032
+ const record = { ...(value as Record<string, unknown>) };
3033
+ if (Object.prototype.hasOwnProperty.call(record, '_metadata')) {
3034
+ record.metadata = record._metadata;
3035
+ delete record._metadata;
3036
+ }
3037
+ return record;
3038
+ }
3039
+
3040
+ export function compactPlayResultForTransport(value: unknown): unknown {
3041
+ return projectPlayResultForTransport(value).output;
3042
+ }
3043
+
3044
+ export function projectPlayResultForTransport(
3045
+ value: unknown,
3046
+ ): PlayResultTransportProjection {
3047
+ const record = normalizeTopLevelPlayReturn(value);
3048
+ const output: Record<string, unknown> = {};
3049
+
3050
+ for (const [key, entry] of Object.entries(record)) {
3051
+ if (isPlayDataset(entry)) {
3052
+ output[key] = entry.toJSON();
3053
+ continue;
3054
+ }
3055
+ if (isSerializedPlayDataset(entry)) {
3056
+ output[key] = entry;
3057
+ continue;
3058
+ }
3059
+ output[key] = entry;
3060
+ }
3061
+
3062
+ assertCustomerPlayReturnWithinLimit({
3063
+ value: output,
3064
+ });
3065
+ return { output, warnings: [] };
3066
+ }
3067
+
3068
+ function serializePlayResult<T>(value: T): T {
3069
+ return JSON.parse(JSON.stringify(value)) as T;
3070
+ }
3071
+
3072
+ export function deriveOutputRowCount(
3073
+ result: unknown,
3074
+ normalizedRows: Record<string, unknown>[],
3075
+ ): number {
3076
+ if (result && typeof result === 'object' && !Array.isArray(result)) {
3077
+ const record = result as Record<string, unknown>;
3078
+ if (isPlayDataset(record.rows)) {
3079
+ return record.rows.toJSON().count;
3080
+ }
3081
+ if (isPlayDataset(record.results)) {
3082
+ return record.results.toJSON().count;
3083
+ }
3084
+ if (isPlayDataset(record.leads)) {
3085
+ return record.leads.toJSON().count;
3086
+ }
3087
+ if (isPlayDataset(record.contacts)) {
3088
+ return record.contacts.toJSON().count;
3089
+ }
3090
+ if (isSerializedPlayDataset(record.rows)) {
3091
+ return record.rows.count;
3092
+ }
3093
+ if (isSerializedPlayDataset(record.results)) {
3094
+ return record.results.count;
3095
+ }
3096
+ if (isSerializedPlayDataset(record.leads)) {
3097
+ return record.leads.count;
3098
+ }
3099
+ if (isSerializedPlayDataset(record.contacts)) {
3100
+ return record.contacts.count;
3101
+ }
3102
+ }
3103
+ return normalizedRows.length;
3104
+ }
3105
+
3106
+ export function extractResultTableNamespace(result: unknown): string | null {
3107
+ if (result && typeof result === 'object' && !Array.isArray(result)) {
3108
+ const record = result as Record<string, unknown>;
3109
+ for (const candidate of [
3110
+ record.rows,
3111
+ record.results,
3112
+ record.leads,
3113
+ record.contacts,
3114
+ ]) {
3115
+ if (isPlayDataset(candidate)) {
3116
+ return candidate.toJSON().tableNamespace ?? null;
3117
+ }
3118
+ }
3119
+ for (const candidate of [
3120
+ record.rows,
3121
+ record.results,
3122
+ record.leads,
3123
+ record.contacts,
3124
+ ]) {
3125
+ if (isSerializedPlayDataset(candidate)) {
3126
+ return candidate.tableNamespace ?? null;
3127
+ }
3128
+ }
3129
+ }
3130
+ return null;
3131
+ }
3132
+
3133
+ async function waitForConfigFile(configPath: string): Promise<void> {
3134
+ const startedAt = Date.now();
3135
+ for (;;) {
3136
+ try {
3137
+ await readFile(configPath);
3138
+ return;
3139
+ } catch (error) {
3140
+ const code = (error as { code?: unknown }).code;
3141
+ if (code !== 'ENOENT') {
3142
+ throw error;
3143
+ }
3144
+ if (Date.now() - startedAt > WAIT_CONFIG_TIMEOUT_MS) {
3145
+ throw new Error(
3146
+ `Timed out waiting for play runner config at ${configPath}.`,
3147
+ );
3148
+ }
3149
+ await new Promise((resolve) => setTimeout(resolve, WAIT_CONFIG_POLL_MS));
3150
+ }
3151
+ }
3152
+ }
3153
+
3154
+ export async function materializeTransportedArtifact(
3155
+ config: PlayRunnerExecutionConfig,
3156
+ ): Promise<PlayRunnerExecutionConfig> {
3157
+ const bundledCodePath = config.artifactTransport?.bundledCodePath?.trim();
3158
+ if (!bundledCodePath) {
3159
+ if (!config.artifact.bundledCode) {
3160
+ throw new Error('Play runner config is missing artifact bundledCode.');
3161
+ }
3162
+ return config;
3163
+ }
3164
+
3165
+ const artifactBuffer = await readFile(bundledCodePath);
3166
+ const bundledCode =
3167
+ config.artifactTransport?.bundledCodeEncoding === 'gzip'
3168
+ ? gunzipSync(artifactBuffer).toString('utf-8')
3169
+ : artifactBuffer.toString('utf-8');
3170
+ if (!bundledCode) {
3171
+ throw new Error(
3172
+ `Play runner artifact file ${bundledCodePath} did not contain bundledCode.`,
3173
+ );
3174
+ }
3175
+ const sourceMapPath = config.artifactTransport?.sourceMapPath?.trim();
3176
+ const sourceMap = sourceMapPath
3177
+ ? await readFile(sourceMapPath, 'utf-8')
3178
+ : config.artifact.sourceMap;
3179
+
3180
+ return {
3181
+ ...config,
3182
+ artifact: {
3183
+ ...config.artifact,
3184
+ bundledCode,
3185
+ sourceMap,
3186
+ },
3187
+ };
3188
+ }
3189
+
3190
+ async function main() {
3191
+ emitWorkerLog(
3192
+ `[perf] process main elapsed_ms=${Math.round(performance.now())}`,
3193
+ );
3194
+ const waitForConfig = process.argv[2] === '--wait-config';
3195
+ const configPath = waitForConfig ? process.argv[3] : process.argv[2];
3196
+ if (!configPath) {
3197
+ throw new Error(
3198
+ 'Usage: node play-runner.cjs [--wait-config] <config.json|config.json.gz>',
3199
+ );
3200
+ }
3201
+ if (waitForConfig) {
3202
+ await waitForConfigFile(configPath);
3203
+ }
3204
+ const configLoadStartedAt = Date.now();
3205
+ const configBuffer = await readFile(configPath);
3206
+ const configJson = configPath.endsWith('.gz')
3207
+ ? gunzipSync(configBuffer).toString('utf-8')
3208
+ : configBuffer.toString('utf-8');
3209
+ const parsedConfig = JSON.parse(configJson) as PlayRunnerExecutionConfig;
3210
+ writeRunnerStartupDiagnostic(
3211
+ process.env.DEEPLINE_PLAY_RUNNER_STARTUP_DIAGNOSTIC_PATH,
3212
+ { phase: 'booted' },
3213
+ );
3214
+ emitWorkerLog(
3215
+ `[perf] config load elapsed_ms=${Date.now() - configLoadStartedAt}`,
3216
+ );
3217
+
3218
+ // Set terminal push wiring before any startup work that can fail. A failed
3219
+ // artifact materialization can then report its terminal whenever the gateway
3220
+ // is reachable instead of relying only on the detached ceiling salvage.
3221
+ runnerTerminalPushContext = terminalPushContextFromConfig(parsedConfig);
3222
+ const executionController = new AbortController();
3223
+ const heartbeat = startRunnerPushHeartbeatFromConfig(parsedConfig, () => {
3224
+ executionController.abort(
3225
+ new Error('Play runner stopped by the runtime heartbeat channel.'),
3226
+ );
3227
+ });
3228
+ try {
3229
+ const artifactMaterializeStartedAt = Date.now();
3230
+ const config = await materializeTransportedArtifactWhileRunnerParks(
3231
+ parsedConfig,
3232
+ heartbeat,
3233
+ );
3234
+ emitWorkerLog(
3235
+ `[perf] artifact materialize elapsed_ms=${Date.now() - artifactMaterializeStartedAt}`,
3236
+ );
3237
+
3238
+ runnerProgressPusher = startRunnerProgressPusherFromConfig(config);
3239
+ const runtimeDeadline = startConfiguredPlayRunnerRuntimeDeadline({
3240
+ config,
3241
+ onDeadline: async (runtimeLimitSeconds) => {
3242
+ executionController.abort(
3243
+ new Error(
3244
+ `Play runner reached its ${runtimeLimitSeconds}-second runtime limit.`,
3245
+ ),
3246
+ );
3247
+ heartbeat?.stop();
3248
+ await terminateRunnerForRuntimeDeadline(runtimeLimitSeconds);
3249
+ },
3250
+ onError: (error) => {
3251
+ console.error('[play-runner.runtime-deadline] terminal_failed', {
3252
+ error: error instanceof Error ? error.message : String(error),
3253
+ });
3254
+ process.exit(1);
3255
+ },
3256
+ });
3257
+ writeRunnerRuntimeStartedMarker(
3258
+ process.env.DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH,
3259
+ );
3260
+ let result: PlayRunnerResult | null = null;
3261
+ let executionFailed = false;
3262
+ let executionError: unknown = undefined;
3263
+ try {
3264
+ result = await executePlayRunner(config, {
3265
+ signal: executionController.signal,
3266
+ });
3267
+ } catch (error) {
3268
+ executionFailed = true;
3269
+ executionError = error;
3270
+ }
3271
+ if (!runtimeDeadline.claimCompletion()) {
3272
+ await runtimeDeadline.deadlineSettled;
3273
+ throw new Error(
3274
+ 'Runtime deadline handler returned without terminating the runner.',
3275
+ );
3276
+ }
3277
+ // claimCompletion() is the exact customer-execution boundary. Publish the
3278
+ // external fence immediately, before formatting failures or serializing the
3279
+ // terminal payload, so a run that finishes at the deadline receives the
3280
+ // terminal-settlement grace instead of being reclassified as a timeout.
3281
+ const workCompletedAtMs = Date.now();
3282
+ writeRunnerRuntimeCompletedMarker(
3283
+ process.env.DEEPLINE_PLAY_RUNNER_RUNTIME_COMPLETED_PATH,
3284
+ { at: workCompletedAtMs },
3285
+ );
3286
+ if (executionFailed) {
3287
+ ensureSourceMapSupport();
3288
+ const message = formatRunnerFailure(executionError);
3289
+ result = {
3290
+ status: 'failed',
3291
+ error: message,
3292
+ errors: [normalizePlayRunFailure(executionError)],
3293
+ logs: [],
3294
+ stats: {},
3295
+ steps: [],
3296
+ tableNamespace: null,
3297
+ };
3298
+ }
3299
+ if (!result) {
3300
+ throw new Error('Play runner completed without a result.');
3301
+ }
3302
+ (
3303
+ result as PlayRunnerResult & { workCompletedAtMs?: number }
3304
+ ).workCompletedAtMs = workCompletedAtMs;
3305
+ const terminalResult = runnerTerminalPushContext
3306
+ ? runnerTerminalResultForTransport(result)
3307
+ : null;
3308
+ heartbeat?.stop();
3309
+ // Ordering guarantee for the durable trace (ADR 0016): the pushed terminal
3310
+ // wakes the parked worker whose finalize reads the observations table, so
3311
+ // in-flight observation posts must land first. Bounded — never fails or
3312
+ // delays the terminal beyond the cap; no-op for undiagrammed plays.
3313
+ // Runner progress joins the same terminal handoff immediately below.
3314
+ await drainPendingDocflowObservationPosts();
3315
+ if (terminalResult) {
3316
+ // Persist the exact terminal line locally before any network settlement.
3317
+ // If the gateway remains unavailable through the grace period, the shell
3318
+ // epilogue can still salvage this authored result instead of inventing a
3319
+ // runtime-limit failure after customer code already finished.
3320
+ await writeRunnerEventToStdout({
3321
+ type: 'result',
3322
+ result: terminalResult,
3323
+ });
3324
+ }
3325
+ // Push execution (B2-final): the pushed terminal is what WAKES the parked
3326
+ // worker attempt. Push before exiting; on exhaustion the captured output
3327
+ // file remains the ceiling-timeout salvage source.
3328
+ if (runnerTerminalPushContext && terminalResult) {
3329
+ // Every terminal boundary can wake a successor attempt. Transfer all
3330
+ // unacknowledged observations into that same ordered handoff before the
3331
+ // wake is visible; draining a suspended/failed attempt afterward lets
3332
+ // the successor claim the log channel and fence its predecessor's tail.
3333
+ const finalProgress = runnerProgressPusher?.sealForTerminal() ?? {
3334
+ events: [],
3335
+ liveLogOffset: 0,
3336
+ };
3337
+ runnerProgressPusher = null;
3338
+ await pushRunnerTerminalIfConfigured(terminalResult, {
3339
+ warn: createPostTerminalDiagnosticWriter(),
3340
+ finalProgressLogOffset: finalProgress.liveLogOffset,
3341
+ ...(finalProgress.events.length
3342
+ ? {
3343
+ finalProgressEvents: finalProgress.events,
3344
+ }
3345
+ : {}),
3346
+ });
3347
+ return exitRunner(0);
3348
+ }
3349
+ await runnerProgressPusher?.stop();
3350
+ runnerProgressPusher = null;
3351
+ // The parent process owns play status from the structured result event.
3352
+ // Keep the process exit clean so remote command transports do not drop the
3353
+ // final JSON line on non-zero exits.
3354
+ await emitAndExit(
3355
+ {
3356
+ type: 'result',
3357
+ result,
3358
+ },
3359
+ 0,
3360
+ );
3361
+ } finally {
3362
+ heartbeat?.stop();
3363
+ const pusher = runnerProgressPusher;
3364
+ runnerProgressPusher = null;
3365
+ await pusher?.stop().catch(() => undefined);
3366
+ }
3367
+ }
3368
+
3369
+ export async function waitForRunnerExecutionAcknowledgement(
3370
+ heartbeat: ReturnType<typeof startRunnerPushHeartbeatFromConfig>,
3371
+ timeoutMs = RUNNER_EXECUTION_ACK_TIMEOUT_MS,
3372
+ ): Promise<void> {
3373
+ if (!heartbeat) return;
3374
+ const deadline = Date.now() + Math.max(1, timeoutMs);
3375
+ let retryDelayMs = 100;
3376
+ while (Date.now() < deadline) {
3377
+ heartbeat.requestHeartbeat();
3378
+ const remainingMs = Math.max(1, deadline - Date.now());
3379
+ const acknowledged = await Promise.race([
3380
+ heartbeat.executionAcknowledged.then(() => true),
3381
+ new Promise<false>((resolve) =>
3382
+ setTimeout(resolve, Math.min(retryDelayMs, remainingMs), false),
3383
+ ),
3384
+ ]);
3385
+ if (acknowledged) return;
3386
+ retryDelayMs = Math.min(1_000, retryDelayMs * 2);
3387
+ }
3388
+ throw new Error(
3389
+ `RUNTIME_SANDBOX_START_FAILED: scheduler did not acknowledge the detached runner park within ${Math.max(1, timeoutMs)}ms. Customer play code was not started.`,
3390
+ );
3391
+ }
3392
+
3393
+ /**
3394
+ * Begins local artifact readback as soon as the runner has parsed its config,
3395
+ * while preserving the durable runner-park fence before customer code receives
3396
+ * that artifact. The artifact contains source text only; materializing it does
3397
+ * not evaluate user code or make any runtime-state transition.
3398
+ *
3399
+ * Keep a handled materialization result while waiting for the acknowledgement:
3400
+ * a failed read must still wait for the initial heartbeat so the configured
3401
+ * terminal push can report the authored failure through the normal path.
3402
+ */
3403
+ export async function materializeTransportedArtifactWhileRunnerParks(
3404
+ parsedConfig: PlayRunnerExecutionConfig,
3405
+ heartbeat: ReturnType<typeof startRunnerPushHeartbeatFromConfig>,
3406
+ materialize: (
3407
+ config: PlayRunnerExecutionConfig,
3408
+ ) => Promise<PlayRunnerExecutionConfig> = materializeTransportedArtifact,
3409
+ ): Promise<PlayRunnerExecutionConfig> {
3410
+ const materialization = materialize(parsedConfig).then(
3411
+ (config) => ({ ok: true as const, config }),
3412
+ (error) => ({ ok: false as const, error }),
3413
+ );
3414
+
3415
+ // The first gateway heartbeat is the durable readiness fence. The worker
3416
+ // observes that same Postgres fact directly. It still gates every call to
3417
+ // executePlayRunner; only file I/O overlaps the handoff.
3418
+ await heartbeat?.firstHeartbeat;
3419
+ await waitForRunnerExecutionAcknowledgement(heartbeat);
3420
+
3421
+ const outcome = await materialization;
3422
+ if ('config' in outcome) {
3423
+ return outcome.config;
3424
+ }
3425
+ throw outcome.error;
3426
+ }
3427
+
3428
+ export function writeRunnerRuntimeStartedMarker(
3429
+ path: string | null | undefined,
3430
+ detail: { pid?: number; at?: number } = {},
3431
+ ): void {
3432
+ const runtimeStartedPath = path?.trim();
3433
+ if (!runtimeStartedPath) return;
3434
+ writeFileSync(
3435
+ runtimeStartedPath,
3436
+ JSON.stringify({
3437
+ started: true,
3438
+ pid: detail.pid ?? process.pid,
3439
+ at: detail.at ?? Date.now(),
3440
+ }),
3441
+ 'utf-8',
3442
+ );
3443
+ }
3444
+
3445
+ export function writeRunnerRuntimeCompletedMarker(
3446
+ path: string | null | undefined,
3447
+ detail: { pid?: number; at?: number } = {},
3448
+ ): void {
3449
+ const runtimeCompletedPath = path?.trim();
3450
+ if (!runtimeCompletedPath) return;
3451
+ writeFileSync(
3452
+ runtimeCompletedPath,
3453
+ JSON.stringify({
3454
+ completed: true,
3455
+ pid: detail.pid ?? process.pid,
3456
+ at: detail.at ?? Date.now(),
3457
+ }),
3458
+ 'utf-8',
3459
+ );
3460
+ }
3461
+
3462
+ /**
3463
+ * A tiny, strictly non-secret handoff record for detached Daytona startup.
3464
+ * The worker reads it only when the scheduler did not see the first heartbeat;
3465
+ * stdout is deliberately not inspected because it may contain customer data.
3466
+ */
3467
+ export function writeRunnerStartupDiagnostic(
3468
+ path: string | null | undefined,
3469
+ detail:
3470
+ | { phase: 'booted' }
3471
+ | { phase: 'heartbeat_transport'; event: RunnerGatewayTransportEvent },
3472
+ ): void {
3473
+ const diagnosticPath = path?.trim();
3474
+ if (!diagnosticPath) return;
3475
+ try {
3476
+ writeFileSync(
3477
+ diagnosticPath,
3478
+ JSON.stringify({ schemaVersion: 1, at: Date.now(), ...detail }),
3479
+ 'utf-8',
3480
+ );
3481
+ } catch {
3482
+ // Diagnostics must never affect the runner's durable control path.
3483
+ }
3484
+ }
3485
+
3486
+ /** Terminal push wiring (set once in main after the config loads). */
3487
+ let runnerTerminalPushContext: {
3488
+ gatewayBaseUrl: string;
3489
+ executorToken: string;
3490
+ runId: string;
3491
+ } | null = null;
3492
+
3493
+ function terminalPushContextFromConfig(
3494
+ config: PlayRunnerExecutionConfig,
3495
+ ): typeof runnerTerminalPushContext {
3496
+ const push = config.context.runnerPushExecution;
3497
+ const gatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
3498
+ const executorToken = config.context.executorToken?.trim();
3499
+ if (!push || !gatewayBaseUrl || !executorToken) {
3500
+ return null;
3501
+ }
3502
+ return { gatewayBaseUrl, executorToken, runId: push.runId };
3503
+ }
3504
+
3505
+ async function pushRunnerTerminalIfConfigured(
3506
+ result: PlayRunnerResult | PlayRunnerTerminalTransportResult,
3507
+ options?: Pick<
3508
+ PushRunnerTerminalOptions,
3509
+ 'warn' | 'finalProgressEvents' | 'finalProgressLogOffset'
3510
+ >,
3511
+ ): Promise<void> {
3512
+ const context = runnerTerminalPushContext;
3513
+ if (!context) return;
3514
+ await pushRunnerTerminal({
3515
+ gatewayBaseUrl: context.gatewayBaseUrl,
3516
+ executorToken: context.executorToken,
3517
+ runId: context.runId,
3518
+ result,
3519
+ ...options,
3520
+ });
3521
+ }
3522
+
3523
+ /**
3524
+ * Start the push-execution live progress pusher when the launch enabled push
3525
+ * execution (Daytona sandbox runner). Returns null for worker-babysat backends,
3526
+ * which keep their sidecar-file progress path.
3527
+ */
3528
+ function startRunnerProgressPusherFromConfig(
3529
+ config: PlayRunnerExecutionConfig,
3530
+ ): RunnerProgressPusher | null {
3531
+ const push = config.context.runnerPushExecution;
3532
+ const gatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
3533
+ const executorToken = config.context.executorToken?.trim();
3534
+ const playName = config.context.playName?.trim();
3535
+ if (!push || !gatewayBaseUrl || !executorToken || !playName) {
3536
+ return null;
3537
+ }
3538
+ return createRunnerProgressPusher({
3539
+ gatewayBaseUrl,
3540
+ executorToken,
3541
+ runId: push.runId,
3542
+ playName,
3543
+ });
3544
+ }
3545
+
3546
+ /**
3547
+ * Start the push-execution lease heartbeat when the launch enabled it (Daytona
3548
+ * sandbox runner). Returns null for worker-babysat backends. On the gateway's
3549
+ * terminal cancellation channel — or a gateway that stays unreachable past the
3550
+ * lease window — the loop calls back here to terminate the runner loudly, the
3551
+ * same way the worker's CancelledTask path aborts the attempt today.
3552
+ */
3553
+ function startRunnerPushHeartbeatFromConfig(
3554
+ config: PlayRunnerExecutionConfig,
3555
+ onSelfTerminate?: (stop: RunnerHeartbeatStop) => void,
3556
+ ): ReturnType<typeof startRunnerHeartbeatLoop> | null {
3557
+ const push = config.context.runnerPushExecution;
3558
+ const gatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
3559
+ const executorToken = config.context.executorToken?.trim();
3560
+ if (!push || !gatewayBaseUrl || !executorToken) {
3561
+ return null;
3562
+ }
3563
+ return startRunnerHeartbeatLoop({
3564
+ gatewayBaseUrl,
3565
+ executorToken,
3566
+ runId: push.runId,
3567
+ leaseSeconds: push.leaseSeconds,
3568
+ onSelfTerminate: (stop) => {
3569
+ onSelfTerminate?.(stop);
3570
+ void terminateRunnerForHeartbeatStop(stop);
3571
+ },
3572
+ onTransport: (event) => {
3573
+ writeRunnerStartupDiagnostic(
3574
+ process.env.DEEPLINE_PLAY_RUNNER_STARTUP_DIAGNOSTIC_PATH,
3575
+ { phase: 'heartbeat_transport', event },
3576
+ );
3577
+ },
3578
+ });
3579
+ }
3580
+
3581
+ function terminateRunnerForHeartbeatStop(
3582
+ stop: RunnerHeartbeatStop,
3583
+ ): Promise<never> {
3584
+ const error =
3585
+ stop.reason === 'terminal'
3586
+ ? stop.terminal === 'cancelled'
3587
+ ? 'Play run was cancelled; sandbox runner self-terminated on the receipt gateway heartbeat cancellation channel.'
3588
+ : 'Play run already failed in the scheduler; sandbox runner self-terminated on the receipt gateway heartbeat cancellation channel.'
3589
+ : `Sandbox runner could not reach the receipt gateway for ${stop.reason === 'unreachable' ? stop.sinceMs : 0}ms (past the lease window); self-terminating — the parked attempt's ceiling timeout salvages this result from the captured output. ${formatRunnerGatewayTransportSummary(stop.reason === 'unreachable' ? stop.gatewayTransport : undefined)}`;
3590
+ const failedResult = runnerTerminalResultForTransport({
3591
+ status: 'failed',
3592
+ error,
3593
+ logs: [],
3594
+ stats: {},
3595
+ steps: [],
3596
+ tableNamespace: null,
3597
+ });
3598
+ return emitAndExit(
3599
+ {
3600
+ type: 'result',
3601
+ result: failedResult,
3602
+ },
3603
+ stop.reason === 'terminal' && stop.terminal === 'cancelled' ? 0 : 1,
3604
+ );
3605
+ }
3606
+
3607
+ async function terminateRunnerForRuntimeDeadline(
3608
+ runtimeLimitSeconds: number,
3609
+ ): Promise<never> {
3610
+ const error = runtimeLimitExceededCause(runtimeLimitSeconds);
3611
+ const failedResult = runnerTerminalResultForTransport({
3612
+ status: 'failed',
3613
+ error,
3614
+ errors: [runtimeLimitExceededFailure(runtimeLimitSeconds, error)],
3615
+ logs: [],
3616
+ stats: {},
3617
+ steps: [],
3618
+ tableNamespace: null,
3619
+ });
3620
+ // Stop customer code immediately. The structured stdout event is captured by
3621
+ // the Daytona shell wrapper, whose crash epilogue owns the bounded gateway
3622
+ // push during the outer grace period. Awaiting network/progress work here
3623
+ // would let customer code continue dispatching after its deadline.
3624
+ return emitAndExit(
3625
+ {
3626
+ type: 'result',
3627
+ result: failedResult,
3628
+ },
3629
+ 0,
3630
+ );
3631
+ }
3632
+
3633
+ export function runPlayRunnerMain(): void {
3634
+ main().catch(async (error) => {
3635
+ ensureSourceMapSupport();
3636
+ const message = formatRunnerFailure(error);
3637
+ const failedResult: PlayRunnerResult = {
3638
+ status: 'failed',
3639
+ error: message,
3640
+ errors: [normalizePlayRunFailure(error)],
3641
+ logs: [],
3642
+ stats: {},
3643
+ steps: [],
3644
+ tableNamespace: null,
3645
+ };
3646
+ const terminalResult = runnerTerminalResultForTransport(failedResult);
3647
+ // Write the salvageable terminal before its gateway push. Startup failures
3648
+ // and unexpected settlement failures must retain their real diagnosis even
3649
+ // if the push itself stalls or exhausts retries.
3650
+ await writeRunnerEventToStdout({
3651
+ type: 'result',
3652
+ result: terminalResult,
3653
+ }).catch(() => undefined);
3654
+ // Push execution: wake the parked worker attempt NOW instead of making it
3655
+ // wait for the ceiling-timeout salvage. Best-effort — the output file
3656
+ // still carries this result for the salvage path.
3657
+ await pushRunnerTerminalIfConfigured(terminalResult, {
3658
+ warn: createPostTerminalDiagnosticWriter(),
3659
+ }).catch(() => undefined);
3660
+ void exitRunner(0);
3661
+ });
3662
+ }