deepline 0.3.88 → 0.3.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (280) hide show
  1. package/contracts/install-commands.json +29 -0
  2. package/dist/bundling-sources/sdk/src/client.ts +652 -39
  3. package/dist/bundling-sources/sdk/src/config.ts +77 -0
  4. package/dist/bundling-sources/sdk/src/http.ts +76 -30
  5. package/dist/bundling-sources/sdk/src/index.ts +10 -0
  6. package/dist/bundling-sources/sdk/src/monitor-fleet-contract.ts +1 -1
  7. package/dist/bundling-sources/sdk/src/pagination.ts +25 -0
  8. package/dist/bundling-sources/sdk/src/play.ts +15 -1
  9. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +3 -3
  10. package/dist/bundling-sources/sdk/src/release.ts +16 -9
  11. package/dist/bundling-sources/sdk/src/types.ts +106 -4
  12. package/dist/bundling-sources/shared_libs/integration-contracts/AGENTS.md +9 -0
  13. package/dist/bundling-sources/shared_libs/integration-contracts/README.md +3 -0
  14. package/dist/bundling-sources/shared_libs/integration-contracts/index.ts +2 -0
  15. package/dist/bundling-sources/shared_libs/monitors/analytics-install-guides.ts +77 -0
  16. package/dist/bundling-sources/shared_libs/monitors/analytics-public.ts +10 -0
  17. package/dist/bundling-sources/shared_libs/monitors/billing/charges.ts +125 -0
  18. package/dist/bundling-sources/shared_libs/monitors/billing/policy-builders.ts +686 -0
  19. package/dist/bundling-sources/shared_libs/monitors/billing/pricing.ts +54 -0
  20. package/dist/bundling-sources/shared_libs/monitors/billing/public-pricing.ts +72 -0
  21. package/dist/bundling-sources/shared_libs/monitors/billing/session.ts +45 -0
  22. package/dist/bundling-sources/shared_libs/monitors/billing/state.ts +454 -0
  23. package/dist/bundling-sources/shared_libs/monitors/billing/tamradar.ts +260 -0
  24. package/dist/bundling-sources/shared_libs/monitors/billing/types.ts +215 -0
  25. package/dist/bundling-sources/shared_libs/monitors/callback-security.ts +333 -0
  26. package/dist/bundling-sources/shared_libs/monitors/cursor-page.ts +42 -0
  27. package/dist/bundling-sources/shared_libs/monitors/deactivation-reasons.ts +16 -0
  28. package/dist/bundling-sources/shared_libs/monitors/framework-managed-payload-fields.ts +71 -0
  29. package/dist/bundling-sources/shared_libs/monitors/generated/provider-monitor-callback-admission-targets.ts +16 -0
  30. package/dist/bundling-sources/shared_libs/monitors/http-error-response.ts +270 -0
  31. package/dist/bundling-sources/shared_libs/monitors/inbound-publish-key.ts +39 -0
  32. package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +82 -0
  33. package/dist/bundling-sources/shared_libs/monitors/inventory-reconciliation.ts +59 -0
  34. package/dist/bundling-sources/shared_libs/monitors/lifecycle-notifications.ts +471 -0
  35. package/dist/bundling-sources/shared_libs/monitors/org-monitor-limits.ts +1 -1
  36. package/dist/bundling-sources/shared_libs/monitors/package.json +36 -0
  37. package/dist/bundling-sources/shared_libs/monitors/provider-callback-verification.ts +1125 -0
  38. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +305 -0
  39. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-capability.ts +964 -0
  40. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-consumer-health.ts +56 -0
  41. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-errors.ts +131 -0
  42. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-identity-claims.ts +535 -0
  43. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-legacy-aliases.ts +16 -0
  44. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-priority-execution.ts +56 -0
  45. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-public-aliases.ts +28 -0
  46. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-runtime-read-eligibility.ts +38 -0
  47. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-state.ts +122 -0
  48. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-status.ts +10 -0
  49. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-upstream-identity.ts +112 -0
  50. package/dist/bundling-sources/shared_libs/monitors/provider-webhook-monitor-runtime.ts +300 -0
  51. package/dist/bundling-sources/shared_libs/monitors/quickstart-prompts.ts +22 -0
  52. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts +245 -0
  53. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-message.ts +38 -0
  54. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-ack-probe.ts +94 -0
  55. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-delivery.ts +60 -0
  56. package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue.ts +60 -0
  57. package/dist/bundling-sources/shared_libs/monitors/snitcher-install-validation.ts +93 -0
  58. package/dist/bundling-sources/shared_libs/monitors/snitcher-ip2company.ts +332 -0
  59. package/dist/bundling-sources/shared_libs/monitors/snitcher-radar.ts +221 -0
  60. package/dist/bundling-sources/shared_libs/monitors/starter-templates.ts +105 -0
  61. package/dist/bundling-sources/shared_libs/monitors/tracking-provider-guidance.ts +73 -0
  62. package/dist/bundling-sources/shared_libs/monitors/tracking-provider-integrations.ts +81 -0
  63. package/dist/bundling-sources/shared_libs/monitors/tsconfig.json +10 -0
  64. package/dist/bundling-sources/shared_libs/observability/build-context.ts +13 -0
  65. package/dist/bundling-sources/shared_libs/observability/log-metadata.ts +160 -0
  66. package/dist/bundling-sources/shared_libs/observability/request-context.ts +68 -0
  67. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +654 -0
  68. package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +35 -0
  69. package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +5 -0
  70. package/dist/bundling-sources/shared_libs/play-data-plane/neon-constants.ts +9 -0
  71. package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +272 -0
  72. package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts +6 -11
  73. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +135 -12
  74. package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +66 -1
  75. package/dist/bundling-sources/shared_libs/play-runtime/async-play-invocation.ts +77 -0
  76. package/dist/bundling-sources/shared_libs/play-runtime/bounceban-batching.ts +137 -0
  77. package/dist/bundling-sources/shared_libs/play-runtime/checkpoint-boundaries.ts +70 -0
  78. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1159 -166
  79. package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +1 -32
  80. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +103 -0
  81. package/dist/bundling-sources/shared_libs/play-runtime/daytona-payload-contract.ts +68 -0
  82. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +103 -0
  83. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +725 -0
  84. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +20 -1
  85. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +29 -0
  86. package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +29 -0
  87. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +442 -0
  88. package/dist/bundling-sources/shared_libs/play-runtime/execution-protocol.ts +7 -1
  89. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +18 -0
  90. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +9 -0
  91. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +1 -1
  92. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +1 -1
  93. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +66 -0
  94. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +145 -0
  95. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/ports.ts +21 -0
  96. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/postgres-store.ts +170 -0
  97. package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +1322 -0
  98. package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +2 -2
  99. package/dist/bundling-sources/shared_libs/play-runtime/lane-retirement-gate.ts +189 -0
  100. package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +112 -0
  101. package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +8 -0
  102. package/dist/bundling-sources/shared_libs/play-runtime/perf-trace.ts +127 -0
  103. package/dist/bundling-sources/shared_libs/play-runtime/play-artifact-runtime-contract.ts +5 -0
  104. package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +24 -1
  105. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +56 -0
  106. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +13 -0
  107. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/job.ts +41 -0
  108. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/ports.ts +17 -0
  109. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +54 -0
  110. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/contracts.ts +114 -0
  111. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/engine-suspension.ts +108 -0
  112. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/index.ts +36 -0
  113. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/job.ts +97 -0
  114. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/ports.ts +104 -0
  115. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/postgres-fence-adapter.ts +263 -0
  116. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +64 -0
  117. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/contracts.ts +29 -0
  118. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +56 -0
  119. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/index.ts +15 -0
  120. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/job.ts +65 -0
  121. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/ports.ts +17 -0
  122. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts +42 -0
  123. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/index.ts +17 -0
  124. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +142 -0
  125. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/ports.ts +42 -0
  126. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/postgres-adapter.ts +102 -0
  127. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +60 -0
  128. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/index.ts +16 -0
  129. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +78 -0
  130. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +54 -0
  131. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +71 -0
  132. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +4 -0
  133. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +28 -0
  134. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +4 -0
  135. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +5 -8
  136. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +23 -2
  137. package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +168 -0
  138. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +1 -1
  139. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +90 -13
  140. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +6 -12
  141. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-start-admission.ts +68 -0
  142. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +21 -0
  143. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +0 -1
  144. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +14 -43
  145. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +67 -213
  146. package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts +201 -0
  147. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +20 -0
  148. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +6 -0
  149. package/dist/bundling-sources/shared_libs/play-runtime/runtime-env.ts +12 -0
  150. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +67 -0
  151. package/dist/bundling-sources/shared_libs/play-runtime/runtime-release-environment.ts +2 -0
  152. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-contract.ts +68 -0
  153. package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +38 -2
  154. package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +108 -0
  155. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +4 -0
  156. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +3 -0
  157. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +37 -1
  158. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts +88 -0
  159. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +124 -0
  160. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-run-task.ts +68 -0
  161. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts +193 -0
  162. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts +269 -0
  163. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-suffix.ts +73 -0
  164. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +6642 -0
  165. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +897 -0
  166. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-metrics.ts +120 -0
  167. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +76 -0
  168. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +476 -0
  169. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +47 -0
  170. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +147 -0
  171. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/index.ts +27 -0
  172. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/lazy.ts +50 -0
  173. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/play-run-recovery.ts +98 -0
  174. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts +260 -0
  175. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +1416 -0
  176. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +1255 -0
  177. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +942 -0
  178. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +14636 -0
  179. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +888 -0
  180. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +355 -0
  181. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +46 -0
  182. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runner-push.ts +60 -0
  183. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-api-origin.ts +33 -0
  184. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-gateway-health.ts +64 -0
  185. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +835 -0
  186. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts +1218 -0
  187. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-operation.ts +207 -0
  188. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-scheduler-schema.ts +10 -0
  189. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts +123 -0
  190. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-launch-read-observability.ts +78 -0
  191. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +169 -0
  192. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/terminal-contract.ts +20 -0
  193. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-queue-metrics.ts +34 -0
  194. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +249 -0
  195. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +7 -0
  196. package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +296 -0
  197. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +3 -3
  198. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +20 -1
  199. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +60 -15
  200. package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +20 -0
  201. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +1 -1
  202. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +1 -1
  203. package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +12 -0
  204. package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +14 -1
  205. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +165 -0
  206. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +15 -23
  207. package/dist/bundling-sources/shared_libs/play-runtime/workflow-executor-auth.ts +169 -0
  208. package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +122 -0
  209. package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +8 -3
  210. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +454 -17
  211. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +19 -4
  212. package/dist/bundling-sources/shared_libs/plays/contracts.ts +8 -14
  213. package/dist/bundling-sources/shared_libs/plays/core.ts +11 -6
  214. package/dist/bundling-sources/shared_libs/plays/deployment-status.ts +1 -1
  215. package/dist/bundling-sources/shared_libs/plays/docflow-binding-owner.ts +16 -11
  216. package/dist/bundling-sources/shared_libs/plays/docflow.ts +1 -1
  217. package/dist/bundling-sources/shared_libs/plays/email-status.ts +1 -1
  218. package/dist/bundling-sources/shared_libs/plays/play-grep-search.ts +39 -0
  219. package/dist/bundling-sources/shared_libs/plays/resolve-static-pipeline.ts +1 -1
  220. package/dist/bundling-sources/shared_libs/plays/share/copy.ts +271 -0
  221. package/dist/bundling-sources/shared_libs/plays/share/metrics.ts +140 -0
  222. package/dist/bundling-sources/shared_libs/plays/share/public-read-model.ts +86 -0
  223. package/dist/bundling-sources/shared_libs/plays/share/seo-quality.ts +181 -0
  224. package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts +658 -0
  225. package/dist/bundling-sources/shared_libs/plays/share/types.ts +197 -0
  226. package/dist/bundling-sources/shared_libs/plays/slack-hitl-timeout-error.ts +37 -0
  227. package/dist/bundling-sources/shared_libs/plays/slug.ts +115 -0
  228. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +50 -15
  229. package/dist/bundling-sources/shared_libs/plays/tool-category-descriptions.ts +2 -2
  230. package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +1 -1
  231. package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +877 -0
  232. package/dist/bundling-sources/shared_libs/product-notifications/configuration-draft.ts +456 -0
  233. package/dist/bundling-sources/shared_libs/product-notifications/configuration.ts +566 -0
  234. package/dist/bundling-sources/shared_libs/product-notifications/content.ts +28 -0
  235. package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +60 -0
  236. package/dist/bundling-sources/shared_libs/product-notifications/email.ts +246 -0
  237. package/dist/bundling-sources/shared_libs/product-notifications/evaluation.ts +280 -0
  238. package/dist/bundling-sources/shared_libs/product-notifications/events.ts +433 -0
  239. package/dist/bundling-sources/shared_libs/product-notifications/legacy-slack.ts +80 -0
  240. package/dist/bundling-sources/shared_libs/product-notifications/overflow.ts +18 -0
  241. package/dist/bundling-sources/shared_libs/product-notifications/rolling-limit.ts +24 -0
  242. package/dist/bundling-sources/shared_libs/product-notifications/slack-delivery.ts +624 -0
  243. package/dist/bundling-sources/shared_libs/product-notifications/slack-payload.ts +140 -0
  244. package/dist/bundling-sources/shared_libs/product-notifications/slack-transport.ts +75 -0
  245. package/dist/bundling-sources/shared_libs/product-notifications/workspace.ts +38 -0
  246. package/dist/bundling-sources/shared_libs/runtime-env.ts +12 -0
  247. package/dist/bundling-sources/shared_libs/security/email-blacklist.ts +42 -0
  248. package/dist/bundling-sources/shared_libs/security/header-safe-env.ts +35 -0
  249. package/dist/bundling-sources/shared_libs/security/internal-actor.ts +50 -0
  250. package/dist/bundling-sources/shared_libs/security/secret-redaction.ts +267 -0
  251. package/dist/bundling-sources/shared_libs/security/slack-sanitize.ts +133 -0
  252. package/dist/cli/getting-started.d.mts +52 -0
  253. package/dist/cli/getting-started.d.ts +52 -0
  254. package/dist/cli/getting-started.js +65 -0
  255. package/dist/cli/getting-started.mjs +36 -0
  256. package/dist/cli/index.d.mts +9 -0
  257. package/dist/cli/index.d.ts +9 -0
  258. package/dist/cli/index.js +1611 -453
  259. package/dist/cli/index.mjs +1596 -447
  260. package/dist/{compiler-manifest-Dbov16F7.d.mts → compiler-manifest-BaJfE-5H.d.mts} +136 -10
  261. package/dist/{compiler-manifest-Dbov16F7.d.ts → compiler-manifest-BaJfE-5H.d.ts} +136 -10
  262. package/dist/index.d.mts +910 -155
  263. package/dist/index.d.ts +910 -155
  264. package/dist/index.js +550 -64
  265. package/dist/index.mjs +550 -64
  266. package/dist/install-integrity.json +193 -4
  267. package/dist/monitor-fleet-contract.d.mts +138 -0
  268. package/dist/monitor-fleet-contract.d.ts +138 -0
  269. package/dist/monitor-fleet-contract.js +404 -0
  270. package/dist/monitor-fleet-contract.mjs +367 -0
  271. package/dist/plays/bundle-play-file.d.mts +8 -3
  272. package/dist/plays/bundle-play-file.d.ts +8 -3
  273. package/dist/plays/bundle-play-file.mjs +100 -20
  274. package/dist/release.d.mts +255 -0
  275. package/dist/release.d.ts +255 -0
  276. package/dist/release.js +215 -0
  277. package/dist/release.mjs +188 -0
  278. package/package.json +23 -4
  279. /package/dist/bundling-sources/shared_libs/{integrations/bettercontact-execution-policy.ts → integration-contracts/bettercontact/execution-policy.ts} +0 -0
  280. /package/dist/bundling-sources/shared_libs/{integrations/theirstack-execution-policy.ts → integration-contracts/theirstack/execution-policy.ts} +0 -0
@@ -109,7 +109,11 @@ import {
109
109
  type ToolResponseContract,
110
110
  type ToolResponseView,
111
111
  } from '../plays/tool-response-contract';
112
- import { ProviderExhaustedError } from './run-failure';
112
+ import {
113
+ isProviderExhaustedFailure,
114
+ ProviderExhaustedError,
115
+ } from './run-failure';
116
+ import { PROVIDER_EXHAUSTED_MAX_WAIT_MS } from './governor/app-runtime-rate-state-backend';
113
117
  import {
114
118
  buildPlayContractCompatibility,
115
119
  normalizePlayContractCompatibility,
@@ -122,6 +126,10 @@ import {
122
126
  type ToolExecutionErrorSchemaVersion,
123
127
  type ToolExecutionFailureV1,
124
128
  } from '../plays/tool-execution-error';
129
+ import {
130
+ SLACK_HITL_TIMEOUT_ERROR_CODE,
131
+ SlackHitlTimeoutError,
132
+ } from '../plays/slack-hitl-timeout-error';
125
133
  import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
126
134
  import { pacingPolicyForTool } from './pacing';
127
135
  import {
@@ -177,6 +185,7 @@ import {
177
185
  buildDurableToolAggregateReceiptKey,
178
186
  buildDurableToolCallAuthScopeDigest,
179
187
  buildDurableToolCallCacheKey,
188
+ buildSourceRunToolReceiptManifestKey,
180
189
  buildLegacyRevisionedDurableToolCallCacheKey,
181
190
  buildDurableToolPayloadProviderIdempotencyKey,
182
191
  buildProviderOperationIdempotencyKey,
@@ -353,7 +362,11 @@ import type {
353
362
  PlayRowReadOrder,
354
363
  PlayRowMeta,
355
364
  PlaySheetCellProducer,
365
+ PlayAsyncCallOptions,
356
366
  } from './ctx-types';
367
+ import { assertKnownResolvedBoundary } from './checkpoint-boundaries';
368
+ import type { AsyncPlayRunRef } from '../plays/authoring-contract';
369
+ import { admitAsyncPlayLineage } from './async-play-invocation';
357
370
  import {
358
371
  MAX_CELL_READ_REFS,
359
372
  MAX_ROW_READ_CELLS,
@@ -1982,6 +1995,19 @@ function normalizeStepDescription(
1982
1995
  return trimmed ? trimmed : undefined;
1983
1996
  }
1984
1997
 
1998
+ function resolvePlayReferenceName(playRef: unknown): string {
1999
+ if (typeof playRef === 'string') return playRef;
2000
+ if (
2001
+ playRef === null ||
2002
+ (typeof playRef !== 'object' && typeof playRef !== 'function')
2003
+ ) {
2004
+ return '';
2005
+ }
2006
+ const reference = playRef as { playName?: unknown; name?: unknown };
2007
+ if (typeof reference.playName === 'string') return reference.playName;
2008
+ return typeof reference.name === 'string' ? reference.name : '';
2009
+ }
2010
+
1985
2011
  function emptyCheckpoint(): PlayCheckpoint {
1986
2012
  return {
1987
2013
  completedBatches: {},
@@ -1996,6 +2022,14 @@ function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
1996
2022
  ...frame,
1997
2023
  completedRowKeys: [...frame.completedRowKeys],
1998
2024
  pendingRowKeys: [...frame.pendingRowKeys],
2025
+ ...(frame.suspendedInputRows
2026
+ ? {
2027
+ suspendedInputRows: frame.suspendedInputRows.map((row) => ({
2028
+ inputIndex: row.inputIndex,
2029
+ data: cloneCsvAliasedRow(row.data),
2030
+ })),
2031
+ }
2032
+ : {}),
1999
2033
  };
2000
2034
  }
2001
2035
 
@@ -2089,6 +2123,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2089
2123
  * waterfall miss.
2090
2124
  */
2091
2125
  private deeplineInsufficientCreditsFailure: ToolExecutionError | null = null;
2126
+ /**
2127
+ * A Slack HITL timeout is terminal only after its card finalizer has proved
2128
+ * the Slack replacement. Keep that terminal result outside authored promise
2129
+ * control flow so catching the rejected tool call cannot settle the Play or
2130
+ * start a later durable operation.
2131
+ */
2132
+ private slackHitlTimeoutFailure: ToolExecutionError | null = null;
2092
2133
  private toolCallResolvers = new Map<
2093
2134
  string,
2094
2135
  { resolve: (value: unknown) => void; reject: (reason: unknown) => void }
@@ -2103,6 +2144,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2103
2144
  >();
2104
2145
  private readonly ctxFetchHttpFailureDiagnosticIdentities = new Set<string>();
2105
2146
  private checkpoint: PlayCheckpoint;
2147
+ /**
2148
+ * A durable boundary is a control-flow handoff, not a customer-catchable
2149
+ * business failure. Keep it on the context as the source of truth so a
2150
+ * broad catch cannot turn the invocation into a normal completed result.
2151
+ */
2152
+ private pendingDurableSuspension: PlayExecutionSuspension | null = null;
2106
2153
  private readonly durableCallCacheEpochMs: number;
2107
2154
  /**
2108
2155
  * Durable tool receipts are the replay/cache authority for the execution
@@ -2215,6 +2262,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2215
2262
  timeoutMs?: number;
2216
2263
  receiptWaitMs?: number;
2217
2264
  }): Promise<TOutput> => {
2265
+ this.assertNoPendingDurableSuspension();
2218
2266
  if (!request || typeof request !== 'object' || Array.isArray(request)) {
2219
2267
  throw new Error(
2220
2268
  'ctx.tools.execute requires a request object: ctx.tools.execute({ id, tool, input, description }).',
@@ -2281,6 +2329,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2281
2329
  : undefined,
2282
2330
  ).catch((error: unknown) => {
2283
2331
  this.recordDeeplineInsufficientCreditsFailure(error);
2332
+ this.recordSlackHitlTimeoutFailure(error);
2284
2333
  throw error;
2285
2334
  }) as Promise<TOutput>;
2286
2335
  },
@@ -2303,6 +2352,62 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2303
2352
  }
2304
2353
  }
2305
2354
 
2355
+ private recordSlackHitlTimeoutFailure(error: unknown): void {
2356
+ if (
2357
+ error instanceof ToolExecutionError &&
2358
+ error.code === SLACK_HITL_TIMEOUT_ERROR_CODE
2359
+ ) {
2360
+ if (!this.slackHitlTimeoutFailure) {
2361
+ this.slackHitlTimeoutFailure = error;
2362
+ this.runtimeLog(
2363
+ `[slack_hitl.timeout_terminal] code=${SLACK_HITL_TIMEOUT_ERROR_CODE} ` +
2364
+ 'card_finalization=confirmed outcome=run_fatal',
2365
+ );
2366
+ }
2367
+ }
2368
+ }
2369
+
2370
+ /**
2371
+ * Called by the runner before a normal result settles. This preserves a
2372
+ * terminal Slack timeout even if authored code catches it and returns a
2373
+ * value. It intentionally remains separate from provider errors, which are
2374
+ * ordinary customer-handled waterfall misses.
2375
+ */
2376
+ assertNoSlackHitlTimeoutFailure(): void {
2377
+ if (this.slackHitlTimeoutFailure) {
2378
+ throw this.slackHitlTimeoutFailure;
2379
+ }
2380
+ }
2381
+
2382
+ /** Called by the runner after customer code returns, before a success settles. */
2383
+ assertNoRunFatalToolFailure(): void {
2384
+ this.assertNoSlackHitlTimeoutFailure();
2385
+ this.assertNoDeeplineInsufficientCreditsFailure();
2386
+ }
2387
+
2388
+ /**
2389
+ * Called at ctx effect boundaries and by the runner before it can settle a
2390
+ * normal result. This makes suspension sticky even when authored code uses a
2391
+ * broad catch around `await ctx.sleep(...)`.
2392
+ */
2393
+ assertNoPendingDurableSuspension(): void {
2394
+ this.assertNoSlackHitlTimeoutFailure();
2395
+ if (this.pendingDurableSuspension) {
2396
+ throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
2397
+ }
2398
+ }
2399
+
2400
+ /**
2401
+ * The single seam that hands this invocation back to the scheduler. The
2402
+ * first suspension wins: a later boundary reached by code that caught the
2403
+ * first one cannot replace the handoff the scheduler already owes a resume.
2404
+ */
2405
+ private suspendDurably(suspension: PlayExecutionSuspension): never {
2406
+ this.pendingDurableSuspension ??= suspension;
2407
+ this.#options.onBatchComplete?.(this.checkpoint);
2408
+ throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
2409
+ }
2410
+
2306
2411
  async tool<TOutput = unknown>(
2307
2412
  key: string,
2308
2413
  toolId: string,
@@ -2416,6 +2521,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2416
2521
  options.artifactRuntimeContract ??
2417
2522
  resolvePlayArtifactRuntimeContract({
2418
2523
  authoringContractEdition: options.authoringContractEdition,
2524
+ slackHitlTimeoutOutcome: options.slackHitlTimeoutOutcome,
2419
2525
  toolResponseContract: options.toolResponseContract,
2420
2526
  toolResponseReceiptRevision: options.toolResponseReceiptRevision,
2421
2527
  });
@@ -2475,11 +2581,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2475
2581
  options.playId ||
2476
2582
  options.workflowId ||
2477
2583
  'anonymous-play';
2478
- const rootRunId = options.runId ?? options.workflowId ?? 'anonymous-run';
2584
+ const currentRunId = options.runId ?? options.workflowId ?? 'anonymous-run';
2585
+ const rootRunId = options.rootRunId ?? currentRunId;
2479
2586
  this.executionScope =
2480
2587
  options.executionScope ??
2481
2588
  createRootRunExecutionScope({
2482
- runId: rootRunId,
2589
+ runId: currentRunId,
2483
2590
  playId: rootPlayId,
2484
2591
  attempt:
2485
2592
  typeof options.runAttempt === 'number' &&
@@ -2489,7 +2596,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2489
2596
  receiptNamespace: options.runtimeReceiptScope ?? rootPlayId,
2490
2597
  authority: {
2491
2598
  orgId: options.orgId ?? 'org',
2492
- workflowId: options.workflowId ?? rootRunId,
2599
+ workflowId: options.workflowId ?? currentRunId,
2493
2600
  integrationMode: options.integrationMode ?? null,
2494
2601
  capabilities: DEFAULT_RUNTIME_EXECUTION_CAPABILITIES,
2495
2602
  },
@@ -2505,7 +2612,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2505
2612
  adapter: 'cjs_node20',
2506
2613
  scope: {
2507
2614
  orgId: options.orgId ?? 'org',
2508
- rootRunId: options.runId ?? options.workflowId ?? 'run',
2615
+ rootRunId,
2509
2616
  },
2510
2617
  rateState: options.rateState ?? new InMemoryRateStateBackend(),
2511
2618
  budgetState: options.budgetState,
@@ -2710,6 +2817,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
2710
2817
  private assertInlineChildContract(
2711
2818
  reason: 'dataset_child' | 'suspending_child',
2712
2819
  ): void {
2820
+ this.assertNoPendingDurableSuspension();
2713
2821
  const composition = this.activeInlineComposition;
2714
2822
  if (!composition) return;
2715
2823
  throw new Error(ctxRunPlayInlineOnlyMessage(composition.playName, reason));
@@ -3389,6 +3497,223 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
3389
3497
  };
3390
3498
  }
3391
3499
 
3500
+ /**
3501
+ * Record tiny pointers after canonical tool receipts settle. A rerun can
3502
+ * then recover each exact source-run receipt before trying the ordinary
3503
+ * cross-run cache. The pointers never contain provider input, credentials,
3504
+ * or provider responses.
3505
+ *
3506
+ * This deliberately uses the same bulk receipt seam as canonical result
3507
+ * persistence. A map can settle hundreds of results at once; falling back
3508
+ * to one read/claim/complete triplet per pointer would make compatibility
3509
+ * bookkeeping contend with the customer's provider work.
3510
+ */
3511
+ private async persistSourceRunToolReceiptManifests(
3512
+ inputs: Array<{
3513
+ toolId: string;
3514
+ logicalCallId?: string | null;
3515
+ executionAuthScopeDigest?: string | null;
3516
+ resultReceiptKey: string;
3517
+ }>,
3518
+ ): Promise<void> {
3519
+ const sourceRunId = this.currentReceiptOwnerRunId.trim();
3520
+ if (
3521
+ !sourceRunId ||
3522
+ !(
3523
+ this.#options.claimRuntimeStepReceipt ||
3524
+ this.#options.claimRuntimeStepReceipts
3525
+ ) ||
3526
+ !(
3527
+ this.#options.completeRuntimeStepReceipt ||
3528
+ this.#options.completeRuntimeStepReceipts
3529
+ )
3530
+ ) {
3531
+ return;
3532
+ }
3533
+ try {
3534
+ const providerActionVersions = new Map<string, string>();
3535
+ await Promise.all(
3536
+ [...new Set(inputs.map((input) => input.toolId))].map(
3537
+ async (toolId) => {
3538
+ providerActionVersions.set(
3539
+ toolId,
3540
+ (
3541
+ await this.#options.getToolActionCacheVersion?.(toolId)
3542
+ )?.trim() ?? '',
3543
+ );
3544
+ },
3545
+ ),
3546
+ );
3547
+ const manifests = inputs.flatMap((input) => {
3548
+ const logicalCallId = input.logicalCallId?.trim();
3549
+ const resultReceiptKey = input.resultReceiptKey.trim();
3550
+ if (!logicalCallId || !resultReceiptKey) return [];
3551
+ const providerActionVersion =
3552
+ providerActionVersions.get(input.toolId) ?? '';
3553
+ if (!providerActionVersion) return [];
3554
+ const key = buildSourceRunToolReceiptManifestKey({
3555
+ orgId: this.#options.orgId,
3556
+ sourceRunId,
3557
+ logicalCallId,
3558
+ });
3559
+ return [
3560
+ {
3561
+ key,
3562
+ output: {
3563
+ kind: 'source-run-tool-receipt-manifest-v1' as const,
3564
+ toolId: input.toolId,
3565
+ providerActionVersion,
3566
+ executionAuthScopeDigest:
3567
+ input.executionAuthScopeDigest?.trim() ?? null,
3568
+ resultReceiptKey,
3569
+ },
3570
+ },
3571
+ ];
3572
+ });
3573
+ if (manifests.length === 0) return;
3574
+
3575
+ // Several row callbacks can refer to the same logical call. The first
3576
+ // canonical receipt is authoritative, so one idempotent pointer write is
3577
+ // enough.
3578
+ const uniqueManifests = [
3579
+ ...new Map(
3580
+ manifests.map((manifest) => [manifest.key, manifest]),
3581
+ ).values(),
3582
+ ];
3583
+ const existing = await this.getRuntimeStepReceipts(
3584
+ uniqueManifests.map((manifest) => manifest.key),
3585
+ );
3586
+ const missing = uniqueManifests.filter((manifest) => {
3587
+ const receipt = existing.get(manifest.key);
3588
+ return receipt?.status !== 'completed' && receipt?.status !== 'skipped';
3589
+ });
3590
+ if (missing.length === 0) return;
3591
+
3592
+ const claims = await this.claimRuntimeStepReceipts(
3593
+ missing.map((manifest) => manifest.key),
3594
+ sourceRunId,
3595
+ );
3596
+ await this.completeRuntimeStepReceipts(
3597
+ missing.flatMap((manifest) => {
3598
+ const claim = claims.get(manifest.key);
3599
+ return claim?.leaseId
3600
+ ? [
3601
+ {
3602
+ key: manifest.key,
3603
+ runId: sourceRunId,
3604
+ leaseId: claim.leaseId,
3605
+ output: manifest.output,
3606
+ },
3607
+ ]
3608
+ : [];
3609
+ }),
3610
+ );
3611
+ } catch (error) {
3612
+ // The canonical result receipt has already settled. A telemetry pointer
3613
+ // must never turn that successful provider result into a failed run.
3614
+ this.runtimeLog(
3615
+ `Tool source-run receipt manifest write skipped: ${error instanceof Error ? error.message : String(error)}`,
3616
+ );
3617
+ }
3618
+ }
3619
+
3620
+ private async persistSourceRunToolReceiptManifest(input: {
3621
+ toolId: string;
3622
+ logicalCallId?: string | null;
3623
+ executionAuthScopeDigest?: string | null;
3624
+ resultReceiptKey: string;
3625
+ }): Promise<void> {
3626
+ await this.persistSourceRunToolReceiptManifests([input]);
3627
+ }
3628
+
3629
+ private async recoverSourceRunToolReceipt(input: {
3630
+ toolId: string;
3631
+ logicalCallId: string;
3632
+ executionAuthScopeDigest?: string | null;
3633
+ cacheKey: string;
3634
+ }): Promise<ToolExecuteResult | null> {
3635
+ const sourceRunId = this.#options.replayedFromRunId?.trim();
3636
+ if (!sourceRunId || !this.#options.getRuntimeStepReceipt) return null;
3637
+
3638
+ const manifestKey = buildSourceRunToolReceiptManifestKey({
3639
+ orgId: this.#options.orgId,
3640
+ sourceRunId,
3641
+ logicalCallId: input.logicalCallId,
3642
+ });
3643
+ const manifestReceipt = await this.withRuntimeReceiptHydrationTurn(
3644
+ async () => await this.getRuntimeStepReceipt(manifestKey),
3645
+ );
3646
+ if (
3647
+ !manifestReceipt ||
3648
+ (manifestReceipt.status !== 'completed' &&
3649
+ manifestReceipt.status !== 'skipped')
3650
+ ) {
3651
+ return null;
3652
+ }
3653
+ const manifest = recordOrNull(this.runtimeReceiptOutput(manifestReceipt));
3654
+ if (
3655
+ !manifest ||
3656
+ manifest.kind !== 'source-run-tool-receipt-manifest-v1' ||
3657
+ manifest.toolId !== input.toolId ||
3658
+ typeof manifest.resultReceiptKey !== 'string' ||
3659
+ !manifest.resultReceiptKey.trim() ||
3660
+ typeof manifest.providerActionVersion !== 'string'
3661
+ ) {
3662
+ return null;
3663
+ }
3664
+ const currentActionVersion =
3665
+ (await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
3666
+ '';
3667
+ const compatibleActionVersions = new Set([
3668
+ currentActionVersion,
3669
+ ...(
3670
+ (await this.#options.getToolActionCacheReadVersions?.(input.toolId)) ??
3671
+ []
3672
+ ).map((version) => String(version || '').trim()),
3673
+ ]);
3674
+ if (!compatibleActionVersions.has(manifest.providerActionVersion.trim())) {
3675
+ return null;
3676
+ }
3677
+ const manifestAuthScopeDigest =
3678
+ typeof manifest.executionAuthScopeDigest === 'string'
3679
+ ? manifest.executionAuthScopeDigest.trim() || null
3680
+ : null;
3681
+ if (
3682
+ manifestAuthScopeDigest !==
3683
+ (input.executionAuthScopeDigest?.trim() || null)
3684
+ ) {
3685
+ return null;
3686
+ }
3687
+ const resultReceiptKey = manifest.resultReceiptKey.trim();
3688
+ const sourceReceipt = await this.withRuntimeReceiptHydrationTurn(
3689
+ async () => await this.getRuntimeStepReceipt(resultReceiptKey),
3690
+ );
3691
+ if (
3692
+ !sourceReceipt ||
3693
+ (sourceReceipt.status !== 'completed' &&
3694
+ sourceReceipt.status !== 'skipped')
3695
+ ) {
3696
+ return null;
3697
+ }
3698
+ const stored = this.runtimeReceiptOutput(sourceReceipt);
3699
+ const result = isSerializedToolExecuteResult(stored)
3700
+ ? deserializeToolExecuteResult(stored)
3701
+ : isToolExecuteResult(stored)
3702
+ ? stored
3703
+ : null;
3704
+ if (!result || !this.canReuseToolResultForCurrentResponseContract(result)) {
3705
+ return null;
3706
+ }
3707
+ return cloneToolExecuteResultWithExecution(
3708
+ result,
3709
+ toolExecutionMetadataForOutcome({
3710
+ kind: 'source_run',
3711
+ cacheKey: input.cacheKey,
3712
+ receiptKey: resultReceiptKey,
3713
+ }),
3714
+ );
3715
+ }
3716
+
3392
3717
  private async releaseRuntimeStepReceipt(
3393
3718
  key: string,
3394
3719
  _runId: string,
@@ -4206,6 +4531,47 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4206
4531
  return buildDurableToolCallCacheKey(identity);
4207
4532
  }
4208
4533
 
4534
+ /**
4535
+ * The first key is the only write/provider identity. Later keys are explicit
4536
+ * action-local read fallbacks, used only when source metadata says their
4537
+ * completed result remains compatible. Deploy and generated-catalog state
4538
+ * never participate in this list.
4539
+ */
4540
+ private async compatibleDurableToolCallCacheKeysForScope(input: {
4541
+ toolId: string;
4542
+ requestInput: Record<string, unknown>;
4543
+ executionAuthScopeDigest?: string | null;
4544
+ staleAfterSeconds?: number | null;
4545
+ playLocalScope?: string | null;
4546
+ }): Promise<string[]> {
4547
+ const identity = await this.durableToolCallCacheIdentity(input);
4548
+ const primaryVersion = identity.providerActionVersion?.trim();
4549
+ if (!primaryVersion) {
4550
+ throw new Error(
4551
+ `Tool ${input.toolId} is missing a provider action compatibility version.`,
4552
+ );
4553
+ }
4554
+ const readVersions = await this.#options.getToolActionCacheReadVersions?.(
4555
+ input.toolId,
4556
+ );
4557
+ const versions = [
4558
+ primaryVersion,
4559
+ ...(readVersions ?? [])
4560
+ .map((version) => String(version || '').trim())
4561
+ .filter(Boolean),
4562
+ ];
4563
+ return [
4564
+ ...new Set(
4565
+ versions.map((providerActionVersion) =>
4566
+ buildDurableToolCallCacheKey({
4567
+ ...identity,
4568
+ providerActionVersion,
4569
+ }),
4570
+ ),
4571
+ ),
4572
+ ];
4573
+ }
4574
+
4209
4575
  private async legacyDurableToolCallCacheKeyForScope(input: {
4210
4576
  toolId: string;
4211
4577
  requestInput: Record<string, unknown>;
@@ -4263,6 +4629,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4263
4629
  });
4264
4630
  }
4265
4631
 
4632
+ private async compatibleDurableToolCallCacheKeys(input: {
4633
+ toolId: string;
4634
+ requestInput: Record<string, unknown>;
4635
+ executionAuthScopeDigest?: string | null;
4636
+ staleAfterSeconds?: number | null;
4637
+ }): Promise<string[]> {
4638
+ return await this.compatibleDurableToolCallCacheKeysForScope({
4639
+ ...input,
4640
+ playLocalScope: this.currentGovernance.currentPlayId,
4641
+ });
4642
+ }
4643
+
4266
4644
  private async legacyDurableToolCallCacheKey(input: {
4267
4645
  toolId: string;
4268
4646
  requestInput: Record<string, unknown>;
@@ -4429,6 +4807,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4429
4807
  }) => Promise<T>;
4430
4808
  },
4431
4809
  ): Promise<T> {
4810
+ this.assertNoPendingDurableSuspension();
4432
4811
  let releaseExternalCallSlot: (() => void) | null = null;
4433
4812
  const execute = async (leaseId: string | null): Promise<T> =>
4434
4813
  await opts.execute({
@@ -4471,7 +4850,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4471
4850
  cacheEpochMs: this.durableCallCacheEpochMs,
4472
4851
  });
4473
4852
  try {
4474
- return await executeWithDurableRuntimeReceipt<T>({
4853
+ const output = await executeWithDurableRuntimeReceipt<T>({
4475
4854
  operation,
4476
4855
  id,
4477
4856
  runId: this.currentReceiptOwnerRunId,
@@ -4499,6 +4878,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4499
4878
  log: (message) => this.runtimeLog(message),
4500
4879
  execute: ({ leaseId }) => execute(leaseId),
4501
4880
  });
4881
+ // Every receipt publication and cache return crosses this one success
4882
+ // seam. A customer catch cannot turn a pending durable handoff into a
4883
+ // reusable completed value.
4884
+ this.assertNoPendingDurableSuspension();
4885
+ return output;
4502
4886
  } finally {
4503
4887
  const release = releaseExternalCallSlot as (() => void) | null;
4504
4888
  release?.();
@@ -4705,6 +5089,62 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
4705
5089
  });
4706
5090
  }
4707
5091
 
5092
+ /**
5093
+ * A forced call bypasses the immutable Tool Result Receipt, so its physical
5094
+ * gateway invocation must bypass that receipt's replay identity too. Keep
5095
+ * the transport key deterministic for one logical call: a retry of that
5096
+ * call still converges, while a later explicit force gets a fresh identity.
5097
+ */
5098
+ private toolTransportReceiptKey(input: {
5099
+ toolId: string;
5100
+ resultReceiptKey: string;
5101
+ providerOperationKey: string;
5102
+ force?: boolean;
5103
+ }): string {
5104
+ if (input.force !== true) return input.resultReceiptKey;
5105
+ return `${buildDurableToolReceiptPrefix({
5106
+ orgId: this.#options.orgId,
5107
+ toolId: input.toolId,
5108
+ })}${stableDigest(`forced-provider-invocation:${input.providerOperationKey}`)}`;
5109
+ }
5110
+
5111
+ /**
5112
+ * Ordinary operations do not carry provider-idempotency metadata, but the
5113
+ * execution relay still needs a semantic signal that an explicit force is
5114
+ * a new provider operation. Unsafe-write operations already carry their
5115
+ * forced provider key directly, so only ordinary calls need this attested
5116
+ * canonical-operation salt.
5117
+ */
5118
+ private providerForceRefreshIdentitiesForToolCall(input: {
5119
+ operationKey: string;
5120
+ force?: boolean;
5121
+ requiresExecutionFence?: boolean;
5122
+ logicalCallId?: string;
5123
+ }): string[] {
5124
+ if (input.force !== true || input.requiresExecutionFence === true) {
5125
+ return [];
5126
+ }
5127
+ return [
5128
+ this.providerIdempotencyKeyForToolCall({
5129
+ operationKey: input.operationKey,
5130
+ force: true,
5131
+ logicalCallId: input.logicalCallId,
5132
+ }),
5133
+ ];
5134
+ }
5135
+
5136
+ private providerForceRefreshOptionsForToolCall(input: {
5137
+ operationKey: string;
5138
+ force?: boolean;
5139
+ requiresExecutionFence?: boolean;
5140
+ logicalCallId?: string;
5141
+ }): Pick<ToolExecutionApiOptions, 'providerOperationForceRefreshIdentities'> {
5142
+ const identities = this.providerForceRefreshIdentitiesForToolCall(input);
5143
+ return identities.length > 0
5144
+ ? { providerOperationForceRefreshIdentities: identities }
5145
+ : {};
5146
+ }
5147
+
4708
5148
  /**
4709
5149
  * Bounded per-cell producer trace (ADR 0018).
4710
5150
  *
@@ -6125,6 +6565,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6125
6565
  input: TInput,
6126
6566
  options?: { description?: string },
6127
6567
  ): Promise<TOutput> {
6568
+ this.assertNoPendingDurableSuspension();
6128
6569
  if (!isRuntimeStepProgram(program)) {
6129
6570
  throw new Error('ctx.runSteps(program, input) requires steps().');
6130
6571
  }
@@ -6149,6 +6590,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6149
6590
  input: MapFieldDefinition<T, TColumns>,
6150
6591
  options?: RuntimeDatasetOptions<T>,
6151
6592
  ): Promise<PlayDataset<Record<string, unknown>>> {
6593
+ this.assertNoPendingDurableSuspension();
6152
6594
  if (rowContext.getStore()) {
6153
6595
  throw new Error(
6154
6596
  'Nested ctx.dataset() is not supported. Flatten your columns into one dataset, or keep custom per-row logic inside a single column.',
@@ -6305,6 +6747,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6305
6747
  this.#options.playName &&
6306
6748
  this.#options.runId,
6307
6749
  );
6750
+ // The invocation id is deterministic for a replay: mapInvocationIndex is
6751
+ // rebuilt in authored call order. Look up a previously suspended plan now,
6752
+ // before a stateful source query can decide that a sibling action is no
6753
+ // longer eligible because an earlier action already completed.
6754
+ const replayMapInvocationId = `${normalizedMapNamespace}:${this.mapInvocationIndex}`;
6755
+ const suspendedInputRows = shouldStreamRuntimeBackedDataset
6756
+ ? null
6757
+ : (this.checkpoint.mapFrames?.[replayMapInvocationId]
6758
+ ?.suspendedInputRows ?? null);
6308
6759
  if (shouldStreamRuntimeBackedDataset) {
6309
6760
  // Net-new admission happens one source page at a time. Its progress and
6310
6761
  // returned dataset describe admitted rows, not every provider/CSV row
@@ -6954,11 +7405,24 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6954
7405
  },
6955
7406
  });
6956
7407
  }
6957
- const rawMaterializedItems = await materializePlayDatasetInput(items, {
6958
- onRow: (row) => {
6959
- materializationMemoryTracker.track(row);
6960
- },
6961
- });
7408
+ const rawMaterializedEntries = suspendedInputRows?.length
7409
+ ? suspendedInputRows.map((row) => ({
7410
+ item: cloneCsvAliasedRow(row.data) as T,
7411
+ inputIndex: row.inputIndex,
7412
+ }))
7413
+ : (
7414
+ await materializePlayDatasetInput(items, {
7415
+ onRow: (row) => {
7416
+ materializationMemoryTracker.track(row);
7417
+ },
7418
+ })
7419
+ ).map((item, inputIndex) => ({ item, inputIndex }));
7420
+ const rawMaterializedItems = rawMaterializedEntries.map(
7421
+ (entry) => entry.item,
7422
+ );
7423
+ const originalInputIndexByItem = new Map<T, number>(
7424
+ rawMaterializedEntries.map(({ item, inputIndex }) => [item, inputIndex]),
7425
+ );
6962
7426
  // Silent dedupe of duplicate explicit map keys: keep the first row per
6963
7427
  // canonical key and drop subsequent duplicates. Deduping the materialized
6964
7428
  // input here keeps every downstream derivation (rawItems, itemsToProcess,
@@ -6971,7 +7435,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6971
7435
  resolver: (item, index) =>
6972
7436
  explicitKeyResolver(
6973
7437
  this.toOutputRow(item as Record<string, unknown>),
6974
- index,
7438
+ originalInputIndexByItem.get(item) ?? index,
6975
7439
  ),
6976
7440
  })
6977
7441
  : { rows: rawMaterializedItems, droppedCount: 0, duplicateKeys: [] };
@@ -6988,7 +7452,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
6988
7452
  itemsToProcess = materializedItems.map((item) =>
6989
7453
  this.toOutputRow(item as Record<string, unknown>),
6990
7454
  );
6991
- itemOriginalIndexes = materializedItems.map((_item, index) => index);
7455
+ itemOriginalIndexes = materializedItems.map(
7456
+ (item, index) => originalInputIndexByItem.get(item) ?? index,
7457
+ );
6992
7458
  const mapStartSeedRowsByKey = new Map<string, Record<string, unknown>>();
6993
7459
  if (this.#options.onMapStart) {
6994
7460
  // ALWAYS stamp the canonical seed row key — including no-explicit-key
@@ -7376,6 +7842,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
7376
7842
  pendingRowsCount: 0,
7377
7843
  failedRowsCount: durableFailedRows,
7378
7844
  activeBoundaryId: null,
7845
+ suspendedInputRows: undefined,
7379
7846
  updatedAt: Date.now(),
7380
7847
  });
7381
7848
  }
@@ -8224,6 +8691,67 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8224
8691
  });
8225
8692
  return WAITING_ROW;
8226
8693
  }
8694
+ if (isPlayExecutionSuspendedError(error)) {
8695
+ const completedKeys = new Set(
8696
+ completedRowsToPersist.map((row) => row.key),
8697
+ );
8698
+ const suspendedInputRows = items.flatMap((item, index) => {
8699
+ const rowKey = executionRowKey(
8700
+ this.toOutputRow(item as Record<string, unknown>),
8701
+ index,
8702
+ );
8703
+ if (completedKeys.has(rowKey)) return [];
8704
+ return [
8705
+ {
8706
+ inputIndex: executionRowIndex(index),
8707
+ data: this.toPersistedOutputRow(
8708
+ this.toOutputRow(item as Record<string, unknown>),
8709
+ ),
8710
+ },
8711
+ ];
8712
+ });
8713
+ const existingFrame =
8714
+ this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
8715
+ this.setMapFrame({
8716
+ ...(existingFrame ?? {
8717
+ mapInvocationId: mapScope.mapInvocationId,
8718
+ mapNodeId: mapScope.mapNodeId ?? null,
8719
+ logicalNamespace: mapScope.logicalNamespace,
8720
+ artifactTableNamespace: mapScope.artifactTableNamespace,
8721
+ totalRows,
8722
+ completedRowKeys: [],
8723
+ pendingRowKeys: [],
8724
+ startedAt: Date.now(),
8725
+ }),
8726
+ status: 'suspended',
8727
+ pendingRowKeys: suspendedInputRows.map(({ data, inputIndex }) =>
8728
+ executionRowKey(data, inputIndex),
8729
+ ),
8730
+ suspendedInputRows,
8731
+ activeBoundaryId:
8732
+ error.suspension.kind === 'sleep'
8733
+ ? error.suspension.boundaryId
8734
+ : null,
8735
+ updatedAt: Date.now(),
8736
+ });
8737
+ this.emitExecutionEvent({
8738
+ type: 'map.suspended',
8739
+ mapInvocationId: mapScope.mapInvocationId,
8740
+ mapNodeId: mapScope.mapNodeId ?? null,
8741
+ logicalNamespace: mapScope.logicalNamespace,
8742
+ artifactTableNamespace: mapScope.artifactTableNamespace,
8743
+ completedRows: completedKeys.size,
8744
+ failedRows: 0,
8745
+ totalRows,
8746
+ at: Date.now(),
8747
+ });
8748
+ // ctx.sleep() emitted its boundary checkpoint before control reached
8749
+ // this row catch. Emit once more after recording the admitted plan so
8750
+ // the scheduler can resume the same action set if the worker exits
8751
+ // immediately after handing off the suspension.
8752
+ this.#options.onBatchComplete?.(this.checkpoint);
8753
+ throw error;
8754
+ }
8227
8755
  const fieldName = rowContext.getStore()?.fieldName;
8228
8756
  updateMapFrameProgress({
8229
8757
  status: 'failed',
@@ -8257,16 +8785,51 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8257
8785
  try {
8258
8786
  settledResults[idx] = await runRow(idx);
8259
8787
  } catch (error) {
8260
- firstRowError = error;
8788
+ firstRowError = this.pendingDurableSuspension
8789
+ ? new PlayExecutionSuspendedError(this.pendingDurableSuspension)
8790
+ : (firstRowError ?? error);
8791
+ this.wakeToolDispatcher();
8261
8792
  throw error;
8262
8793
  }
8263
8794
  }
8264
8795
  });
8265
8796
 
8266
- await this.drainQueuedWork(workers);
8797
+ // Drain queued work but exit early if a suspension error occurs.
8798
+ // When a worker suspends (e.g., ctx.sleep), we must propagate the
8799
+ // suspension immediately without waiting for other concurrent rows to
8800
+ // settle. This prevents restart loops and false completions.
8801
+ await this.drainQueuedWork(workers, () => {
8802
+ if (firstRowError != null) {
8803
+ if (
8804
+ isPlayExecutionSuspendedError(firstRowError) ||
8805
+ isPlayRowExecutionSuspendedError(firstRowError)
8806
+ )
8807
+ return firstRowError;
8808
+ }
8809
+ return null;
8810
+ });
8267
8811
  this.lastDatasetStep = datasetStep;
8268
8812
  this.activeDatasetStep = null;
8269
8813
 
8814
+ // If drainQueuedWork exited early due to suspension, throw it immediately
8815
+ // WITHOUT waiting for Promise.allSettled(workers). The worker promises
8816
+ // may never settle (e.g., sibling rows blocked on long-running operations),
8817
+ // and we must not block suspension propagation on them.
8818
+ if (firstRowError != null) {
8819
+ if (
8820
+ isPlayExecutionSuspendedError(firstRowError) ||
8821
+ isPlayRowExecutionSuspendedError(firstRowError)
8822
+ ) {
8823
+ // Use the existing failure barrier to commit accepted terminal rows
8824
+ // before the caller hands this invocation back to the scheduler.
8825
+ throw new FailFastMapRowsError({
8826
+ cause: firstRowError,
8827
+ completedRows: completedRowsToPersist,
8828
+ failedRows: failedRowsToPersist,
8829
+ });
8830
+ }
8831
+ }
8832
+
8270
8833
  const settled = await Promise.allSettled(workers);
8271
8834
  const rejected = settled.find(
8272
8835
  (result): result is PromiseRejectedResult => result.status === 'rejected',
@@ -8356,8 +8919,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8356
8919
  await incrementalPersistence?.checkpoint(checkpointUpdates);
8357
8920
  await incrementalPersistence?.flush();
8358
8921
  this.pendingRowEventBoundaries = [];
8359
- this.#options.onBatchComplete?.(this.checkpoint);
8360
- throw new PlayExecutionSuspendedError({
8922
+ this.suspendDurably({
8361
8923
  kind: 'integration_event_batch',
8362
8924
  boundaries: uniqueBoundaries,
8363
8925
  });
@@ -8452,7 +9014,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8452
9014
  if (
8453
9015
  (!continueOnProviderUnavailable &&
8454
9016
  step.continueOnProviderUnavailable !== true) ||
8455
- (!(error instanceof ProviderExhaustedError) &&
9017
+ (!isProviderExhaustedFailure(error) &&
8456
9018
  !isProviderUnavailable(error))
8457
9019
  ) {
8458
9020
  throw error;
@@ -8537,6 +9099,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8537
9099
  let value: unknown;
8538
9100
  try {
8539
9101
  value = await runStep();
9102
+ // A step-program resolver has the same publication boundary as a
9103
+ // normal mapped field. Its customer callback may catch `ctx.sleep()`;
9104
+ // do not turn that parked invocation into a waterfall output/cell.
9105
+ this.assertNoPendingDurableSuspension();
8540
9106
  } catch (error) {
8541
9107
  // A provider admission, transient upstream failure, or exhausted
8542
9108
  // managed provider account is a visible unavailable leg, not a row
@@ -8545,8 +9111,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8545
9111
  // author hand-writing error handling.
8546
9112
  if (
8547
9113
  !continueOnProviderUnavailable ||
8548
- (!(error instanceof ProviderExhaustedError) &&
8549
- !isProviderUnavailable(error))
9114
+ (!isProviderExhaustedFailure(error) && !isProviderUnavailable(error))
8550
9115
  ) {
8551
9116
  throw error;
8552
9117
  }
@@ -8755,7 +9320,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
8755
9320
 
8756
9321
  const source = Function.prototype.toString.call(resolver);
8757
9322
  return (
8758
- !source.includes('.tools.execute(') && !source.includes('.runPlay(')
9323
+ !source.includes('.tools.execute(') &&
9324
+ !source.includes('.runPlay(') &&
9325
+ !source.includes('.runPlayAsync(')
8759
9326
  );
8760
9327
  });
8761
9328
  }
@@ -9117,7 +9684,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9117
9684
  });
9118
9685
  }
9119
9686
 
9120
- private async drainQueuedWork<T>(promises: Promise<T>[]): Promise<void> {
9687
+ private async drainQueuedWork<T>(
9688
+ promises: Promise<T>[],
9689
+ earlyExitCheck?: () => unknown,
9690
+ ): Promise<void> {
9121
9691
  // One dispatcher owns ready work, fixed lane coalescing deadlines, and the
9122
9692
  // bounded in-flight registry. A row resumes after its own receipt persists;
9123
9693
  // it never waits for unrelated sibling calls from the previous column.
@@ -9149,6 +9719,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9149
9719
  try {
9150
9720
  let pass = 0;
9151
9721
  while (true) {
9722
+ // Check for early exit FIRST, before any dispatch or wait logic.
9723
+ // When a row suspends, we must exit immediately without waiting for
9724
+ // sibling rows or tool dispatcher wake events.
9725
+ const suspension = earlyExitCheck?.();
9726
+ if (suspension != null) {
9727
+ // Queued calls have not started; reject them at the suspension fence.
9728
+ // Already-dispatched calls must settle while this attempt still owns
9729
+ // their receipts. Runner teardown releases leases, so detaching live
9730
+ // provider requests here could duplicate them on the next resume.
9731
+ this.rejectQueuedToolCalls(suspension);
9732
+ await Promise.allSettled([...inFlightToolExecutions]);
9733
+ break;
9734
+ }
9735
+
9152
9736
  if (this.toolDispatcherFailure != null) {
9153
9737
  this.rejectQueuedToolCalls(this.toolDispatcherFailure);
9154
9738
  await Promise.allSettled([...inFlightToolExecutions]);
@@ -9237,7 +9821,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9237
9821
  return resolver;
9238
9822
  }
9239
9823
 
9240
- return await resolver(row, this, fields, index, previousCell);
9824
+ const value = await resolver(row, this, fields, index, previousCell);
9825
+ // This is the common user-code return seam for every mapped field path
9826
+ // (incremental, pure, and step-program adapters). Check before callers
9827
+ // serialize a cell, emit a completion event, or advance a map frame.
9828
+ this.assertNoPendingDurableSuspension();
9829
+ return value;
9241
9830
  }
9242
9831
 
9243
9832
  private toOutputRow<T>(item: T): Record<string, unknown> {
@@ -9358,12 +9947,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9358
9947
  !eventWaitHandler &&
9359
9948
  !isQueryResultDatasetReadRequest(toolId, input) &&
9360
9949
  !isAlwaysFreshIntegrationTool(toolId);
9361
- let durableCacheKey = await this.durableToolCallCacheKey({
9362
- toolId,
9363
- requestInput: input,
9364
- executionAuthScopeDigest,
9365
- staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
9366
- });
9950
+ let compatibleDurableCacheKeys =
9951
+ await this.compatibleDurableToolCallCacheKeys({
9952
+ toolId,
9953
+ requestInput: input,
9954
+ executionAuthScopeDigest,
9955
+ staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
9956
+ });
9957
+ let durableCacheKey = compatibleDurableCacheKeys[0];
9958
+ if (!durableCacheKey) {
9959
+ throw new Error(`Tool ${toolId} is missing a primary durable cache key.`);
9960
+ }
9367
9961
  let legacyDurableCacheKey = await this.legacyDurableToolCallCacheKey({
9368
9962
  toolId,
9369
9963
  requestInput: input,
@@ -9390,7 +9984,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9390
9984
  // incompatible state deliberately forgoes caching until it is repaired.
9391
9985
  let bypassDirectResultCache = false;
9392
9986
  const checkpointCacheKeys = [
9393
- durableCacheKey,
9987
+ ...compatibleDurableCacheKeys,
9394
9988
  ...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
9395
9989
  ];
9396
9990
 
@@ -9408,6 +10002,28 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9408
10002
  if (toolId === 'run_javascript') {
9409
10003
  throw new Error(DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE);
9410
10004
  }
10005
+ // A user-requested rerun first asks its exact source run whether this
10006
+ // logical call already completed. This is stronger than the shared cache:
10007
+ // it is an explicit, authenticated pointer to one known source receipt.
10008
+ if (cacheableToolResult && !toolCachePolicy.force) {
10009
+ const sourceRunRecovered = await this.recoverSourceRunToolReceipt({
10010
+ toolId,
10011
+ logicalCallId,
10012
+ executionAuthScopeDigest,
10013
+ cacheKey: durableCacheKey,
10014
+ });
10015
+ if (sourceRunRecovered) {
10016
+ this.runtimeLog(
10017
+ `Calling tool: ${toolId} recovered from source run ${this.#options.replayedFromRunId}`,
10018
+ );
10019
+ this.emitToolCallLifecycle('started', toolId, normalizedKey);
10020
+ this.emitToolCallLifecycle('settled', toolId, normalizedKey, {
10021
+ outcome: 'cached',
10022
+ durationMs: 0,
10023
+ });
10024
+ return this.materializeToolExecutionResult(sourceRunRecovered);
10025
+ }
10026
+ }
9411
10027
  // Tool-call budget is charged by the Governor's acquireToolSlot in
9412
10028
  // callToolExecutionAPI, so each real outbound call is counted exactly once
9413
10029
  // (cached/checkpoint recoveries never acquire a slot and so never charge).
@@ -9476,6 +10092,45 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9476
10092
  // reader seam rather than teaching the deep Module an untyped result.
9477
10093
  // New direct calls use the real Tool Result Receipt Adapter below.
9478
10094
  if (cacheableToolResult && !toolCachePolicy.force) {
10095
+ // A source action can explicitly carry forward a previous result
10096
+ // contract after a local version bump. Read those exact older keys
10097
+ // before the canonical key; do not derive fallbacks from a deploy.
10098
+ for (const compatibleCacheKey of compatibleDurableCacheKeys.slice(
10099
+ 1,
10100
+ )) {
10101
+ const compatibleReceipt =
10102
+ await this.withRuntimeReceiptHydrationTurn(
10103
+ async () =>
10104
+ await this.getRuntimeStepReceipt(compatibleCacheKey),
10105
+ );
10106
+ if (
10107
+ compatibleReceipt &&
10108
+ (compatibleReceipt.status === 'completed' ||
10109
+ compatibleReceipt.status === 'skipped') &&
10110
+ compatibleReceipt.output !== undefined
10111
+ ) {
10112
+ const recoveredCompatible = markToolExecuteResultExecutionOutcome(
10113
+ this.runtimeReceiptOutput(compatibleReceipt),
10114
+ toolExecutionOutcomeForDurableReceipt({
10115
+ source: 'cache',
10116
+ receiptKey: compatibleCacheKey,
10117
+ cacheKey: directCacheKey,
10118
+ }),
10119
+ );
10120
+ if (
10121
+ this.canReuseToolResultForCurrentResponseContract(
10122
+ recoveredCompatible,
10123
+ )
10124
+ ) {
10125
+ this.runtimeLog(
10126
+ `Calling tool: ${toolId} recovered from an explicit action compatibility receipt`,
10127
+ );
10128
+ return isToolExecuteResult(recoveredCompatible)
10129
+ ? this.materializeToolExecutionResult(recoveredCompatible)
10130
+ : recoveredCompatible;
10131
+ }
10132
+ }
10133
+ }
9479
10134
  const historicalReceipt = await this.withRuntimeReceiptHydrationTurn(
9480
10135
  async () => await this.getRuntimeStepReceipt(directCacheKey),
9481
10136
  );
@@ -9592,14 +10247,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9592
10247
  }
9593
10248
  const useDirectResultReceipts =
9594
10249
  cacheableToolResult && !bypassDirectResultCache;
10250
+ const providerOperationKey = this.providerIdempotencyKeyForToolCall({
10251
+ operationKey: providerIdempotencyKeyBase,
10252
+ force: cacheableToolResult && toolCachePolicy.force,
10253
+ logicalCallId,
10254
+ });
9595
10255
  const physicalDirectKey = useDirectResultReceipts
9596
- ? directCacheKey
10256
+ ? this.toolTransportReceiptKey({
10257
+ toolId,
10258
+ resultReceiptKey: directCacheKey,
10259
+ providerOperationKey,
10260
+ force: toolCachePolicy.force,
10261
+ })
9597
10262
  : `${buildDurableToolReceiptPrefix({
9598
10263
  orgId: this.#options.orgId ?? 'unknown-org',
9599
10264
  toolId,
9600
10265
  })}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`;
9601
10266
  const resultReceiptKey = directResultReceiptKey;
9602
- const providerOperationKey = providerIdempotencyKeyBase;
9603
10267
  const toolRetryPolicy = await this.#options
9604
10268
  .getToolRetryPolicy?.(toolId, input)
9605
10269
  .catch(() => null);
@@ -9665,16 +10329,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9665
10329
  ? resultReceiptKey
9666
10330
  : physicalDirectKey,
9667
10331
  providerIdempotencyKey:
9668
- this.providerIdempotencyKeyForToolCall({
9669
- operationKey:
9670
- useDirectResultReceipts && operationKey
9671
- ? providerIdempotencyKeyBase
9672
- : physicalDirectKey,
9673
- force: cacheableToolResult && toolCachePolicy.force,
9674
- logicalCallId,
9675
- }),
10332
+ useDirectResultReceipts && operationKey
10333
+ ? providerOperationKey
10334
+ : physicalDirectKey,
9676
10335
  }
9677
10336
  : {}),
10337
+ ...(useDirectResultReceipts && operationKey
10338
+ ? this.providerForceRefreshOptionsForToolCall({
10339
+ operationKey: providerIdempotencyKeyBase,
10340
+ force: toolCachePolicy.force,
10341
+ requiresExecutionFence:
10342
+ toolRetryPolicy?.requiresExecutionFence,
10343
+ logicalCallId,
10344
+ })
10345
+ : {}),
9678
10346
  playNodeScope: buildPlayNodeScope({
9679
10347
  toolId,
9680
10348
  callKey: normalizedKey,
@@ -9721,6 +10389,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9721
10389
  : {}),
9722
10390
  withCompletedReceiptHydration: (hydrate) =>
9723
10391
  this.withRuntimeReceiptHydrationTurn(hydrate),
10392
+ onStored: async ({ resultReceiptKey }) => {
10393
+ await this.persistSourceRunToolReceiptManifest({
10394
+ toolId,
10395
+ logicalCallId,
10396
+ executionAuthScopeDigest,
10397
+ resultReceiptKey,
10398
+ });
10399
+ },
9724
10400
  ...(toolRetryPolicy?.requiresExecutionFence === true
9725
10401
  ? {
9726
10402
  executionLock: {
@@ -9898,9 +10574,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9898
10574
  providerIdempotencyKeyBase,
9899
10575
  cacheable: cacheableToolResult,
9900
10576
  receiptKey: cacheableToolResult ? durableCacheKey : null,
9901
- legacyReceiptKeys: legacyDurableCacheKey
9902
- ? [legacyDurableCacheKey]
9903
- : [],
10577
+ legacyReceiptKeys: [
10578
+ ...compatibleDurableCacheKeys.slice(1),
10579
+ ...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
10580
+ ],
9904
10581
  executionAuthScopeDigest,
9905
10582
  requiresExecutionFence:
9906
10583
  rowToolRetryPolicy?.requiresExecutionFence === true,
@@ -9968,12 +10645,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9968
10645
  this.invalidateToolAuthScopeDigest(toolId);
9969
10646
  executionAuthScopeDigest =
9970
10647
  (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
9971
- durableCacheKey = await this.durableToolCallCacheKey({
9972
- toolId,
9973
- requestInput: input,
9974
- executionAuthScopeDigest,
9975
- staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
9976
- });
10648
+ compatibleDurableCacheKeys =
10649
+ await this.compatibleDurableToolCallCacheKeys({
10650
+ toolId,
10651
+ requestInput: input,
10652
+ executionAuthScopeDigest,
10653
+ staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
10654
+ });
10655
+ durableCacheKey = compatibleDurableCacheKeys[0];
10656
+ if (!durableCacheKey) {
10657
+ throw new Error(
10658
+ `Tool ${toolId} is missing a primary durable cache key after auth refresh.`,
10659
+ );
10660
+ }
9977
10661
  legacyDurableCacheKey = await this.legacyDurableToolCallCacheKey({
9978
10662
  toolId,
9979
10663
  requestInput: input,
@@ -9993,7 +10677,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
9993
10677
  checkpointCacheKeys.splice(
9994
10678
  0,
9995
10679
  checkpointCacheKeys.length,
9996
- durableCacheKey,
10680
+ ...compatibleDurableCacheKeys,
9997
10681
  ...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
9998
10682
  );
9999
10683
  }
@@ -10019,6 +10703,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10019
10703
  workflowId: this.#options.workflowId,
10020
10704
  orgId: this.#options.orgId,
10021
10705
  executorToken: this.#options.executorToken,
10706
+ slackHitlDefaultTimeoutMs:
10707
+ this.currentRuntimeSemantics.slackHitl.defaultTimeoutMs,
10708
+ slackHitlTimeoutOutcome:
10709
+ this.currentRuntimeContract.slackHitlTimeoutOutcome,
10022
10710
  };
10023
10711
  const basePreparedBoundary = await handler.prepare({
10024
10712
  payload: input,
@@ -10039,23 +10727,65 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10039
10727
  }
10040
10728
  : basePreparedBoundary;
10041
10729
 
10042
- const existing =
10043
- this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId];
10730
+ const existing = assertKnownResolvedBoundary(
10731
+ preparedBoundary.boundaryId,
10732
+ this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId],
10733
+ );
10044
10734
  if (existing?.kind === 'integration_event' && 'output' in existing) {
10045
10735
  const output = recordOrNull(existing.output);
10046
10736
  if (output?.timed_out === true) {
10047
- await handler.onTimeout?.({
10048
- payload: input,
10049
- context: eventContext,
10050
- boundary: {
10737
+ // The runtime proxy is recreated for each worker invocation. Preserve
10738
+ // the policy that armed this boundary so its timeout finalizer and
10739
+ // parking behavior survive a durable replay.
10740
+ const timeoutPolicy =
10741
+ existing.timeoutPolicy ?? preparedBoundary.timeoutPolicy;
10742
+ // An explicit boundary policy may deliberately omit parking (the
10743
+ // new on_timeout: "return_null" escape hatch). Only fall back to the
10744
+ // handler's historical default when this is an old boundary with no
10745
+ // persisted policy at all.
10746
+ const suspendAfterTimeoutMs = timeoutPolicy
10747
+ ? timeoutPolicy.suspendAfterTimeoutMs
10748
+ : handler.suspendAfterTimeoutMs;
10749
+ try {
10750
+ await handler.onTimeout?.({
10751
+ payload: input,
10752
+ context: eventContext,
10753
+ boundary: {
10754
+ boundaryId: preparedBoundary.boundaryId,
10755
+ eventKey: preparedBoundary.eventKey,
10756
+ timeoutMs: preparedBoundary.timeoutMs,
10757
+ provider: existing.provider ?? handler.provider,
10758
+ toolId: existing.toolId ?? handler.toolId,
10759
+ ...(existing.messageRef
10760
+ ? { messageRef: existing.messageRef }
10761
+ : {}),
10762
+ ...(timeoutPolicy ? { timeoutPolicy } : {}),
10763
+ },
10764
+ });
10765
+ } catch (error) {
10766
+ this.recordSlackHitlTimeoutFailure(error);
10767
+ throw error;
10768
+ }
10769
+ if (timeoutPolicy?.timeoutOutcome === 'error') {
10770
+ // The provider finalizer above must complete first. That ordering
10771
+ // makes an expired Slack card and the terminal Play state agree,
10772
+ // while provider finalization failures stay retryable instead.
10773
+ const error = new SlackHitlTimeoutError(timeoutPolicy.timeoutError);
10774
+ this.recordSlackHitlTimeoutFailure(error);
10775
+ throw error;
10776
+ }
10777
+ if (suspendAfterTimeoutMs !== undefined) {
10778
+ this.runtimeLog(
10779
+ `Integration event ${preparedBoundary.boundaryId}: timeout finalized externally; keeping Play parked`,
10780
+ );
10781
+ this.suspendDurably({
10782
+ kind: 'integration_event',
10051
10783
  boundaryId: preparedBoundary.boundaryId,
10052
10784
  eventKey: preparedBoundary.eventKey,
10053
- timeoutMs: preparedBoundary.timeoutMs,
10054
- provider: existing.provider ?? handler.provider,
10055
- toolId: existing.toolId ?? handler.toolId,
10785
+ timeoutMs: suspendAfterTimeoutMs,
10056
10786
  ...(existing.messageRef ? { messageRef: existing.messageRef } : {}),
10057
- },
10058
- });
10787
+ });
10788
+ }
10059
10789
  }
10060
10790
  this.runtimeLog(
10061
10791
  `Integration event ${preparedBoundary.boundaryId}: recovered response from checkpoint`,
@@ -10091,6 +10821,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10091
10821
  timeoutMs: boundary.timeoutMs,
10092
10822
  provider: boundary.provider,
10093
10823
  toolId: boundary.toolId,
10824
+ ...(boundary.timeoutPolicy
10825
+ ? { timeoutPolicy: boundary.timeoutPolicy }
10826
+ : {}),
10094
10827
  ...(rowScope
10095
10828
  ? {
10096
10829
  scope: {
@@ -10105,9 +10838,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10105
10838
  ...(boundary.messageRef ? { messageRef: boundary.messageRef } : {}),
10106
10839
  },
10107
10840
  };
10108
- this.#options.onBatchComplete?.(this.checkpoint);
10109
-
10110
10841
  if (rowScope) {
10842
+ this.#options.onBatchComplete?.(this.checkpoint);
10111
10843
  throw new PlayRowExecutionSuspendedError({
10112
10844
  boundaryId: boundary.boundaryId,
10113
10845
  eventKey: boundary.eventKey,
@@ -10116,7 +10848,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10116
10848
  });
10117
10849
  }
10118
10850
 
10119
- throw new PlayExecutionSuspendedError({
10851
+ this.suspendDurably({
10120
10852
  kind: 'integration_event',
10121
10853
  boundaryId: boundary.boundaryId,
10122
10854
  eventKey: boundary.eventKey,
@@ -10131,6 +10863,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10131
10863
  input: Record<string, unknown>,
10132
10864
  options?: PlayCallOptions,
10133
10865
  ): Promise<TOutput> {
10866
+ this.assertNoPendingDurableSuspension();
10134
10867
  if (
10135
10868
  arguments.length === 3 &&
10136
10869
  typeof playRef === 'object' &&
@@ -10140,19 +10873,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10140
10873
  'ctx.runPlay(...) signature is ctx.runPlay(key, playRef, input, { description }). Add a stable call key as the first argument.',
10141
10874
  );
10142
10875
  }
10143
- const resolvedName =
10144
- typeof playRef === 'string'
10145
- ? playRef
10146
- : playRef && typeof playRef.playName === 'string'
10147
- ? playRef.playName
10148
- : playRef && typeof playRef.name === 'string'
10149
- ? playRef.name
10150
- : '';
10876
+ const resolvedName = resolvePlayReferenceName(playRef);
10151
10877
 
10152
10878
  if (!resolvedName.trim()) {
10153
10879
  throw new Error('ctx.runPlay(...) requires a resolvable play name.');
10154
10880
  }
10155
- validatePlayAuthoringField('ctx.runPlay.playRef', playRef);
10881
+ // Imported Play dependencies are callable proxy functions at runtime. The
10882
+ // authoring contract describes their durable identity, so validate the
10883
+ // normalized name rather than requiring the ephemeral handle itself to be
10884
+ // JSON-shaped.
10885
+ validatePlayAuthoringField('ctx.runPlay.playRef', resolvedName);
10156
10886
  validatePlayAuthoringField('ctx.runPlay.input', input);
10157
10887
  assertNoSecretTaint(input, 'ctx.runPlay input');
10158
10888
  validatePlayAuthoringField('ctx.runPlay.key', key);
@@ -10258,12 +10988,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10258
10988
  runtimeContract: resolvePlayArtifactRuntimeContract({
10259
10989
  authoringContractEdition:
10260
10990
  childCompatibility.authoringContractEdition,
10991
+ slackHitlTimeoutOutcome:
10992
+ childCompatibility.slackHitlTimeoutOutcome,
10261
10993
  toolResponseContract: childToolResponseContract,
10262
10994
  toolResponseReceiptRevision: childToolResponseReceiptRevision,
10263
10995
  }),
10264
10996
  },
10265
10997
  () => this.executeResolvedPlay(resolvedPlay, this, input),
10266
10998
  );
10999
+ // Inline composition shares this Play Invocation. A child may catch a
11000
+ // parent suspension, but it cannot publish child success while that
11001
+ // handoff is still pending.
11002
+ this.assertNoPendingDurableSuspension();
10267
11003
  this.inlineChildAggregates.ok += 1;
10268
11004
  if (rowStore) {
10269
11005
  this.emitScopedFieldMetaUpdate({
@@ -10313,6 +11049,81 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10313
11049
  }
10314
11050
  }
10315
11051
 
11052
+ async runPlayAsync(
11053
+ key: string,
11054
+ playRef: string | { playName?: string; name?: string },
11055
+ input: Record<string, unknown>,
11056
+ options: PlayAsyncCallOptions,
11057
+ ): Promise<AsyncPlayRunRef> {
11058
+ const resolvedName = resolvePlayReferenceName(playRef);
11059
+ if (!resolvedName.trim()) {
11060
+ throw new Error(
11061
+ 'ctx.runPlayAsync(...) requires a resolvable Play name or definition handle.',
11062
+ );
11063
+ }
11064
+ validatePlayAuthoringField('ctx.runPlayAsync.playRef', resolvedName);
11065
+ validatePlayAuthoringField('ctx.runPlayAsync.input', input);
11066
+ assertNoSecretTaint(input, 'ctx.runPlayAsync input');
11067
+ validatePlayAuthoringField('ctx.runPlayAsync.key', key);
11068
+ validatePlayAuthoringField(
11069
+ 'ctx.runPlayAsync.options.description',
11070
+ options?.description,
11071
+ );
11072
+ const normalizedKey = this.normalizeContextKey(key, 'runPlayAsync');
11073
+ if (!this.#options.launchPlayAsync) {
11074
+ throw new Error(
11075
+ 'ctx.runPlayAsync(...) is unavailable because no async Play admission Adapter was configured.',
11076
+ );
11077
+ }
11078
+ const ancestryPlayIds =
11079
+ this.currentExecutionGovernor.snapshot().ancestryPlayIds;
11080
+ admitAsyncPlayLineage({
11081
+ ancestryPlayIds,
11082
+ childPlayName: resolvedName,
11083
+ });
11084
+
11085
+ const rowScope = rowContext.getStore();
11086
+ const invocationKey = sha256Hex(
11087
+ stableStringify({
11088
+ parentRunId: this.currentExecutionScope.logical.runId,
11089
+ callerScope: this.currentExecutionScope.receipt.namespace,
11090
+ key: normalizedKey,
11091
+ row: rowScope
11092
+ ? {
11093
+ tableNamespace: rowScope.tableNamespace ?? null,
11094
+ rowKey: rowScope.rowKey ?? null,
11095
+ rowId: rowScope.rowKey ? null : rowScope.rowId,
11096
+ fieldName: rowScope.fieldName ?? null,
11097
+ }
11098
+ : null,
11099
+ }),
11100
+ );
11101
+ const launched = await this.#options.launchPlayAsync({
11102
+ key: normalizedKey,
11103
+ invocationKey,
11104
+ playRef: resolvedName,
11105
+ input,
11106
+ description: options.description,
11107
+ schedulerSchema: this.#options.runtimeSchedulerSchema ?? null,
11108
+ ancestryPlayIds,
11109
+ });
11110
+ const step = {
11111
+ type: 'play_launch' as const,
11112
+ playId: resolvedName,
11113
+ runId: launched.runId,
11114
+ description: normalizeStepDescription(options?.description),
11115
+ };
11116
+ if (this.activeDatasetStep) {
11117
+ this.activeDatasetStep.substeps.push(step);
11118
+ } else {
11119
+ this.steps.push(step);
11120
+ }
11121
+ this.log(
11122
+ `ctx.runPlayAsync(${normalizedKey}): admitted ${resolvedName} as ${launched.runId}`,
11123
+ );
11124
+ return launched;
11125
+ }
11126
+
10316
11127
  /**
10317
11128
  * Replay-stable receipt namespace for primitives executed by one inline
10318
11129
  * `ctx.runPlay` invocation. Format
@@ -10577,7 +11388,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10577
11388
  `sleep-${this.sleepBoundaryIndex}-${delayMs}`,
10578
11389
  );
10579
11390
  this.sleepBoundaryIndex += 1;
10580
- const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
11391
+ const existing = assertKnownResolvedBoundary(
11392
+ boundaryId,
11393
+ this.checkpoint.resolvedBoundaries?.[boundaryId],
11394
+ );
10581
11395
  if (existing?.kind === 'sleep' && existing.completedAt !== undefined) {
10582
11396
  return;
10583
11397
  }
@@ -10588,8 +11402,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10588
11402
  delayMs,
10589
11403
  },
10590
11404
  };
10591
- this.#options.onBatchComplete?.(this.checkpoint);
10592
- throw new PlayExecutionSuspendedError({
11405
+ this.suspendDurably({
10593
11406
  kind: 'sleep',
10594
11407
  boundaryId,
10595
11408
  delayMs,
@@ -10759,7 +11572,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10759
11572
  )}`,
10760
11573
  );
10761
11574
 
10762
- const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
11575
+ const existing = assertKnownResolvedBoundary(
11576
+ boundaryId,
11577
+ this.checkpoint.resolvedBoundaries?.[boundaryId],
11578
+ );
10763
11579
  // A transient request deliberately has no durable recovery path, so
10764
11580
  // its response stays in the running Play rather than a checkpoint.
10765
11581
  if (
@@ -10951,6 +11767,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10951
11767
  options?.transient !== true &&
10952
11768
  !this.durableDirectToolResultsBackedByReceipts
10953
11769
  ) {
11770
+ // Publishing a fetch boundary makes the result replay-visible.
11771
+ // A caught durable suspension is still the invocation's outcome,
11772
+ // so never checkpoint over a pending handoff.
11773
+ this.assertNoPendingDurableSuspension();
10954
11774
  this.checkpoint.resolvedBoundaries = {
10955
11775
  ...(this.checkpoint.resolvedBoundaries ?? {}),
10956
11776
  [boundaryId]: {
@@ -10998,7 +11818,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
10998
11818
  `step-${scope}:${normalizedKey}${boundarySuffix}`,
10999
11819
  );
11000
11820
  const executeStep = async (): Promise<T> => {
11001
- const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
11821
+ const existing = assertKnownResolvedBoundary(
11822
+ boundaryId,
11823
+ this.checkpoint.resolvedBoundaries?.[boundaryId],
11824
+ );
11002
11825
  if (existing?.kind === 'step' && 'output' in existing) {
11003
11826
  this.runtimeLog(
11004
11827
  `ctx.step(${normalizedKey}): recovered result from checkpoint`,
@@ -11008,6 +11831,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11008
11831
 
11009
11832
  const output = await run();
11010
11833
  assertJsonSerializableStepOutput(normalizedKey, output);
11834
+ // A caught durable suspension is still the invocation's outcome. Never
11835
+ // publish an enclosing step fallback that would make a later resume skip
11836
+ // the suspended work.
11837
+ this.assertNoPendingDurableSuspension();
11011
11838
  this.checkpoint.resolvedBoundaries = {
11012
11839
  ...(this.checkpoint.resolvedBoundaries ?? {}),
11013
11840
  [boundaryId]: {
@@ -11355,6 +12182,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11355
12182
  output: serializeToolExecuteResult(receipt.result),
11356
12183
  })),
11357
12184
  );
12185
+ await this.persistSourceRunToolReceiptManifests(
12186
+ receipts.flatMap((receipt) => {
12187
+ const request = input.requestForReceipt.get(
12188
+ receipt.resultReceiptKey,
12189
+ );
12190
+ return request
12191
+ ? [
12192
+ {
12193
+ toolId: input.toolId,
12194
+ logicalCallId: request.logicalCallId,
12195
+ executionAuthScopeDigest: request.executionAuthScopeDigest,
12196
+ resultReceiptKey: receipt.resultReceiptKey,
12197
+ },
12198
+ ]
12199
+ : [];
12200
+ }),
12201
+ );
11358
12202
  return await recordsFor(
11359
12203
  receipts.map((receipt) => receipt.resultReceiptKey),
11360
12204
  completed,
@@ -11566,19 +12410,38 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11566
12410
  }
11567
12411
  let releaseToolSlot: () => void = () => undefined;
11568
12412
  try {
12413
+ const providerOperationKey = this.providerIdempotencyKeyForToolCall(
12414
+ {
12415
+ operationKey:
12416
+ request.providerIdempotencyKeyBase ?? request.receiptKey!,
12417
+ force: request.force === true,
12418
+ logicalCallId: request.logicalCallId,
12419
+ },
12420
+ );
11569
12421
  const result = await this.executeMappedToolCallThroughModule({
11570
12422
  request,
11571
12423
  dispatchToolId: toolId,
11572
12424
  dispatchPayload: request.input,
11573
12425
  resultReceiptKey: request.receiptKey!,
11574
- providerOperationKey:
11575
- request.providerIdempotencyKeyBase ?? request.receiptKey!,
12426
+ providerOperationKey,
11576
12427
  resultReceipts: receipts,
11577
12428
  apiOptions: {
11578
12429
  playNodeScope: playNodeScopeForToolCallRequest(request),
11579
- durableCallReceiptKey: request.receiptKey,
12430
+ durableCallReceiptKey: this.toolTransportReceiptKey({
12431
+ toolId,
12432
+ resultReceiptKey: request.receiptKey!,
12433
+ providerOperationKey,
12434
+ force: request.force,
12435
+ }),
11580
12436
  completedResultCache: true,
11581
12437
  executionAuthScopeDigest: request.executionAuthScopeDigest,
12438
+ ...this.providerForceRefreshOptionsForToolCall({
12439
+ operationKey:
12440
+ request.providerIdempotencyKeyBase ?? request.receiptKey!,
12441
+ force: request.force,
12442
+ requiresExecutionFence: request.requiresExecutionFence,
12443
+ logicalCallId: request.logicalCallId,
12444
+ }),
11582
12445
  timeoutMs:
11583
12446
  typeof request.timeoutMs === 'number' && request.timeoutMs > 0
11584
12447
  ? request.timeoutMs
@@ -11618,7 +12481,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11618
12481
  if (requests.length === 0) return [];
11619
12482
 
11620
12483
  const requestForReceipt = new Map<string, ToolCallRequest>();
11621
- const requestByCallId = new Map<string, ToolCallRequest>();
11622
12484
  for (const request of requests) {
11623
12485
  const receiptKey = request.receiptKey?.trim();
11624
12486
  if (!receiptKey) {
@@ -11627,13 +12489,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11627
12489
  );
11628
12490
  }
11629
12491
  requestForReceipt.set(receiptKey, request);
11630
- requestByCallId.set(request.callId, request);
11631
12492
  }
11632
12493
  const store = this.createRuntimeStepToolCallReceiptCohortStore({
11633
12494
  toolId,
11634
12495
  requestForReceipt,
11635
12496
  });
11636
- const compiledBatches = compileRequestsWithStrategy({
12497
+ const plannedBatches = compileRequestsWithStrategy({
11637
12498
  requests,
11638
12499
  strategy,
11639
12500
  getPayload: (request: ToolCallRequest) => request.input,
@@ -11656,9 +12517,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11656
12517
  const batchParallelismCeiling =
11657
12518
  this.governor.policy.pacing.workerToolBatchDefaultParallelism;
11658
12519
  const batchSize =
11659
- compiledBatches.length > 0
12520
+ plannedBatches.length > 0
11660
12521
  ? await this.resourceGovernor.suggestedToolParallelism(
11661
- compiledBatches[0]!.batchOperation,
12522
+ plannedBatches[0]!.batchOperation,
11662
12523
  batchParallelismCeiling,
11663
12524
  )
11664
12525
  : batchParallelismCeiling;
@@ -11697,27 +12558,36 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11697
12558
  };
11698
12559
 
11699
12560
  await executeChunkedRequests({
11700
- requests: compiledBatches,
12561
+ requests: plannedBatches,
11701
12562
  batchSize,
11702
- execute: async (batch) => {
12563
+ execute: async (plannedBatch) => {
11703
12564
  if (this.persistenceLatch.tripped) {
11704
12565
  this.persistenceLatch.preventedCallCount +=
11705
- batch.memberRequests.length;
12566
+ plannedBatch.memberRequests.length;
11706
12567
  throw new RuntimePersistenceCircuitOpenError(this.persistenceLatch);
11707
12568
  }
11708
- const batchRequestByCallId = new Map(
11709
- batch.memberRequests.map((request) => [request.callId, request]),
12569
+ const requestByCallId = new Map(
12570
+ plannedBatch.memberRequests.map((request) => [
12571
+ request.callId,
12572
+ request,
12573
+ ]),
11710
12574
  );
11711
- const dispatchDetails = new Map<
12575
+ type NativeBatchPhysicalDispatchDetails = {
12576
+ members: ToolCallRequest[];
12577
+ compiled: CompiledRequestBatch<ToolCallRequest>;
12578
+ providerOperationKey: string;
12579
+ forceRefreshIdentities: string[];
12580
+ receiptLeaseExpiresAt: string | null;
12581
+ timeoutMs: number | undefined;
12582
+ };
12583
+ const nativeBatchPhysicalDispatchByAggregateReceipt = new Map<
11712
12584
  string,
11713
- {
11714
- members: ToolCallRequest[];
11715
- compiled: CompiledRequestBatch<ToolCallRequest>;
11716
- providerOperationKey: string;
11717
- receiptLeaseExpiresAt: string | null;
11718
- timeoutMs: number | undefined;
11719
- }
12585
+ NativeBatchPhysicalDispatchDetails
11720
12586
  >();
12587
+ const aggregateReceiptKeyByOwnedMemberIds = new Map<string, string>();
12588
+ const ownedMemberIdsKey = (
12589
+ members: ReadonlyArray<{ memberId: string }>,
12590
+ ): string => JSON.stringify(members.map((member) => member.memberId));
11721
12591
  const physicalToolCall = {
11722
12592
  call: async (
11723
12593
  physical: Parameters<
@@ -11730,7 +12600,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11730
12600
  'Native batch physical Tool Call requires compatibility identities.',
11731
12601
  );
11732
12602
  }
11733
- const detail = dispatchDetails.get(compatibility.resultReceiptKey);
12603
+ const detail = nativeBatchPhysicalDispatchByAggregateReceipt.get(
12604
+ compatibility.resultReceiptKey,
12605
+ );
11734
12606
  if (!detail) {
11735
12607
  throw new Error(
11736
12608
  'Native batch physical Tool Call lost its dispatch details.',
@@ -11752,7 +12624,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11752
12624
  call.tool.id,
11753
12625
  call.payload,
11754
12626
  {
11755
- durableCallReceiptKey: compatibility.resultReceiptKey,
12627
+ durableCallReceiptKey: this.toolTransportReceiptKey({
12628
+ toolId: call.tool.id,
12629
+ resultReceiptKey: compatibility.resultReceiptKey,
12630
+ providerOperationKey: detail.providerOperationKey,
12631
+ force: detail.members.some(
12632
+ (request) => request.force === true,
12633
+ ),
12634
+ }),
11756
12635
  completedResultCache: true,
11757
12636
  playNodeScope: playNodeScopeForToolCallRequest(
11758
12637
  detail.members[0]!,
@@ -11760,6 +12639,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11760
12639
  ),
11761
12640
  executionAuthScopeDigest:
11762
12641
  detail.members[0]?.executionAuthScopeDigest ?? null,
12642
+ ...(detail.forceRefreshIdentities.length > 0
12643
+ ? {
12644
+ providerOperationForceRefreshIdentities:
12645
+ detail.forceRefreshIdentities,
12646
+ }
12647
+ : {}),
11763
12648
  receiptLeaseExpiresAt: detail.receiptLeaseExpiresAt,
11764
12649
  timeoutMs: detail.timeoutMs,
11765
12650
  },
@@ -11793,7 +12678,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11793
12678
  runId: this.currentReceiptOwnerRunId,
11794
12679
  attempt: this.currentRunAttempt,
11795
12680
  },
11796
- members: batch.memberRequests.map((request) => ({
12681
+ members: plannedBatch.memberRequests.map((request) => ({
11797
12682
  memberId: request.callId,
11798
12683
  forceFailedRefresh: request.forceFailedRefresh === true,
11799
12684
  waitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
@@ -11831,53 +12716,92 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11831
12716
  },
11832
12717
  },
11833
12718
  })),
11834
- buildPhysicalCall: ({ members }) => {
11835
- const owned = members.map(
11836
- (member) => batchRequestByCallId.get(member.memberId)!,
12719
+ buildPhysicalCall: async ({ members: ownedMembers }) => {
12720
+ // `ownedMembers` are the rows we still need to execute.
12721
+ const ownedMemberRequests = ownedMembers.map(
12722
+ (member) => requestByCallId.get(member.memberId)!,
12723
+ );
12724
+ const compiled = compileOwnedPhysicalBatch(ownedMemberRequests);
12725
+ const receiptKeys = ownedMemberRequests.map(
12726
+ (request) => request.receiptKey!,
11837
12727
  );
11838
- const receiptKeys = owned.map((request) => request.receiptKey!);
11839
12728
  const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
11840
12729
  receiptKeys,
11841
12730
  prefix: 'batch',
11842
12731
  aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
11843
12732
  orgId: this.#options.orgId,
11844
- toolId: batch.batchOperation,
12733
+ toolId: compiled.batchOperation,
11845
12734
  }),
11846
12735
  });
11847
- const compiled = compileOwnedPhysicalBatch(owned);
11848
- if (compiled.batchOperation !== batch.batchOperation) {
11849
- throw new Error(
11850
- 'Native batch owned members changed provider operation during compilation.',
11851
- );
11852
- }
11853
- const providerOperationKey = aggregateReceiptKey;
11854
- dispatchDetails.set(aggregateReceiptKey, {
11855
- members: owned,
11856
- compiled,
11857
- providerOperationKey,
11858
- receiptLeaseExpiresAt: owned.reduce<string | null>(
11859
- (earliest, request) => {
12736
+ const providerActionVersion =
12737
+ (
12738
+ await this.#options.getToolActionCacheVersion?.(
12739
+ compiled.batchOperation,
12740
+ )
12741
+ )?.trim() ?? '';
12742
+ const forceRefreshIdentities = ownedMemberRequests.flatMap(
12743
+ (request) =>
12744
+ this.providerForceRefreshIdentitiesForToolCall({
12745
+ operationKey:
12746
+ this.providerOperationKeyForQueuedToolCall(request),
12747
+ force: request.force,
12748
+ requiresExecutionFence: request.requiresExecutionFence,
12749
+ logicalCallId: request.logicalCallId,
12750
+ }),
12751
+ );
12752
+ const providerOperationKey =
12753
+ buildDurableToolPayloadProviderIdempotencyKey({
12754
+ orgId: this.#options.orgId,
12755
+ toolId: compiled.batchOperation,
12756
+ executionAuthScopeDigest:
12757
+ ownedMemberRequests[0]?.executionAuthScopeDigest ?? null,
12758
+ providerActionVersion,
12759
+ payload: compiled.batchPayload,
12760
+ forceRefreshIdentities,
12761
+ });
12762
+ const aggregateDispatchDetails: NativeBatchPhysicalDispatchDetails =
12763
+ {
12764
+ members: ownedMemberRequests,
12765
+ compiled,
12766
+ providerOperationKey,
12767
+ forceRefreshIdentities,
12768
+ receiptLeaseExpiresAt: ownedMemberRequests.reduce<
12769
+ string | null
12770
+ >((earliest, request) => {
11860
12771
  const expiresAt = request.receiptLeaseExpiresAt ?? null;
11861
12772
  if (!expiresAt) return earliest;
11862
12773
  if (!earliest) return expiresAt;
11863
12774
  return Date.parse(expiresAt) < Date.parse(earliest)
11864
12775
  ? expiresAt
11865
12776
  : earliest;
11866
- },
11867
- null,
11868
- ),
11869
- timeoutMs: owned.every(
11870
- (request) =>
11871
- typeof request.timeoutMs === 'number' &&
11872
- Number.isFinite(request.timeoutMs) &&
11873
- request.timeoutMs > 0,
11874
- )
11875
- ? Math.max(...owned.map((request) => request.timeoutMs!))
11876
- : undefined,
11877
- });
12777
+ }, null),
12778
+ timeoutMs: ownedMemberRequests.every(
12779
+ (request) =>
12780
+ typeof request.timeoutMs === 'number' &&
12781
+ Number.isFinite(request.timeoutMs) &&
12782
+ request.timeoutMs > 0,
12783
+ )
12784
+ ? Math.max(
12785
+ ...ownedMemberRequests.map(
12786
+ (request) => request.timeoutMs!,
12787
+ ),
12788
+ )
12789
+ : undefined,
12790
+ };
12791
+ nativeBatchPhysicalDispatchByAggregateReceipt.set(
12792
+ aggregateReceiptKey,
12793
+ aggregateDispatchDetails,
12794
+ );
12795
+ aggregateReceiptKeyByOwnedMemberIds.set(
12796
+ ownedMemberIdsKey(ownedMembers),
12797
+ aggregateReceiptKey,
12798
+ );
11878
12799
  return {
11879
- ...members[0]!.call,
11880
- tool: { id: batch.batchOperation, providerActionVersion: '' },
12800
+ ...ownedMembers[0]!.call,
12801
+ tool: {
12802
+ id: compiled.batchOperation,
12803
+ providerActionVersion,
12804
+ },
11881
12805
  payload: compiled.batchPayload,
11882
12806
  compatibility: {
11883
12807
  resultReceiptKey: aggregateReceiptKey,
@@ -11885,34 +12809,29 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11885
12809
  },
11886
12810
  };
11887
12811
  },
11888
- splitResult: async ({ result: aggregate, members }) => {
11889
- const receiptKeys = members.map((member) => {
11890
- const request = batchRequestByCallId.get(member.memberId);
11891
- if (!request?.receiptKey) {
11892
- throw new Error(
11893
- `Native batch Tool Call lost receipt for ${member.memberId}.`,
11894
- );
11895
- }
11896
- return request.receiptKey;
11897
- });
11898
- const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
11899
- receiptKeys,
11900
- prefix: 'batch',
11901
- aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
11902
- orgId: this.#options.orgId,
11903
- toolId: batch.batchOperation,
11904
- }),
11905
- });
11906
- const detail = dispatchDetails.get(aggregateReceiptKey);
12812
+ splitResult: async ({ result: aggregate, members: ownedMembers }) => {
12813
+ const aggregateReceiptKey = aggregateReceiptKeyByOwnedMemberIds.get(
12814
+ ownedMemberIdsKey(ownedMembers),
12815
+ );
12816
+ if (!aggregateReceiptKey) {
12817
+ throw new Error(
12818
+ 'Native batch Tool Call lost its physical aggregate receipt identity.',
12819
+ );
12820
+ }
12821
+ const detail =
12822
+ nativeBatchPhysicalDispatchByAggregateReceipt.get(
12823
+ aggregateReceiptKey,
12824
+ );
11907
12825
  if (!detail) {
11908
12826
  throw new Error(
11909
12827
  'Native batch Tool Call lost its split compilation details.',
11910
12828
  );
11911
12829
  }
11912
12830
  if (
11913
- detail.members.length !== members.length ||
12831
+ detail.members.length !== ownedMembers.length ||
11914
12832
  !detail.members.every(
11915
- (request, index) => request.callId === members[index]?.memberId,
12833
+ (request, index) =>
12834
+ request.callId === ownedMembers[index]?.memberId,
11916
12835
  )
11917
12836
  ) {
11918
12837
  throw new Error(
@@ -11923,8 +12842,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11923
12842
  legacyResultForBatchSplitter(aggregate),
11924
12843
  );
11925
12844
  return await Promise.all(
11926
- members.map(async (member, index) => {
11927
- const request = batchRequestByCallId.get(member.memberId);
12845
+ ownedMembers.map(async (member, index) => {
12846
+ const request = requestByCallId.get(member.memberId);
11928
12847
  if (!request) {
11929
12848
  throw new Error(
11930
12849
  `Native batch Tool Call lost member ${member.memberId}.`,
@@ -11954,7 +12873,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11954
12873
  );
11955
12874
  },
11956
12875
  });
11957
- return { batch, result };
12876
+ return { batch: plannedBatch, result };
11958
12877
  },
11959
12878
  onChunkComplete: async (entries) => {
11960
12879
  for (const entry of entries) {
@@ -11964,6 +12883,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
11964
12883
  for (const request of entry.request.memberRequests) {
11965
12884
  await reject(request, entry.error);
11966
12885
  }
12886
+ // A provider-unavailable physical batch is a logical outcome for
12887
+ // each member, not a map-fatal scheduler failure. Each rejected
12888
+ // row resolver now receives the typed error above, where a
12889
+ // waterfall may persist the failed leg and advance to its next
12890
+ // provider. Keep infrastructure and contract failures loud.
12891
+ if (
12892
+ isProviderExhaustedFailure(entry.error) ||
12893
+ isProviderUnavailable(entry.error)
12894
+ ) {
12895
+ continue;
12896
+ }
11967
12897
  throw entry.error;
11968
12898
  }
11969
12899
  if (!entry.result) {
@@ -12618,6 +13548,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
12618
13548
  force: owner.force === true,
12619
13549
  logicalCallId: owner.logicalCallId,
12620
13550
  }),
13551
+ ...this.providerForceRefreshOptionsForToolCall({
13552
+ operationKey:
13553
+ this.providerOperationKeyForQueuedToolCall(owner),
13554
+ force: owner.force,
13555
+ requiresExecutionFence: owner.requiresExecutionFence,
13556
+ logicalCallId: owner.logicalCallId,
13557
+ }),
12621
13558
  timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
12622
13559
  owner,
12623
13560
  ]),
@@ -13232,6 +14169,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13232
14169
  force: request.force === true,
13233
14170
  logicalCallId: request.logicalCallId,
13234
14171
  }),
14172
+ ...this.providerForceRefreshOptionsForToolCall({
14173
+ operationKey:
14174
+ this.providerOperationKeyForQueuedToolCall(
14175
+ request,
14176
+ ),
14177
+ force: request.force,
14178
+ requiresExecutionFence:
14179
+ request.requiresExecutionFence,
14180
+ logicalCallId: request.logicalCallId,
14181
+ }),
13235
14182
  receiptLeaseExpiresAt:
13236
14183
  request.receiptLeaseExpiresAt,
13237
14184
  heartbeatReceipt: () =>
@@ -13440,9 +14387,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13440
14387
  const asyncContract = options?.skipAsyncOperationResolution
13441
14388
  ? null
13442
14389
  : ((await this.#options.getToolAsyncOperation?.(toolId)) ?? null);
14390
+ const asyncOperationPoll = options?.skipAsyncOperationResolution
14391
+ ? null
14392
+ : ((await this.#options.getToolAsyncOperationPoll?.(toolId)) ?? null);
13443
14393
  const shouldWaitForAsyncOperation = Boolean(
13444
14394
  asyncContract && asyncOperationShouldWait(asyncContract, input),
13445
14395
  );
14396
+ // An explicitly opted-in caller-managed launch gives the Play author the
14397
+ // provider handle and leaves polling to the authored graph. Do not let a
14398
+ // provider's legacy synchronous helper turn a wide start wave into
14399
+ // long-lived HTTP calls before that graph can make its own poll.
14400
+ const shouldPreferAsyncProviderExecution =
14401
+ asyncContract?.wait.kind === 'caller_managed' &&
14402
+ asyncContract.lifecycle.startAction === toolId &&
14403
+ asyncContract.launch?.preferAsync === true;
13446
14404
  const executionAuthScopeDigest = options?.executionAuthScopeDigest
13447
14405
  ? options.executionAuthScopeDigest.trim() || null
13448
14406
  : ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null);
@@ -13635,13 +14593,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13635
14593
  `Tool ${toolId} requires a provider execution fence but no provider operation identity was supplied.`,
13636
14594
  );
13637
14595
  }
13638
- const canonicalOperationActionVersion = providerExecutionFenceKey
13639
- ? (
13640
- await this.#options.getToolActionCacheVersion?.(toolId)
13641
- )?.trim() || null
13642
- : null;
14596
+ const canonicalOperationActionVersion =
14597
+ providerExecutionFenceKey ||
14598
+ options?.providerOperationForceRefreshIdentities?.length
14599
+ ? (
14600
+ await this.#options.getToolActionCacheVersion?.(toolId)
14601
+ )?.trim() || null
14602
+ : null;
13643
14603
  const canonicalOperation =
13644
- providerExecutionFenceKey && canonicalOperationActionVersion
14604
+ (providerExecutionFenceKey ||
14605
+ options?.providerOperationForceRefreshIdentities?.length) &&
14606
+ canonicalOperationActionVersion
13645
14607
  ? {
13646
14608
  version: 1,
13647
14609
  tool_id: toolId,
@@ -13672,6 +14634,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13672
14634
  ...(shouldWaitForAsyncOperation
13673
14635
  ? { async_operation_wait: true }
13674
14636
  : {}),
14637
+ ...(shouldPreferAsyncProviderExecution
14638
+ ? { prefer_async: true }
14639
+ : {}),
14640
+ ...(asyncOperationPoll
14641
+ ? {
14642
+ async_operation_poll_timeout_ms:
14643
+ asyncOperationPoll.checkTimeoutMs,
14644
+ }
14645
+ : {}),
13675
14646
  ...(usesDurableInvocationFence
13676
14647
  ? {
13677
14648
  invocation_fence_version:
@@ -14242,14 +15213,36 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
14242
15213
  toolId,
14243
15214
  failure.backpressureDelayMs,
14244
15215
  );
15216
+ // A Retry-After beyond the durable pacer's maximum wait has
15217
+ // no useful in-process retry. It is already an exhausted
15218
+ // provider outcome: surface the same typed error the next
15219
+ // provider admission would produce, without sleeping before a
15220
+ // waterfall can choose an independent fallback.
15221
+ if (
15222
+ failure.backpressureDelayMs > PROVIDER_EXHAUSTED_MAX_WAIT_MS
15223
+ ) {
15224
+ throw new ProviderExhaustedError({
15225
+ provider,
15226
+ retryAtMs: Date.now() + failure.backpressureDelayMs,
15227
+ cause: failure.error,
15228
+ });
15229
+ }
14245
15230
  }
14246
15231
  if (
14247
15232
  failure.shouldRetry &&
14248
- (ambiguousTransportReplayAllowed ||
15233
+ (failure.knownNotDispatched ||
15234
+ failure.reason === 'gateway_invocation_in_progress' ||
15235
+ ambiguousTransportReplayAllowed ||
14249
15236
  retrySafeTransientHttp ||
14250
15237
  fixtureReplaySafe)
14251
15238
  ) {
14252
- invocationAttempt += 1;
15239
+ // A gateway follower is retrying transport delivery to the
15240
+ // same durable invocation. Advancing the logical attempt here
15241
+ // would mint a new invocation key while the original owner is
15242
+ // still running, defeating the gateway's replay fence.
15243
+ if (failure.reason !== 'gateway_invocation_in_progress') {
15244
+ invocationAttempt += 1;
15245
+ }
14253
15246
  if (failure.chargeRetryBudget) {
14254
15247
  await this.governor.chargeBudget('retry');
14255
15248
  }