deepline 0.3.88 → 0.3.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/contracts/install-commands.json +29 -0
- package/dist/bundling-sources/sdk/src/client.ts +652 -39
- package/dist/bundling-sources/sdk/src/config.ts +77 -0
- package/dist/bundling-sources/sdk/src/http.ts +76 -30
- package/dist/bundling-sources/sdk/src/index.ts +10 -0
- package/dist/bundling-sources/sdk/src/monitor-fleet-contract.ts +1 -1
- package/dist/bundling-sources/sdk/src/pagination.ts +25 -0
- package/dist/bundling-sources/sdk/src/play.ts +15 -1
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +3 -3
- package/dist/bundling-sources/sdk/src/release.ts +16 -9
- package/dist/bundling-sources/sdk/src/types.ts +106 -4
- package/dist/bundling-sources/shared_libs/integration-contracts/AGENTS.md +9 -0
- package/dist/bundling-sources/shared_libs/integration-contracts/README.md +3 -0
- package/dist/bundling-sources/shared_libs/integration-contracts/index.ts +2 -0
- package/dist/bundling-sources/shared_libs/monitors/analytics-install-guides.ts +77 -0
- package/dist/bundling-sources/shared_libs/monitors/analytics-public.ts +10 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/charges.ts +125 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/policy-builders.ts +686 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/pricing.ts +54 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/public-pricing.ts +72 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/session.ts +45 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/state.ts +454 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/tamradar.ts +260 -0
- package/dist/bundling-sources/shared_libs/monitors/billing/types.ts +215 -0
- package/dist/bundling-sources/shared_libs/monitors/callback-security.ts +333 -0
- package/dist/bundling-sources/shared_libs/monitors/cursor-page.ts +42 -0
- package/dist/bundling-sources/shared_libs/monitors/deactivation-reasons.ts +16 -0
- package/dist/bundling-sources/shared_libs/monitors/framework-managed-payload-fields.ts +71 -0
- package/dist/bundling-sources/shared_libs/monitors/generated/provider-monitor-callback-admission-targets.ts +16 -0
- package/dist/bundling-sources/shared_libs/monitors/http-error-response.ts +270 -0
- package/dist/bundling-sources/shared_libs/monitors/inbound-publish-key.ts +39 -0
- package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +82 -0
- package/dist/bundling-sources/shared_libs/monitors/inventory-reconciliation.ts +59 -0
- package/dist/bundling-sources/shared_libs/monitors/lifecycle-notifications.ts +471 -0
- package/dist/bundling-sources/shared_libs/monitors/org-monitor-limits.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/package.json +36 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-callback-verification.ts +1125 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +305 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-capability.ts +964 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-consumer-health.ts +56 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-errors.ts +131 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-identity-claims.ts +535 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-legacy-aliases.ts +16 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-priority-execution.ts +56 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-public-aliases.ts +28 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-runtime-read-eligibility.ts +38 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-state.ts +122 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-status.ts +10 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-upstream-identity.ts +112 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-webhook-monitor-runtime.ts +300 -0
- package/dist/bundling-sources/shared_libs/monitors/quickstart-prompts.ts +22 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts +245 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-message.ts +38 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-ack-probe.ts +94 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-delivery.ts +60 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue.ts +60 -0
- package/dist/bundling-sources/shared_libs/monitors/snitcher-install-validation.ts +93 -0
- package/dist/bundling-sources/shared_libs/monitors/snitcher-ip2company.ts +332 -0
- package/dist/bundling-sources/shared_libs/monitors/snitcher-radar.ts +221 -0
- package/dist/bundling-sources/shared_libs/monitors/starter-templates.ts +105 -0
- package/dist/bundling-sources/shared_libs/monitors/tracking-provider-guidance.ts +73 -0
- package/dist/bundling-sources/shared_libs/monitors/tracking-provider-integrations.ts +81 -0
- package/dist/bundling-sources/shared_libs/monitors/tsconfig.json +10 -0
- package/dist/bundling-sources/shared_libs/observability/build-context.ts +13 -0
- package/dist/bundling-sources/shared_libs/observability/log-metadata.ts +160 -0
- package/dist/bundling-sources/shared_libs/observability/request-context.ts +68 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +654 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +35 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/neon-constants.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +272 -0
- package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts +6 -11
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +135 -12
- package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/async-play-invocation.ts +77 -0
- package/dist/bundling-sources/shared_libs/play-runtime/bounceban-batching.ts +137 -0
- package/dist/bundling-sources/shared_libs/play-runtime/checkpoint-boundaries.ts +70 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1159 -166
- package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +1 -32
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-payload-contract.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +725 -0
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +20 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +442 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-protocol.ts +7 -1
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +66 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/ports.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/postgres-store.ts +170 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +1322 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/lane-retirement-gate.ts +189 -0
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +112 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/perf-trace.ts +127 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-artifact-runtime-contract.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +24 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/job.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/ports.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/contracts.ts +114 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/engine-suspension.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/index.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/job.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/ports.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/postgres-fence-adapter.ts +263 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/contracts.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/index.ts +15 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/job.ts +65 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/ports.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/index.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +142 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/ports.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/postgres-adapter.ts +102 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +60 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/index.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +5 -8
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +23 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +168 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +90 -13
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +6 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-start-admission.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +14 -43
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +67 -213
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts +201 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-env.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +67 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-release-environment.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-contract.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +38 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +37 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-run-task.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts +269 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-suffix.ts +73 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +6642 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +897 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-metrics.ts +120 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +76 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +476 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +147 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/index.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/lazy.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/play-run-recovery.ts +98 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts +260 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +1416 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +1255 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +942 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +14636 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +888 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +355 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +46 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runner-push.ts +60 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-api-origin.ts +33 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-gateway-health.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +835 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts +1218 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-operation.ts +207 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-scheduler-schema.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts +123 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-launch-read-observability.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +169 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/terminal-contract.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-queue-metrics.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +249 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +296 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +20 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +60 -15
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +14 -1
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +165 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +15 -23
- package/dist/bundling-sources/shared_libs/play-runtime/workflow-executor-auth.ts +169 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +122 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +8 -3
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +454 -17
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +19 -4
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +8 -14
- package/dist/bundling-sources/shared_libs/plays/core.ts +11 -6
- package/dist/bundling-sources/shared_libs/plays/deployment-status.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/docflow-binding-owner.ts +16 -11
- package/dist/bundling-sources/shared_libs/plays/docflow.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/email-status.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/play-grep-search.ts +39 -0
- package/dist/bundling-sources/shared_libs/plays/resolve-static-pipeline.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/share/copy.ts +271 -0
- package/dist/bundling-sources/shared_libs/plays/share/metrics.ts +140 -0
- package/dist/bundling-sources/shared_libs/plays/share/public-read-model.ts +86 -0
- package/dist/bundling-sources/shared_libs/plays/share/seo-quality.ts +181 -0
- package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts +658 -0
- package/dist/bundling-sources/shared_libs/plays/share/types.ts +197 -0
- package/dist/bundling-sources/shared_libs/plays/slack-hitl-timeout-error.ts +37 -0
- package/dist/bundling-sources/shared_libs/plays/slug.ts +115 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +50 -15
- package/dist/bundling-sources/shared_libs/plays/tool-category-descriptions.ts +2 -2
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +877 -0
- package/dist/bundling-sources/shared_libs/product-notifications/configuration-draft.ts +456 -0
- package/dist/bundling-sources/shared_libs/product-notifications/configuration.ts +566 -0
- package/dist/bundling-sources/shared_libs/product-notifications/content.ts +28 -0
- package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +60 -0
- package/dist/bundling-sources/shared_libs/product-notifications/email.ts +246 -0
- package/dist/bundling-sources/shared_libs/product-notifications/evaluation.ts +280 -0
- package/dist/bundling-sources/shared_libs/product-notifications/events.ts +433 -0
- package/dist/bundling-sources/shared_libs/product-notifications/legacy-slack.ts +80 -0
- package/dist/bundling-sources/shared_libs/product-notifications/overflow.ts +18 -0
- package/dist/bundling-sources/shared_libs/product-notifications/rolling-limit.ts +24 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-delivery.ts +624 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-payload.ts +140 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-transport.ts +75 -0
- package/dist/bundling-sources/shared_libs/product-notifications/workspace.ts +38 -0
- package/dist/bundling-sources/shared_libs/runtime-env.ts +12 -0
- package/dist/bundling-sources/shared_libs/security/email-blacklist.ts +42 -0
- package/dist/bundling-sources/shared_libs/security/header-safe-env.ts +35 -0
- package/dist/bundling-sources/shared_libs/security/internal-actor.ts +50 -0
- package/dist/bundling-sources/shared_libs/security/secret-redaction.ts +267 -0
- package/dist/bundling-sources/shared_libs/security/slack-sanitize.ts +133 -0
- package/dist/cli/getting-started.d.mts +52 -0
- package/dist/cli/getting-started.d.ts +52 -0
- package/dist/cli/getting-started.js +65 -0
- package/dist/cli/getting-started.mjs +36 -0
- package/dist/cli/index.d.mts +9 -0
- package/dist/cli/index.d.ts +9 -0
- package/dist/cli/index.js +1611 -453
- package/dist/cli/index.mjs +1596 -447
- package/dist/{compiler-manifest-Dbov16F7.d.mts → compiler-manifest-BaJfE-5H.d.mts} +136 -10
- package/dist/{compiler-manifest-Dbov16F7.d.ts → compiler-manifest-BaJfE-5H.d.ts} +136 -10
- package/dist/index.d.mts +910 -155
- package/dist/index.d.ts +910 -155
- package/dist/index.js +550 -64
- package/dist/index.mjs +550 -64
- package/dist/install-integrity.json +193 -4
- package/dist/monitor-fleet-contract.d.mts +138 -0
- package/dist/monitor-fleet-contract.d.ts +138 -0
- package/dist/monitor-fleet-contract.js +404 -0
- package/dist/monitor-fleet-contract.mjs +367 -0
- package/dist/plays/bundle-play-file.d.mts +8 -3
- package/dist/plays/bundle-play-file.d.ts +8 -3
- package/dist/plays/bundle-play-file.mjs +100 -20
- package/dist/release.d.mts +255 -0
- package/dist/release.d.ts +255 -0
- package/dist/release.js +215 -0
- package/dist/release.mjs +188 -0
- package/package.json +23 -4
- /package/dist/bundling-sources/shared_libs/{integrations/bettercontact-execution-policy.ts → integration-contracts/bettercontact/execution-policy.ts} +0 -0
- /package/dist/bundling-sources/shared_libs/{integrations/theirstack-execution-policy.ts → integration-contracts/theirstack/execution-policy.ts} +0 -0
|
@@ -109,7 +109,11 @@ import {
|
|
|
109
109
|
type ToolResponseContract,
|
|
110
110
|
type ToolResponseView,
|
|
111
111
|
} from '../plays/tool-response-contract';
|
|
112
|
-
import {
|
|
112
|
+
import {
|
|
113
|
+
isProviderExhaustedFailure,
|
|
114
|
+
ProviderExhaustedError,
|
|
115
|
+
} from './run-failure';
|
|
116
|
+
import { PROVIDER_EXHAUSTED_MAX_WAIT_MS } from './governor/app-runtime-rate-state-backend';
|
|
113
117
|
import {
|
|
114
118
|
buildPlayContractCompatibility,
|
|
115
119
|
normalizePlayContractCompatibility,
|
|
@@ -122,6 +126,10 @@ import {
|
|
|
122
126
|
type ToolExecutionErrorSchemaVersion,
|
|
123
127
|
type ToolExecutionFailureV1,
|
|
124
128
|
} from '../plays/tool-execution-error';
|
|
129
|
+
import {
|
|
130
|
+
SLACK_HITL_TIMEOUT_ERROR_CODE,
|
|
131
|
+
SlackHitlTimeoutError,
|
|
132
|
+
} from '../plays/slack-hitl-timeout-error';
|
|
125
133
|
import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
|
|
126
134
|
import { pacingPolicyForTool } from './pacing';
|
|
127
135
|
import {
|
|
@@ -177,6 +185,7 @@ import {
|
|
|
177
185
|
buildDurableToolAggregateReceiptKey,
|
|
178
186
|
buildDurableToolCallAuthScopeDigest,
|
|
179
187
|
buildDurableToolCallCacheKey,
|
|
188
|
+
buildSourceRunToolReceiptManifestKey,
|
|
180
189
|
buildLegacyRevisionedDurableToolCallCacheKey,
|
|
181
190
|
buildDurableToolPayloadProviderIdempotencyKey,
|
|
182
191
|
buildProviderOperationIdempotencyKey,
|
|
@@ -353,7 +362,11 @@ import type {
|
|
|
353
362
|
PlayRowReadOrder,
|
|
354
363
|
PlayRowMeta,
|
|
355
364
|
PlaySheetCellProducer,
|
|
365
|
+
PlayAsyncCallOptions,
|
|
356
366
|
} from './ctx-types';
|
|
367
|
+
import { assertKnownResolvedBoundary } from './checkpoint-boundaries';
|
|
368
|
+
import type { AsyncPlayRunRef } from '../plays/authoring-contract';
|
|
369
|
+
import { admitAsyncPlayLineage } from './async-play-invocation';
|
|
357
370
|
import {
|
|
358
371
|
MAX_CELL_READ_REFS,
|
|
359
372
|
MAX_ROW_READ_CELLS,
|
|
@@ -1982,6 +1995,19 @@ function normalizeStepDescription(
|
|
|
1982
1995
|
return trimmed ? trimmed : undefined;
|
|
1983
1996
|
}
|
|
1984
1997
|
|
|
1998
|
+
function resolvePlayReferenceName(playRef: unknown): string {
|
|
1999
|
+
if (typeof playRef === 'string') return playRef;
|
|
2000
|
+
if (
|
|
2001
|
+
playRef === null ||
|
|
2002
|
+
(typeof playRef !== 'object' && typeof playRef !== 'function')
|
|
2003
|
+
) {
|
|
2004
|
+
return '';
|
|
2005
|
+
}
|
|
2006
|
+
const reference = playRef as { playName?: unknown; name?: unknown };
|
|
2007
|
+
if (typeof reference.playName === 'string') return reference.playName;
|
|
2008
|
+
return typeof reference.name === 'string' ? reference.name : '';
|
|
2009
|
+
}
|
|
2010
|
+
|
|
1985
2011
|
function emptyCheckpoint(): PlayCheckpoint {
|
|
1986
2012
|
return {
|
|
1987
2013
|
completedBatches: {},
|
|
@@ -1996,6 +2022,14 @@ function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
|
|
|
1996
2022
|
...frame,
|
|
1997
2023
|
completedRowKeys: [...frame.completedRowKeys],
|
|
1998
2024
|
pendingRowKeys: [...frame.pendingRowKeys],
|
|
2025
|
+
...(frame.suspendedInputRows
|
|
2026
|
+
? {
|
|
2027
|
+
suspendedInputRows: frame.suspendedInputRows.map((row) => ({
|
|
2028
|
+
inputIndex: row.inputIndex,
|
|
2029
|
+
data: cloneCsvAliasedRow(row.data),
|
|
2030
|
+
})),
|
|
2031
|
+
}
|
|
2032
|
+
: {}),
|
|
1999
2033
|
};
|
|
2000
2034
|
}
|
|
2001
2035
|
|
|
@@ -2089,6 +2123,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2089
2123
|
* waterfall miss.
|
|
2090
2124
|
*/
|
|
2091
2125
|
private deeplineInsufficientCreditsFailure: ToolExecutionError | null = null;
|
|
2126
|
+
/**
|
|
2127
|
+
* A Slack HITL timeout is terminal only after its card finalizer has proved
|
|
2128
|
+
* the Slack replacement. Keep that terminal result outside authored promise
|
|
2129
|
+
* control flow so catching the rejected tool call cannot settle the Play or
|
|
2130
|
+
* start a later durable operation.
|
|
2131
|
+
*/
|
|
2132
|
+
private slackHitlTimeoutFailure: ToolExecutionError | null = null;
|
|
2092
2133
|
private toolCallResolvers = new Map<
|
|
2093
2134
|
string,
|
|
2094
2135
|
{ resolve: (value: unknown) => void; reject: (reason: unknown) => void }
|
|
@@ -2103,6 +2144,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2103
2144
|
>();
|
|
2104
2145
|
private readonly ctxFetchHttpFailureDiagnosticIdentities = new Set<string>();
|
|
2105
2146
|
private checkpoint: PlayCheckpoint;
|
|
2147
|
+
/**
|
|
2148
|
+
* A durable boundary is a control-flow handoff, not a customer-catchable
|
|
2149
|
+
* business failure. Keep it on the context as the source of truth so a
|
|
2150
|
+
* broad catch cannot turn the invocation into a normal completed result.
|
|
2151
|
+
*/
|
|
2152
|
+
private pendingDurableSuspension: PlayExecutionSuspension | null = null;
|
|
2106
2153
|
private readonly durableCallCacheEpochMs: number;
|
|
2107
2154
|
/**
|
|
2108
2155
|
* Durable tool receipts are the replay/cache authority for the execution
|
|
@@ -2215,6 +2262,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2215
2262
|
timeoutMs?: number;
|
|
2216
2263
|
receiptWaitMs?: number;
|
|
2217
2264
|
}): Promise<TOutput> => {
|
|
2265
|
+
this.assertNoPendingDurableSuspension();
|
|
2218
2266
|
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
|
2219
2267
|
throw new Error(
|
|
2220
2268
|
'ctx.tools.execute requires a request object: ctx.tools.execute({ id, tool, input, description }).',
|
|
@@ -2281,6 +2329,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2281
2329
|
: undefined,
|
|
2282
2330
|
).catch((error: unknown) => {
|
|
2283
2331
|
this.recordDeeplineInsufficientCreditsFailure(error);
|
|
2332
|
+
this.recordSlackHitlTimeoutFailure(error);
|
|
2284
2333
|
throw error;
|
|
2285
2334
|
}) as Promise<TOutput>;
|
|
2286
2335
|
},
|
|
@@ -2303,6 +2352,62 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2303
2352
|
}
|
|
2304
2353
|
}
|
|
2305
2354
|
|
|
2355
|
+
private recordSlackHitlTimeoutFailure(error: unknown): void {
|
|
2356
|
+
if (
|
|
2357
|
+
error instanceof ToolExecutionError &&
|
|
2358
|
+
error.code === SLACK_HITL_TIMEOUT_ERROR_CODE
|
|
2359
|
+
) {
|
|
2360
|
+
if (!this.slackHitlTimeoutFailure) {
|
|
2361
|
+
this.slackHitlTimeoutFailure = error;
|
|
2362
|
+
this.runtimeLog(
|
|
2363
|
+
`[slack_hitl.timeout_terminal] code=${SLACK_HITL_TIMEOUT_ERROR_CODE} ` +
|
|
2364
|
+
'card_finalization=confirmed outcome=run_fatal',
|
|
2365
|
+
);
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
/**
|
|
2371
|
+
* Called by the runner before a normal result settles. This preserves a
|
|
2372
|
+
* terminal Slack timeout even if authored code catches it and returns a
|
|
2373
|
+
* value. It intentionally remains separate from provider errors, which are
|
|
2374
|
+
* ordinary customer-handled waterfall misses.
|
|
2375
|
+
*/
|
|
2376
|
+
assertNoSlackHitlTimeoutFailure(): void {
|
|
2377
|
+
if (this.slackHitlTimeoutFailure) {
|
|
2378
|
+
throw this.slackHitlTimeoutFailure;
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
/** Called by the runner after customer code returns, before a success settles. */
|
|
2383
|
+
assertNoRunFatalToolFailure(): void {
|
|
2384
|
+
this.assertNoSlackHitlTimeoutFailure();
|
|
2385
|
+
this.assertNoDeeplineInsufficientCreditsFailure();
|
|
2386
|
+
}
|
|
2387
|
+
|
|
2388
|
+
/**
|
|
2389
|
+
* Called at ctx effect boundaries and by the runner before it can settle a
|
|
2390
|
+
* normal result. This makes suspension sticky even when authored code uses a
|
|
2391
|
+
* broad catch around `await ctx.sleep(...)`.
|
|
2392
|
+
*/
|
|
2393
|
+
assertNoPendingDurableSuspension(): void {
|
|
2394
|
+
this.assertNoSlackHitlTimeoutFailure();
|
|
2395
|
+
if (this.pendingDurableSuspension) {
|
|
2396
|
+
throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
/**
|
|
2401
|
+
* The single seam that hands this invocation back to the scheduler. The
|
|
2402
|
+
* first suspension wins: a later boundary reached by code that caught the
|
|
2403
|
+
* first one cannot replace the handoff the scheduler already owes a resume.
|
|
2404
|
+
*/
|
|
2405
|
+
private suspendDurably(suspension: PlayExecutionSuspension): never {
|
|
2406
|
+
this.pendingDurableSuspension ??= suspension;
|
|
2407
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
2408
|
+
throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
|
|
2409
|
+
}
|
|
2410
|
+
|
|
2306
2411
|
async tool<TOutput = unknown>(
|
|
2307
2412
|
key: string,
|
|
2308
2413
|
toolId: string,
|
|
@@ -2416,6 +2521,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2416
2521
|
options.artifactRuntimeContract ??
|
|
2417
2522
|
resolvePlayArtifactRuntimeContract({
|
|
2418
2523
|
authoringContractEdition: options.authoringContractEdition,
|
|
2524
|
+
slackHitlTimeoutOutcome: options.slackHitlTimeoutOutcome,
|
|
2419
2525
|
toolResponseContract: options.toolResponseContract,
|
|
2420
2526
|
toolResponseReceiptRevision: options.toolResponseReceiptRevision,
|
|
2421
2527
|
});
|
|
@@ -2475,11 +2581,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2475
2581
|
options.playId ||
|
|
2476
2582
|
options.workflowId ||
|
|
2477
2583
|
'anonymous-play';
|
|
2478
|
-
const
|
|
2584
|
+
const currentRunId = options.runId ?? options.workflowId ?? 'anonymous-run';
|
|
2585
|
+
const rootRunId = options.rootRunId ?? currentRunId;
|
|
2479
2586
|
this.executionScope =
|
|
2480
2587
|
options.executionScope ??
|
|
2481
2588
|
createRootRunExecutionScope({
|
|
2482
|
-
runId:
|
|
2589
|
+
runId: currentRunId,
|
|
2483
2590
|
playId: rootPlayId,
|
|
2484
2591
|
attempt:
|
|
2485
2592
|
typeof options.runAttempt === 'number' &&
|
|
@@ -2489,7 +2596,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2489
2596
|
receiptNamespace: options.runtimeReceiptScope ?? rootPlayId,
|
|
2490
2597
|
authority: {
|
|
2491
2598
|
orgId: options.orgId ?? 'org',
|
|
2492
|
-
workflowId: options.workflowId ??
|
|
2599
|
+
workflowId: options.workflowId ?? currentRunId,
|
|
2493
2600
|
integrationMode: options.integrationMode ?? null,
|
|
2494
2601
|
capabilities: DEFAULT_RUNTIME_EXECUTION_CAPABILITIES,
|
|
2495
2602
|
},
|
|
@@ -2505,7 +2612,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2505
2612
|
adapter: 'cjs_node20',
|
|
2506
2613
|
scope: {
|
|
2507
2614
|
orgId: options.orgId ?? 'org',
|
|
2508
|
-
rootRunId
|
|
2615
|
+
rootRunId,
|
|
2509
2616
|
},
|
|
2510
2617
|
rateState: options.rateState ?? new InMemoryRateStateBackend(),
|
|
2511
2618
|
budgetState: options.budgetState,
|
|
@@ -2710,6 +2817,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2710
2817
|
private assertInlineChildContract(
|
|
2711
2818
|
reason: 'dataset_child' | 'suspending_child',
|
|
2712
2819
|
): void {
|
|
2820
|
+
this.assertNoPendingDurableSuspension();
|
|
2713
2821
|
const composition = this.activeInlineComposition;
|
|
2714
2822
|
if (!composition) return;
|
|
2715
2823
|
throw new Error(ctxRunPlayInlineOnlyMessage(composition.playName, reason));
|
|
@@ -3389,6 +3497,223 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3389
3497
|
};
|
|
3390
3498
|
}
|
|
3391
3499
|
|
|
3500
|
+
/**
|
|
3501
|
+
* Record tiny pointers after canonical tool receipts settle. A rerun can
|
|
3502
|
+
* then recover each exact source-run receipt before trying the ordinary
|
|
3503
|
+
* cross-run cache. The pointers never contain provider input, credentials,
|
|
3504
|
+
* or provider responses.
|
|
3505
|
+
*
|
|
3506
|
+
* This deliberately uses the same bulk receipt seam as canonical result
|
|
3507
|
+
* persistence. A map can settle hundreds of results at once; falling back
|
|
3508
|
+
* to one read/claim/complete triplet per pointer would make compatibility
|
|
3509
|
+
* bookkeeping contend with the customer's provider work.
|
|
3510
|
+
*/
|
|
3511
|
+
private async persistSourceRunToolReceiptManifests(
|
|
3512
|
+
inputs: Array<{
|
|
3513
|
+
toolId: string;
|
|
3514
|
+
logicalCallId?: string | null;
|
|
3515
|
+
executionAuthScopeDigest?: string | null;
|
|
3516
|
+
resultReceiptKey: string;
|
|
3517
|
+
}>,
|
|
3518
|
+
): Promise<void> {
|
|
3519
|
+
const sourceRunId = this.currentReceiptOwnerRunId.trim();
|
|
3520
|
+
if (
|
|
3521
|
+
!sourceRunId ||
|
|
3522
|
+
!(
|
|
3523
|
+
this.#options.claimRuntimeStepReceipt ||
|
|
3524
|
+
this.#options.claimRuntimeStepReceipts
|
|
3525
|
+
) ||
|
|
3526
|
+
!(
|
|
3527
|
+
this.#options.completeRuntimeStepReceipt ||
|
|
3528
|
+
this.#options.completeRuntimeStepReceipts
|
|
3529
|
+
)
|
|
3530
|
+
) {
|
|
3531
|
+
return;
|
|
3532
|
+
}
|
|
3533
|
+
try {
|
|
3534
|
+
const providerActionVersions = new Map<string, string>();
|
|
3535
|
+
await Promise.all(
|
|
3536
|
+
[...new Set(inputs.map((input) => input.toolId))].map(
|
|
3537
|
+
async (toolId) => {
|
|
3538
|
+
providerActionVersions.set(
|
|
3539
|
+
toolId,
|
|
3540
|
+
(
|
|
3541
|
+
await this.#options.getToolActionCacheVersion?.(toolId)
|
|
3542
|
+
)?.trim() ?? '',
|
|
3543
|
+
);
|
|
3544
|
+
},
|
|
3545
|
+
),
|
|
3546
|
+
);
|
|
3547
|
+
const manifests = inputs.flatMap((input) => {
|
|
3548
|
+
const logicalCallId = input.logicalCallId?.trim();
|
|
3549
|
+
const resultReceiptKey = input.resultReceiptKey.trim();
|
|
3550
|
+
if (!logicalCallId || !resultReceiptKey) return [];
|
|
3551
|
+
const providerActionVersion =
|
|
3552
|
+
providerActionVersions.get(input.toolId) ?? '';
|
|
3553
|
+
if (!providerActionVersion) return [];
|
|
3554
|
+
const key = buildSourceRunToolReceiptManifestKey({
|
|
3555
|
+
orgId: this.#options.orgId,
|
|
3556
|
+
sourceRunId,
|
|
3557
|
+
logicalCallId,
|
|
3558
|
+
});
|
|
3559
|
+
return [
|
|
3560
|
+
{
|
|
3561
|
+
key,
|
|
3562
|
+
output: {
|
|
3563
|
+
kind: 'source-run-tool-receipt-manifest-v1' as const,
|
|
3564
|
+
toolId: input.toolId,
|
|
3565
|
+
providerActionVersion,
|
|
3566
|
+
executionAuthScopeDigest:
|
|
3567
|
+
input.executionAuthScopeDigest?.trim() ?? null,
|
|
3568
|
+
resultReceiptKey,
|
|
3569
|
+
},
|
|
3570
|
+
},
|
|
3571
|
+
];
|
|
3572
|
+
});
|
|
3573
|
+
if (manifests.length === 0) return;
|
|
3574
|
+
|
|
3575
|
+
// Several row callbacks can refer to the same logical call. The first
|
|
3576
|
+
// canonical receipt is authoritative, so one idempotent pointer write is
|
|
3577
|
+
// enough.
|
|
3578
|
+
const uniqueManifests = [
|
|
3579
|
+
...new Map(
|
|
3580
|
+
manifests.map((manifest) => [manifest.key, manifest]),
|
|
3581
|
+
).values(),
|
|
3582
|
+
];
|
|
3583
|
+
const existing = await this.getRuntimeStepReceipts(
|
|
3584
|
+
uniqueManifests.map((manifest) => manifest.key),
|
|
3585
|
+
);
|
|
3586
|
+
const missing = uniqueManifests.filter((manifest) => {
|
|
3587
|
+
const receipt = existing.get(manifest.key);
|
|
3588
|
+
return receipt?.status !== 'completed' && receipt?.status !== 'skipped';
|
|
3589
|
+
});
|
|
3590
|
+
if (missing.length === 0) return;
|
|
3591
|
+
|
|
3592
|
+
const claims = await this.claimRuntimeStepReceipts(
|
|
3593
|
+
missing.map((manifest) => manifest.key),
|
|
3594
|
+
sourceRunId,
|
|
3595
|
+
);
|
|
3596
|
+
await this.completeRuntimeStepReceipts(
|
|
3597
|
+
missing.flatMap((manifest) => {
|
|
3598
|
+
const claim = claims.get(manifest.key);
|
|
3599
|
+
return claim?.leaseId
|
|
3600
|
+
? [
|
|
3601
|
+
{
|
|
3602
|
+
key: manifest.key,
|
|
3603
|
+
runId: sourceRunId,
|
|
3604
|
+
leaseId: claim.leaseId,
|
|
3605
|
+
output: manifest.output,
|
|
3606
|
+
},
|
|
3607
|
+
]
|
|
3608
|
+
: [];
|
|
3609
|
+
}),
|
|
3610
|
+
);
|
|
3611
|
+
} catch (error) {
|
|
3612
|
+
// The canonical result receipt has already settled. A telemetry pointer
|
|
3613
|
+
// must never turn that successful provider result into a failed run.
|
|
3614
|
+
this.runtimeLog(
|
|
3615
|
+
`Tool source-run receipt manifest write skipped: ${error instanceof Error ? error.message : String(error)}`,
|
|
3616
|
+
);
|
|
3617
|
+
}
|
|
3618
|
+
}
|
|
3619
|
+
|
|
3620
|
+
private async persistSourceRunToolReceiptManifest(input: {
|
|
3621
|
+
toolId: string;
|
|
3622
|
+
logicalCallId?: string | null;
|
|
3623
|
+
executionAuthScopeDigest?: string | null;
|
|
3624
|
+
resultReceiptKey: string;
|
|
3625
|
+
}): Promise<void> {
|
|
3626
|
+
await this.persistSourceRunToolReceiptManifests([input]);
|
|
3627
|
+
}
|
|
3628
|
+
|
|
3629
|
+
private async recoverSourceRunToolReceipt(input: {
|
|
3630
|
+
toolId: string;
|
|
3631
|
+
logicalCallId: string;
|
|
3632
|
+
executionAuthScopeDigest?: string | null;
|
|
3633
|
+
cacheKey: string;
|
|
3634
|
+
}): Promise<ToolExecuteResult | null> {
|
|
3635
|
+
const sourceRunId = this.#options.replayedFromRunId?.trim();
|
|
3636
|
+
if (!sourceRunId || !this.#options.getRuntimeStepReceipt) return null;
|
|
3637
|
+
|
|
3638
|
+
const manifestKey = buildSourceRunToolReceiptManifestKey({
|
|
3639
|
+
orgId: this.#options.orgId,
|
|
3640
|
+
sourceRunId,
|
|
3641
|
+
logicalCallId: input.logicalCallId,
|
|
3642
|
+
});
|
|
3643
|
+
const manifestReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
3644
|
+
async () => await this.getRuntimeStepReceipt(manifestKey),
|
|
3645
|
+
);
|
|
3646
|
+
if (
|
|
3647
|
+
!manifestReceipt ||
|
|
3648
|
+
(manifestReceipt.status !== 'completed' &&
|
|
3649
|
+
manifestReceipt.status !== 'skipped')
|
|
3650
|
+
) {
|
|
3651
|
+
return null;
|
|
3652
|
+
}
|
|
3653
|
+
const manifest = recordOrNull(this.runtimeReceiptOutput(manifestReceipt));
|
|
3654
|
+
if (
|
|
3655
|
+
!manifest ||
|
|
3656
|
+
manifest.kind !== 'source-run-tool-receipt-manifest-v1' ||
|
|
3657
|
+
manifest.toolId !== input.toolId ||
|
|
3658
|
+
typeof manifest.resultReceiptKey !== 'string' ||
|
|
3659
|
+
!manifest.resultReceiptKey.trim() ||
|
|
3660
|
+
typeof manifest.providerActionVersion !== 'string'
|
|
3661
|
+
) {
|
|
3662
|
+
return null;
|
|
3663
|
+
}
|
|
3664
|
+
const currentActionVersion =
|
|
3665
|
+
(await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
|
|
3666
|
+
'';
|
|
3667
|
+
const compatibleActionVersions = new Set([
|
|
3668
|
+
currentActionVersion,
|
|
3669
|
+
...(
|
|
3670
|
+
(await this.#options.getToolActionCacheReadVersions?.(input.toolId)) ??
|
|
3671
|
+
[]
|
|
3672
|
+
).map((version) => String(version || '').trim()),
|
|
3673
|
+
]);
|
|
3674
|
+
if (!compatibleActionVersions.has(manifest.providerActionVersion.trim())) {
|
|
3675
|
+
return null;
|
|
3676
|
+
}
|
|
3677
|
+
const manifestAuthScopeDigest =
|
|
3678
|
+
typeof manifest.executionAuthScopeDigest === 'string'
|
|
3679
|
+
? manifest.executionAuthScopeDigest.trim() || null
|
|
3680
|
+
: null;
|
|
3681
|
+
if (
|
|
3682
|
+
manifestAuthScopeDigest !==
|
|
3683
|
+
(input.executionAuthScopeDigest?.trim() || null)
|
|
3684
|
+
) {
|
|
3685
|
+
return null;
|
|
3686
|
+
}
|
|
3687
|
+
const resultReceiptKey = manifest.resultReceiptKey.trim();
|
|
3688
|
+
const sourceReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
3689
|
+
async () => await this.getRuntimeStepReceipt(resultReceiptKey),
|
|
3690
|
+
);
|
|
3691
|
+
if (
|
|
3692
|
+
!sourceReceipt ||
|
|
3693
|
+
(sourceReceipt.status !== 'completed' &&
|
|
3694
|
+
sourceReceipt.status !== 'skipped')
|
|
3695
|
+
) {
|
|
3696
|
+
return null;
|
|
3697
|
+
}
|
|
3698
|
+
const stored = this.runtimeReceiptOutput(sourceReceipt);
|
|
3699
|
+
const result = isSerializedToolExecuteResult(stored)
|
|
3700
|
+
? deserializeToolExecuteResult(stored)
|
|
3701
|
+
: isToolExecuteResult(stored)
|
|
3702
|
+
? stored
|
|
3703
|
+
: null;
|
|
3704
|
+
if (!result || !this.canReuseToolResultForCurrentResponseContract(result)) {
|
|
3705
|
+
return null;
|
|
3706
|
+
}
|
|
3707
|
+
return cloneToolExecuteResultWithExecution(
|
|
3708
|
+
result,
|
|
3709
|
+
toolExecutionMetadataForOutcome({
|
|
3710
|
+
kind: 'source_run',
|
|
3711
|
+
cacheKey: input.cacheKey,
|
|
3712
|
+
receiptKey: resultReceiptKey,
|
|
3713
|
+
}),
|
|
3714
|
+
);
|
|
3715
|
+
}
|
|
3716
|
+
|
|
3392
3717
|
private async releaseRuntimeStepReceipt(
|
|
3393
3718
|
key: string,
|
|
3394
3719
|
_runId: string,
|
|
@@ -4206,6 +4531,47 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4206
4531
|
return buildDurableToolCallCacheKey(identity);
|
|
4207
4532
|
}
|
|
4208
4533
|
|
|
4534
|
+
/**
|
|
4535
|
+
* The first key is the only write/provider identity. Later keys are explicit
|
|
4536
|
+
* action-local read fallbacks, used only when source metadata says their
|
|
4537
|
+
* completed result remains compatible. Deploy and generated-catalog state
|
|
4538
|
+
* never participate in this list.
|
|
4539
|
+
*/
|
|
4540
|
+
private async compatibleDurableToolCallCacheKeysForScope(input: {
|
|
4541
|
+
toolId: string;
|
|
4542
|
+
requestInput: Record<string, unknown>;
|
|
4543
|
+
executionAuthScopeDigest?: string | null;
|
|
4544
|
+
staleAfterSeconds?: number | null;
|
|
4545
|
+
playLocalScope?: string | null;
|
|
4546
|
+
}): Promise<string[]> {
|
|
4547
|
+
const identity = await this.durableToolCallCacheIdentity(input);
|
|
4548
|
+
const primaryVersion = identity.providerActionVersion?.trim();
|
|
4549
|
+
if (!primaryVersion) {
|
|
4550
|
+
throw new Error(
|
|
4551
|
+
`Tool ${input.toolId} is missing a provider action compatibility version.`,
|
|
4552
|
+
);
|
|
4553
|
+
}
|
|
4554
|
+
const readVersions = await this.#options.getToolActionCacheReadVersions?.(
|
|
4555
|
+
input.toolId,
|
|
4556
|
+
);
|
|
4557
|
+
const versions = [
|
|
4558
|
+
primaryVersion,
|
|
4559
|
+
...(readVersions ?? [])
|
|
4560
|
+
.map((version) => String(version || '').trim())
|
|
4561
|
+
.filter(Boolean),
|
|
4562
|
+
];
|
|
4563
|
+
return [
|
|
4564
|
+
...new Set(
|
|
4565
|
+
versions.map((providerActionVersion) =>
|
|
4566
|
+
buildDurableToolCallCacheKey({
|
|
4567
|
+
...identity,
|
|
4568
|
+
providerActionVersion,
|
|
4569
|
+
}),
|
|
4570
|
+
),
|
|
4571
|
+
),
|
|
4572
|
+
];
|
|
4573
|
+
}
|
|
4574
|
+
|
|
4209
4575
|
private async legacyDurableToolCallCacheKeyForScope(input: {
|
|
4210
4576
|
toolId: string;
|
|
4211
4577
|
requestInput: Record<string, unknown>;
|
|
@@ -4263,6 +4629,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4263
4629
|
});
|
|
4264
4630
|
}
|
|
4265
4631
|
|
|
4632
|
+
private async compatibleDurableToolCallCacheKeys(input: {
|
|
4633
|
+
toolId: string;
|
|
4634
|
+
requestInput: Record<string, unknown>;
|
|
4635
|
+
executionAuthScopeDigest?: string | null;
|
|
4636
|
+
staleAfterSeconds?: number | null;
|
|
4637
|
+
}): Promise<string[]> {
|
|
4638
|
+
return await this.compatibleDurableToolCallCacheKeysForScope({
|
|
4639
|
+
...input,
|
|
4640
|
+
playLocalScope: this.currentGovernance.currentPlayId,
|
|
4641
|
+
});
|
|
4642
|
+
}
|
|
4643
|
+
|
|
4266
4644
|
private async legacyDurableToolCallCacheKey(input: {
|
|
4267
4645
|
toolId: string;
|
|
4268
4646
|
requestInput: Record<string, unknown>;
|
|
@@ -4429,6 +4807,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4429
4807
|
}) => Promise<T>;
|
|
4430
4808
|
},
|
|
4431
4809
|
): Promise<T> {
|
|
4810
|
+
this.assertNoPendingDurableSuspension();
|
|
4432
4811
|
let releaseExternalCallSlot: (() => void) | null = null;
|
|
4433
4812
|
const execute = async (leaseId: string | null): Promise<T> =>
|
|
4434
4813
|
await opts.execute({
|
|
@@ -4471,7 +4850,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4471
4850
|
cacheEpochMs: this.durableCallCacheEpochMs,
|
|
4472
4851
|
});
|
|
4473
4852
|
try {
|
|
4474
|
-
|
|
4853
|
+
const output = await executeWithDurableRuntimeReceipt<T>({
|
|
4475
4854
|
operation,
|
|
4476
4855
|
id,
|
|
4477
4856
|
runId: this.currentReceiptOwnerRunId,
|
|
@@ -4499,6 +4878,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4499
4878
|
log: (message) => this.runtimeLog(message),
|
|
4500
4879
|
execute: ({ leaseId }) => execute(leaseId),
|
|
4501
4880
|
});
|
|
4881
|
+
// Every receipt publication and cache return crosses this one success
|
|
4882
|
+
// seam. A customer catch cannot turn a pending durable handoff into a
|
|
4883
|
+
// reusable completed value.
|
|
4884
|
+
this.assertNoPendingDurableSuspension();
|
|
4885
|
+
return output;
|
|
4502
4886
|
} finally {
|
|
4503
4887
|
const release = releaseExternalCallSlot as (() => void) | null;
|
|
4504
4888
|
release?.();
|
|
@@ -4705,6 +5089,62 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4705
5089
|
});
|
|
4706
5090
|
}
|
|
4707
5091
|
|
|
5092
|
+
/**
|
|
5093
|
+
* A forced call bypasses the immutable Tool Result Receipt, so its physical
|
|
5094
|
+
* gateway invocation must bypass that receipt's replay identity too. Keep
|
|
5095
|
+
* the transport key deterministic for one logical call: a retry of that
|
|
5096
|
+
* call still converges, while a later explicit force gets a fresh identity.
|
|
5097
|
+
*/
|
|
5098
|
+
private toolTransportReceiptKey(input: {
|
|
5099
|
+
toolId: string;
|
|
5100
|
+
resultReceiptKey: string;
|
|
5101
|
+
providerOperationKey: string;
|
|
5102
|
+
force?: boolean;
|
|
5103
|
+
}): string {
|
|
5104
|
+
if (input.force !== true) return input.resultReceiptKey;
|
|
5105
|
+
return `${buildDurableToolReceiptPrefix({
|
|
5106
|
+
orgId: this.#options.orgId,
|
|
5107
|
+
toolId: input.toolId,
|
|
5108
|
+
})}${stableDigest(`forced-provider-invocation:${input.providerOperationKey}`)}`;
|
|
5109
|
+
}
|
|
5110
|
+
|
|
5111
|
+
/**
|
|
5112
|
+
* Ordinary operations do not carry provider-idempotency metadata, but the
|
|
5113
|
+
* execution relay still needs a semantic signal that an explicit force is
|
|
5114
|
+
* a new provider operation. Unsafe-write operations already carry their
|
|
5115
|
+
* forced provider key directly, so only ordinary calls need this attested
|
|
5116
|
+
* canonical-operation salt.
|
|
5117
|
+
*/
|
|
5118
|
+
private providerForceRefreshIdentitiesForToolCall(input: {
|
|
5119
|
+
operationKey: string;
|
|
5120
|
+
force?: boolean;
|
|
5121
|
+
requiresExecutionFence?: boolean;
|
|
5122
|
+
logicalCallId?: string;
|
|
5123
|
+
}): string[] {
|
|
5124
|
+
if (input.force !== true || input.requiresExecutionFence === true) {
|
|
5125
|
+
return [];
|
|
5126
|
+
}
|
|
5127
|
+
return [
|
|
5128
|
+
this.providerIdempotencyKeyForToolCall({
|
|
5129
|
+
operationKey: input.operationKey,
|
|
5130
|
+
force: true,
|
|
5131
|
+
logicalCallId: input.logicalCallId,
|
|
5132
|
+
}),
|
|
5133
|
+
];
|
|
5134
|
+
}
|
|
5135
|
+
|
|
5136
|
+
private providerForceRefreshOptionsForToolCall(input: {
|
|
5137
|
+
operationKey: string;
|
|
5138
|
+
force?: boolean;
|
|
5139
|
+
requiresExecutionFence?: boolean;
|
|
5140
|
+
logicalCallId?: string;
|
|
5141
|
+
}): Pick<ToolExecutionApiOptions, 'providerOperationForceRefreshIdentities'> {
|
|
5142
|
+
const identities = this.providerForceRefreshIdentitiesForToolCall(input);
|
|
5143
|
+
return identities.length > 0
|
|
5144
|
+
? { providerOperationForceRefreshIdentities: identities }
|
|
5145
|
+
: {};
|
|
5146
|
+
}
|
|
5147
|
+
|
|
4708
5148
|
/**
|
|
4709
5149
|
* Bounded per-cell producer trace (ADR 0018).
|
|
4710
5150
|
*
|
|
@@ -6125,6 +6565,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6125
6565
|
input: TInput,
|
|
6126
6566
|
options?: { description?: string },
|
|
6127
6567
|
): Promise<TOutput> {
|
|
6568
|
+
this.assertNoPendingDurableSuspension();
|
|
6128
6569
|
if (!isRuntimeStepProgram(program)) {
|
|
6129
6570
|
throw new Error('ctx.runSteps(program, input) requires steps().');
|
|
6130
6571
|
}
|
|
@@ -6149,6 +6590,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6149
6590
|
input: MapFieldDefinition<T, TColumns>,
|
|
6150
6591
|
options?: RuntimeDatasetOptions<T>,
|
|
6151
6592
|
): Promise<PlayDataset<Record<string, unknown>>> {
|
|
6593
|
+
this.assertNoPendingDurableSuspension();
|
|
6152
6594
|
if (rowContext.getStore()) {
|
|
6153
6595
|
throw new Error(
|
|
6154
6596
|
'Nested ctx.dataset() is not supported. Flatten your columns into one dataset, or keep custom per-row logic inside a single column.',
|
|
@@ -6305,6 +6747,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6305
6747
|
this.#options.playName &&
|
|
6306
6748
|
this.#options.runId,
|
|
6307
6749
|
);
|
|
6750
|
+
// The invocation id is deterministic for a replay: mapInvocationIndex is
|
|
6751
|
+
// rebuilt in authored call order. Look up a previously suspended plan now,
|
|
6752
|
+
// before a stateful source query can decide that a sibling action is no
|
|
6753
|
+
// longer eligible because an earlier action already completed.
|
|
6754
|
+
const replayMapInvocationId = `${normalizedMapNamespace}:${this.mapInvocationIndex}`;
|
|
6755
|
+
const suspendedInputRows = shouldStreamRuntimeBackedDataset
|
|
6756
|
+
? null
|
|
6757
|
+
: (this.checkpoint.mapFrames?.[replayMapInvocationId]
|
|
6758
|
+
?.suspendedInputRows ?? null);
|
|
6308
6759
|
if (shouldStreamRuntimeBackedDataset) {
|
|
6309
6760
|
// Net-new admission happens one source page at a time. Its progress and
|
|
6310
6761
|
// returned dataset describe admitted rows, not every provider/CSV row
|
|
@@ -6954,11 +7405,24 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6954
7405
|
},
|
|
6955
7406
|
});
|
|
6956
7407
|
}
|
|
6957
|
-
const
|
|
6958
|
-
|
|
6959
|
-
|
|
6960
|
-
|
|
6961
|
-
|
|
7408
|
+
const rawMaterializedEntries = suspendedInputRows?.length
|
|
7409
|
+
? suspendedInputRows.map((row) => ({
|
|
7410
|
+
item: cloneCsvAliasedRow(row.data) as T,
|
|
7411
|
+
inputIndex: row.inputIndex,
|
|
7412
|
+
}))
|
|
7413
|
+
: (
|
|
7414
|
+
await materializePlayDatasetInput(items, {
|
|
7415
|
+
onRow: (row) => {
|
|
7416
|
+
materializationMemoryTracker.track(row);
|
|
7417
|
+
},
|
|
7418
|
+
})
|
|
7419
|
+
).map((item, inputIndex) => ({ item, inputIndex }));
|
|
7420
|
+
const rawMaterializedItems = rawMaterializedEntries.map(
|
|
7421
|
+
(entry) => entry.item,
|
|
7422
|
+
);
|
|
7423
|
+
const originalInputIndexByItem = new Map<T, number>(
|
|
7424
|
+
rawMaterializedEntries.map(({ item, inputIndex }) => [item, inputIndex]),
|
|
7425
|
+
);
|
|
6962
7426
|
// Silent dedupe of duplicate explicit map keys: keep the first row per
|
|
6963
7427
|
// canonical key and drop subsequent duplicates. Deduping the materialized
|
|
6964
7428
|
// input here keeps every downstream derivation (rawItems, itemsToProcess,
|
|
@@ -6971,7 +7435,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6971
7435
|
resolver: (item, index) =>
|
|
6972
7436
|
explicitKeyResolver(
|
|
6973
7437
|
this.toOutputRow(item as Record<string, unknown>),
|
|
6974
|
-
index,
|
|
7438
|
+
originalInputIndexByItem.get(item) ?? index,
|
|
6975
7439
|
),
|
|
6976
7440
|
})
|
|
6977
7441
|
: { rows: rawMaterializedItems, droppedCount: 0, duplicateKeys: [] };
|
|
@@ -6988,7 +7452,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6988
7452
|
itemsToProcess = materializedItems.map((item) =>
|
|
6989
7453
|
this.toOutputRow(item as Record<string, unknown>),
|
|
6990
7454
|
);
|
|
6991
|
-
itemOriginalIndexes = materializedItems.map(
|
|
7455
|
+
itemOriginalIndexes = materializedItems.map(
|
|
7456
|
+
(item, index) => originalInputIndexByItem.get(item) ?? index,
|
|
7457
|
+
);
|
|
6992
7458
|
const mapStartSeedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
6993
7459
|
if (this.#options.onMapStart) {
|
|
6994
7460
|
// ALWAYS stamp the canonical seed row key — including no-explicit-key
|
|
@@ -7376,6 +7842,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7376
7842
|
pendingRowsCount: 0,
|
|
7377
7843
|
failedRowsCount: durableFailedRows,
|
|
7378
7844
|
activeBoundaryId: null,
|
|
7845
|
+
suspendedInputRows: undefined,
|
|
7379
7846
|
updatedAt: Date.now(),
|
|
7380
7847
|
});
|
|
7381
7848
|
}
|
|
@@ -8224,6 +8691,67 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8224
8691
|
});
|
|
8225
8692
|
return WAITING_ROW;
|
|
8226
8693
|
}
|
|
8694
|
+
if (isPlayExecutionSuspendedError(error)) {
|
|
8695
|
+
const completedKeys = new Set(
|
|
8696
|
+
completedRowsToPersist.map((row) => row.key),
|
|
8697
|
+
);
|
|
8698
|
+
const suspendedInputRows = items.flatMap((item, index) => {
|
|
8699
|
+
const rowKey = executionRowKey(
|
|
8700
|
+
this.toOutputRow(item as Record<string, unknown>),
|
|
8701
|
+
index,
|
|
8702
|
+
);
|
|
8703
|
+
if (completedKeys.has(rowKey)) return [];
|
|
8704
|
+
return [
|
|
8705
|
+
{
|
|
8706
|
+
inputIndex: executionRowIndex(index),
|
|
8707
|
+
data: this.toPersistedOutputRow(
|
|
8708
|
+
this.toOutputRow(item as Record<string, unknown>),
|
|
8709
|
+
),
|
|
8710
|
+
},
|
|
8711
|
+
];
|
|
8712
|
+
});
|
|
8713
|
+
const existingFrame =
|
|
8714
|
+
this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
|
|
8715
|
+
this.setMapFrame({
|
|
8716
|
+
...(existingFrame ?? {
|
|
8717
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
8718
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
8719
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
8720
|
+
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
8721
|
+
totalRows,
|
|
8722
|
+
completedRowKeys: [],
|
|
8723
|
+
pendingRowKeys: [],
|
|
8724
|
+
startedAt: Date.now(),
|
|
8725
|
+
}),
|
|
8726
|
+
status: 'suspended',
|
|
8727
|
+
pendingRowKeys: suspendedInputRows.map(({ data, inputIndex }) =>
|
|
8728
|
+
executionRowKey(data, inputIndex),
|
|
8729
|
+
),
|
|
8730
|
+
suspendedInputRows,
|
|
8731
|
+
activeBoundaryId:
|
|
8732
|
+
error.suspension.kind === 'sleep'
|
|
8733
|
+
? error.suspension.boundaryId
|
|
8734
|
+
: null,
|
|
8735
|
+
updatedAt: Date.now(),
|
|
8736
|
+
});
|
|
8737
|
+
this.emitExecutionEvent({
|
|
8738
|
+
type: 'map.suspended',
|
|
8739
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
8740
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
8741
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
8742
|
+
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
8743
|
+
completedRows: completedKeys.size,
|
|
8744
|
+
failedRows: 0,
|
|
8745
|
+
totalRows,
|
|
8746
|
+
at: Date.now(),
|
|
8747
|
+
});
|
|
8748
|
+
// ctx.sleep() emitted its boundary checkpoint before control reached
|
|
8749
|
+
// this row catch. Emit once more after recording the admitted plan so
|
|
8750
|
+
// the scheduler can resume the same action set if the worker exits
|
|
8751
|
+
// immediately after handing off the suspension.
|
|
8752
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
8753
|
+
throw error;
|
|
8754
|
+
}
|
|
8227
8755
|
const fieldName = rowContext.getStore()?.fieldName;
|
|
8228
8756
|
updateMapFrameProgress({
|
|
8229
8757
|
status: 'failed',
|
|
@@ -8257,16 +8785,51 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8257
8785
|
try {
|
|
8258
8786
|
settledResults[idx] = await runRow(idx);
|
|
8259
8787
|
} catch (error) {
|
|
8260
|
-
firstRowError =
|
|
8788
|
+
firstRowError = this.pendingDurableSuspension
|
|
8789
|
+
? new PlayExecutionSuspendedError(this.pendingDurableSuspension)
|
|
8790
|
+
: (firstRowError ?? error);
|
|
8791
|
+
this.wakeToolDispatcher();
|
|
8261
8792
|
throw error;
|
|
8262
8793
|
}
|
|
8263
8794
|
}
|
|
8264
8795
|
});
|
|
8265
8796
|
|
|
8266
|
-
|
|
8797
|
+
// Drain queued work but exit early if a suspension error occurs.
|
|
8798
|
+
// When a worker suspends (e.g., ctx.sleep), we must propagate the
|
|
8799
|
+
// suspension immediately without waiting for other concurrent rows to
|
|
8800
|
+
// settle. This prevents restart loops and false completions.
|
|
8801
|
+
await this.drainQueuedWork(workers, () => {
|
|
8802
|
+
if (firstRowError != null) {
|
|
8803
|
+
if (
|
|
8804
|
+
isPlayExecutionSuspendedError(firstRowError) ||
|
|
8805
|
+
isPlayRowExecutionSuspendedError(firstRowError)
|
|
8806
|
+
)
|
|
8807
|
+
return firstRowError;
|
|
8808
|
+
}
|
|
8809
|
+
return null;
|
|
8810
|
+
});
|
|
8267
8811
|
this.lastDatasetStep = datasetStep;
|
|
8268
8812
|
this.activeDatasetStep = null;
|
|
8269
8813
|
|
|
8814
|
+
// If drainQueuedWork exited early due to suspension, throw it immediately
|
|
8815
|
+
// WITHOUT waiting for Promise.allSettled(workers). The worker promises
|
|
8816
|
+
// may never settle (e.g., sibling rows blocked on long-running operations),
|
|
8817
|
+
// and we must not block suspension propagation on them.
|
|
8818
|
+
if (firstRowError != null) {
|
|
8819
|
+
if (
|
|
8820
|
+
isPlayExecutionSuspendedError(firstRowError) ||
|
|
8821
|
+
isPlayRowExecutionSuspendedError(firstRowError)
|
|
8822
|
+
) {
|
|
8823
|
+
// Use the existing failure barrier to commit accepted terminal rows
|
|
8824
|
+
// before the caller hands this invocation back to the scheduler.
|
|
8825
|
+
throw new FailFastMapRowsError({
|
|
8826
|
+
cause: firstRowError,
|
|
8827
|
+
completedRows: completedRowsToPersist,
|
|
8828
|
+
failedRows: failedRowsToPersist,
|
|
8829
|
+
});
|
|
8830
|
+
}
|
|
8831
|
+
}
|
|
8832
|
+
|
|
8270
8833
|
const settled = await Promise.allSettled(workers);
|
|
8271
8834
|
const rejected = settled.find(
|
|
8272
8835
|
(result): result is PromiseRejectedResult => result.status === 'rejected',
|
|
@@ -8356,8 +8919,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8356
8919
|
await incrementalPersistence?.checkpoint(checkpointUpdates);
|
|
8357
8920
|
await incrementalPersistence?.flush();
|
|
8358
8921
|
this.pendingRowEventBoundaries = [];
|
|
8359
|
-
this
|
|
8360
|
-
throw new PlayExecutionSuspendedError({
|
|
8922
|
+
this.suspendDurably({
|
|
8361
8923
|
kind: 'integration_event_batch',
|
|
8362
8924
|
boundaries: uniqueBoundaries,
|
|
8363
8925
|
});
|
|
@@ -8452,7 +9014,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8452
9014
|
if (
|
|
8453
9015
|
(!continueOnProviderUnavailable &&
|
|
8454
9016
|
step.continueOnProviderUnavailable !== true) ||
|
|
8455
|
-
(!(error
|
|
9017
|
+
(!isProviderExhaustedFailure(error) &&
|
|
8456
9018
|
!isProviderUnavailable(error))
|
|
8457
9019
|
) {
|
|
8458
9020
|
throw error;
|
|
@@ -8537,6 +9099,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8537
9099
|
let value: unknown;
|
|
8538
9100
|
try {
|
|
8539
9101
|
value = await runStep();
|
|
9102
|
+
// A step-program resolver has the same publication boundary as a
|
|
9103
|
+
// normal mapped field. Its customer callback may catch `ctx.sleep()`;
|
|
9104
|
+
// do not turn that parked invocation into a waterfall output/cell.
|
|
9105
|
+
this.assertNoPendingDurableSuspension();
|
|
8540
9106
|
} catch (error) {
|
|
8541
9107
|
// A provider admission, transient upstream failure, or exhausted
|
|
8542
9108
|
// managed provider account is a visible unavailable leg, not a row
|
|
@@ -8545,8 +9111,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8545
9111
|
// author hand-writing error handling.
|
|
8546
9112
|
if (
|
|
8547
9113
|
!continueOnProviderUnavailable ||
|
|
8548
|
-
(!(error
|
|
8549
|
-
!isProviderUnavailable(error))
|
|
9114
|
+
(!isProviderExhaustedFailure(error) && !isProviderUnavailable(error))
|
|
8550
9115
|
) {
|
|
8551
9116
|
throw error;
|
|
8552
9117
|
}
|
|
@@ -8755,7 +9320,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8755
9320
|
|
|
8756
9321
|
const source = Function.prototype.toString.call(resolver);
|
|
8757
9322
|
return (
|
|
8758
|
-
!source.includes('.tools.execute(') &&
|
|
9323
|
+
!source.includes('.tools.execute(') &&
|
|
9324
|
+
!source.includes('.runPlay(') &&
|
|
9325
|
+
!source.includes('.runPlayAsync(')
|
|
8759
9326
|
);
|
|
8760
9327
|
});
|
|
8761
9328
|
}
|
|
@@ -9117,7 +9684,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9117
9684
|
});
|
|
9118
9685
|
}
|
|
9119
9686
|
|
|
9120
|
-
private async drainQueuedWork<T>(
|
|
9687
|
+
private async drainQueuedWork<T>(
|
|
9688
|
+
promises: Promise<T>[],
|
|
9689
|
+
earlyExitCheck?: () => unknown,
|
|
9690
|
+
): Promise<void> {
|
|
9121
9691
|
// One dispatcher owns ready work, fixed lane coalescing deadlines, and the
|
|
9122
9692
|
// bounded in-flight registry. A row resumes after its own receipt persists;
|
|
9123
9693
|
// it never waits for unrelated sibling calls from the previous column.
|
|
@@ -9149,6 +9719,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9149
9719
|
try {
|
|
9150
9720
|
let pass = 0;
|
|
9151
9721
|
while (true) {
|
|
9722
|
+
// Check for early exit FIRST, before any dispatch or wait logic.
|
|
9723
|
+
// When a row suspends, we must exit immediately without waiting for
|
|
9724
|
+
// sibling rows or tool dispatcher wake events.
|
|
9725
|
+
const suspension = earlyExitCheck?.();
|
|
9726
|
+
if (suspension != null) {
|
|
9727
|
+
// Queued calls have not started; reject them at the suspension fence.
|
|
9728
|
+
// Already-dispatched calls must settle while this attempt still owns
|
|
9729
|
+
// their receipts. Runner teardown releases leases, so detaching live
|
|
9730
|
+
// provider requests here could duplicate them on the next resume.
|
|
9731
|
+
this.rejectQueuedToolCalls(suspension);
|
|
9732
|
+
await Promise.allSettled([...inFlightToolExecutions]);
|
|
9733
|
+
break;
|
|
9734
|
+
}
|
|
9735
|
+
|
|
9152
9736
|
if (this.toolDispatcherFailure != null) {
|
|
9153
9737
|
this.rejectQueuedToolCalls(this.toolDispatcherFailure);
|
|
9154
9738
|
await Promise.allSettled([...inFlightToolExecutions]);
|
|
@@ -9237,7 +9821,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9237
9821
|
return resolver;
|
|
9238
9822
|
}
|
|
9239
9823
|
|
|
9240
|
-
|
|
9824
|
+
const value = await resolver(row, this, fields, index, previousCell);
|
|
9825
|
+
// This is the common user-code return seam for every mapped field path
|
|
9826
|
+
// (incremental, pure, and step-program adapters). Check before callers
|
|
9827
|
+
// serialize a cell, emit a completion event, or advance a map frame.
|
|
9828
|
+
this.assertNoPendingDurableSuspension();
|
|
9829
|
+
return value;
|
|
9241
9830
|
}
|
|
9242
9831
|
|
|
9243
9832
|
private toOutputRow<T>(item: T): Record<string, unknown> {
|
|
@@ -9358,12 +9947,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9358
9947
|
!eventWaitHandler &&
|
|
9359
9948
|
!isQueryResultDatasetReadRequest(toolId, input) &&
|
|
9360
9949
|
!isAlwaysFreshIntegrationTool(toolId);
|
|
9361
|
-
let
|
|
9362
|
-
|
|
9363
|
-
|
|
9364
|
-
|
|
9365
|
-
|
|
9366
|
-
|
|
9950
|
+
let compatibleDurableCacheKeys =
|
|
9951
|
+
await this.compatibleDurableToolCallCacheKeys({
|
|
9952
|
+
toolId,
|
|
9953
|
+
requestInput: input,
|
|
9954
|
+
executionAuthScopeDigest,
|
|
9955
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
9956
|
+
});
|
|
9957
|
+
let durableCacheKey = compatibleDurableCacheKeys[0];
|
|
9958
|
+
if (!durableCacheKey) {
|
|
9959
|
+
throw new Error(`Tool ${toolId} is missing a primary durable cache key.`);
|
|
9960
|
+
}
|
|
9367
9961
|
let legacyDurableCacheKey = await this.legacyDurableToolCallCacheKey({
|
|
9368
9962
|
toolId,
|
|
9369
9963
|
requestInput: input,
|
|
@@ -9390,7 +9984,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9390
9984
|
// incompatible state deliberately forgoes caching until it is repaired.
|
|
9391
9985
|
let bypassDirectResultCache = false;
|
|
9392
9986
|
const checkpointCacheKeys = [
|
|
9393
|
-
|
|
9987
|
+
...compatibleDurableCacheKeys,
|
|
9394
9988
|
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
9395
9989
|
];
|
|
9396
9990
|
|
|
@@ -9408,6 +10002,28 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9408
10002
|
if (toolId === 'run_javascript') {
|
|
9409
10003
|
throw new Error(DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE);
|
|
9410
10004
|
}
|
|
10005
|
+
// A user-requested rerun first asks its exact source run whether this
|
|
10006
|
+
// logical call already completed. This is stronger than the shared cache:
|
|
10007
|
+
// it is an explicit, authenticated pointer to one known source receipt.
|
|
10008
|
+
if (cacheableToolResult && !toolCachePolicy.force) {
|
|
10009
|
+
const sourceRunRecovered = await this.recoverSourceRunToolReceipt({
|
|
10010
|
+
toolId,
|
|
10011
|
+
logicalCallId,
|
|
10012
|
+
executionAuthScopeDigest,
|
|
10013
|
+
cacheKey: durableCacheKey,
|
|
10014
|
+
});
|
|
10015
|
+
if (sourceRunRecovered) {
|
|
10016
|
+
this.runtimeLog(
|
|
10017
|
+
`Calling tool: ${toolId} recovered from source run ${this.#options.replayedFromRunId}`,
|
|
10018
|
+
);
|
|
10019
|
+
this.emitToolCallLifecycle('started', toolId, normalizedKey);
|
|
10020
|
+
this.emitToolCallLifecycle('settled', toolId, normalizedKey, {
|
|
10021
|
+
outcome: 'cached',
|
|
10022
|
+
durationMs: 0,
|
|
10023
|
+
});
|
|
10024
|
+
return this.materializeToolExecutionResult(sourceRunRecovered);
|
|
10025
|
+
}
|
|
10026
|
+
}
|
|
9411
10027
|
// Tool-call budget is charged by the Governor's acquireToolSlot in
|
|
9412
10028
|
// callToolExecutionAPI, so each real outbound call is counted exactly once
|
|
9413
10029
|
// (cached/checkpoint recoveries never acquire a slot and so never charge).
|
|
@@ -9476,6 +10092,45 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9476
10092
|
// reader seam rather than teaching the deep Module an untyped result.
|
|
9477
10093
|
// New direct calls use the real Tool Result Receipt Adapter below.
|
|
9478
10094
|
if (cacheableToolResult && !toolCachePolicy.force) {
|
|
10095
|
+
// A source action can explicitly carry forward a previous result
|
|
10096
|
+
// contract after a local version bump. Read those exact older keys
|
|
10097
|
+
// before the canonical key; do not derive fallbacks from a deploy.
|
|
10098
|
+
for (const compatibleCacheKey of compatibleDurableCacheKeys.slice(
|
|
10099
|
+
1,
|
|
10100
|
+
)) {
|
|
10101
|
+
const compatibleReceipt =
|
|
10102
|
+
await this.withRuntimeReceiptHydrationTurn(
|
|
10103
|
+
async () =>
|
|
10104
|
+
await this.getRuntimeStepReceipt(compatibleCacheKey),
|
|
10105
|
+
);
|
|
10106
|
+
if (
|
|
10107
|
+
compatibleReceipt &&
|
|
10108
|
+
(compatibleReceipt.status === 'completed' ||
|
|
10109
|
+
compatibleReceipt.status === 'skipped') &&
|
|
10110
|
+
compatibleReceipt.output !== undefined
|
|
10111
|
+
) {
|
|
10112
|
+
const recoveredCompatible = markToolExecuteResultExecutionOutcome(
|
|
10113
|
+
this.runtimeReceiptOutput(compatibleReceipt),
|
|
10114
|
+
toolExecutionOutcomeForDurableReceipt({
|
|
10115
|
+
source: 'cache',
|
|
10116
|
+
receiptKey: compatibleCacheKey,
|
|
10117
|
+
cacheKey: directCacheKey,
|
|
10118
|
+
}),
|
|
10119
|
+
);
|
|
10120
|
+
if (
|
|
10121
|
+
this.canReuseToolResultForCurrentResponseContract(
|
|
10122
|
+
recoveredCompatible,
|
|
10123
|
+
)
|
|
10124
|
+
) {
|
|
10125
|
+
this.runtimeLog(
|
|
10126
|
+
`Calling tool: ${toolId} recovered from an explicit action compatibility receipt`,
|
|
10127
|
+
);
|
|
10128
|
+
return isToolExecuteResult(recoveredCompatible)
|
|
10129
|
+
? this.materializeToolExecutionResult(recoveredCompatible)
|
|
10130
|
+
: recoveredCompatible;
|
|
10131
|
+
}
|
|
10132
|
+
}
|
|
10133
|
+
}
|
|
9479
10134
|
const historicalReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
9480
10135
|
async () => await this.getRuntimeStepReceipt(directCacheKey),
|
|
9481
10136
|
);
|
|
@@ -9592,14 +10247,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9592
10247
|
}
|
|
9593
10248
|
const useDirectResultReceipts =
|
|
9594
10249
|
cacheableToolResult && !bypassDirectResultCache;
|
|
10250
|
+
const providerOperationKey = this.providerIdempotencyKeyForToolCall({
|
|
10251
|
+
operationKey: providerIdempotencyKeyBase,
|
|
10252
|
+
force: cacheableToolResult && toolCachePolicy.force,
|
|
10253
|
+
logicalCallId,
|
|
10254
|
+
});
|
|
9595
10255
|
const physicalDirectKey = useDirectResultReceipts
|
|
9596
|
-
?
|
|
10256
|
+
? this.toolTransportReceiptKey({
|
|
10257
|
+
toolId,
|
|
10258
|
+
resultReceiptKey: directCacheKey,
|
|
10259
|
+
providerOperationKey,
|
|
10260
|
+
force: toolCachePolicy.force,
|
|
10261
|
+
})
|
|
9597
10262
|
: `${buildDurableToolReceiptPrefix({
|
|
9598
10263
|
orgId: this.#options.orgId ?? 'unknown-org',
|
|
9599
10264
|
toolId,
|
|
9600
10265
|
})}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`;
|
|
9601
10266
|
const resultReceiptKey = directResultReceiptKey;
|
|
9602
|
-
const providerOperationKey = providerIdempotencyKeyBase;
|
|
9603
10267
|
const toolRetryPolicy = await this.#options
|
|
9604
10268
|
.getToolRetryPolicy?.(toolId, input)
|
|
9605
10269
|
.catch(() => null);
|
|
@@ -9665,16 +10329,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9665
10329
|
? resultReceiptKey
|
|
9666
10330
|
: physicalDirectKey,
|
|
9667
10331
|
providerIdempotencyKey:
|
|
9668
|
-
|
|
9669
|
-
|
|
9670
|
-
|
|
9671
|
-
? providerIdempotencyKeyBase
|
|
9672
|
-
: physicalDirectKey,
|
|
9673
|
-
force: cacheableToolResult && toolCachePolicy.force,
|
|
9674
|
-
logicalCallId,
|
|
9675
|
-
}),
|
|
10332
|
+
useDirectResultReceipts && operationKey
|
|
10333
|
+
? providerOperationKey
|
|
10334
|
+
: physicalDirectKey,
|
|
9676
10335
|
}
|
|
9677
10336
|
: {}),
|
|
10337
|
+
...(useDirectResultReceipts && operationKey
|
|
10338
|
+
? this.providerForceRefreshOptionsForToolCall({
|
|
10339
|
+
operationKey: providerIdempotencyKeyBase,
|
|
10340
|
+
force: toolCachePolicy.force,
|
|
10341
|
+
requiresExecutionFence:
|
|
10342
|
+
toolRetryPolicy?.requiresExecutionFence,
|
|
10343
|
+
logicalCallId,
|
|
10344
|
+
})
|
|
10345
|
+
: {}),
|
|
9678
10346
|
playNodeScope: buildPlayNodeScope({
|
|
9679
10347
|
toolId,
|
|
9680
10348
|
callKey: normalizedKey,
|
|
@@ -9721,6 +10389,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9721
10389
|
: {}),
|
|
9722
10390
|
withCompletedReceiptHydration: (hydrate) =>
|
|
9723
10391
|
this.withRuntimeReceiptHydrationTurn(hydrate),
|
|
10392
|
+
onStored: async ({ resultReceiptKey }) => {
|
|
10393
|
+
await this.persistSourceRunToolReceiptManifest({
|
|
10394
|
+
toolId,
|
|
10395
|
+
logicalCallId,
|
|
10396
|
+
executionAuthScopeDigest,
|
|
10397
|
+
resultReceiptKey,
|
|
10398
|
+
});
|
|
10399
|
+
},
|
|
9724
10400
|
...(toolRetryPolicy?.requiresExecutionFence === true
|
|
9725
10401
|
? {
|
|
9726
10402
|
executionLock: {
|
|
@@ -9898,9 +10574,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9898
10574
|
providerIdempotencyKeyBase,
|
|
9899
10575
|
cacheable: cacheableToolResult,
|
|
9900
10576
|
receiptKey: cacheableToolResult ? durableCacheKey : null,
|
|
9901
|
-
legacyReceiptKeys:
|
|
9902
|
-
|
|
9903
|
-
: [],
|
|
10577
|
+
legacyReceiptKeys: [
|
|
10578
|
+
...compatibleDurableCacheKeys.slice(1),
|
|
10579
|
+
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
10580
|
+
],
|
|
9904
10581
|
executionAuthScopeDigest,
|
|
9905
10582
|
requiresExecutionFence:
|
|
9906
10583
|
rowToolRetryPolicy?.requiresExecutionFence === true,
|
|
@@ -9968,12 +10645,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9968
10645
|
this.invalidateToolAuthScopeDigest(toolId);
|
|
9969
10646
|
executionAuthScopeDigest =
|
|
9970
10647
|
(await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
|
|
9971
|
-
|
|
9972
|
-
|
|
9973
|
-
|
|
9974
|
-
|
|
9975
|
-
|
|
9976
|
-
|
|
10648
|
+
compatibleDurableCacheKeys =
|
|
10649
|
+
await this.compatibleDurableToolCallCacheKeys({
|
|
10650
|
+
toolId,
|
|
10651
|
+
requestInput: input,
|
|
10652
|
+
executionAuthScopeDigest,
|
|
10653
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
10654
|
+
});
|
|
10655
|
+
durableCacheKey = compatibleDurableCacheKeys[0];
|
|
10656
|
+
if (!durableCacheKey) {
|
|
10657
|
+
throw new Error(
|
|
10658
|
+
`Tool ${toolId} is missing a primary durable cache key after auth refresh.`,
|
|
10659
|
+
);
|
|
10660
|
+
}
|
|
9977
10661
|
legacyDurableCacheKey = await this.legacyDurableToolCallCacheKey({
|
|
9978
10662
|
toolId,
|
|
9979
10663
|
requestInput: input,
|
|
@@ -9993,7 +10677,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9993
10677
|
checkpointCacheKeys.splice(
|
|
9994
10678
|
0,
|
|
9995
10679
|
checkpointCacheKeys.length,
|
|
9996
|
-
|
|
10680
|
+
...compatibleDurableCacheKeys,
|
|
9997
10681
|
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
9998
10682
|
);
|
|
9999
10683
|
}
|
|
@@ -10019,6 +10703,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10019
10703
|
workflowId: this.#options.workflowId,
|
|
10020
10704
|
orgId: this.#options.orgId,
|
|
10021
10705
|
executorToken: this.#options.executorToken,
|
|
10706
|
+
slackHitlDefaultTimeoutMs:
|
|
10707
|
+
this.currentRuntimeSemantics.slackHitl.defaultTimeoutMs,
|
|
10708
|
+
slackHitlTimeoutOutcome:
|
|
10709
|
+
this.currentRuntimeContract.slackHitlTimeoutOutcome,
|
|
10022
10710
|
};
|
|
10023
10711
|
const basePreparedBoundary = await handler.prepare({
|
|
10024
10712
|
payload: input,
|
|
@@ -10039,23 +10727,65 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10039
10727
|
}
|
|
10040
10728
|
: basePreparedBoundary;
|
|
10041
10729
|
|
|
10042
|
-
const existing =
|
|
10043
|
-
|
|
10730
|
+
const existing = assertKnownResolvedBoundary(
|
|
10731
|
+
preparedBoundary.boundaryId,
|
|
10732
|
+
this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId],
|
|
10733
|
+
);
|
|
10044
10734
|
if (existing?.kind === 'integration_event' && 'output' in existing) {
|
|
10045
10735
|
const output = recordOrNull(existing.output);
|
|
10046
10736
|
if (output?.timed_out === true) {
|
|
10047
|
-
|
|
10048
|
-
|
|
10049
|
-
|
|
10050
|
-
|
|
10737
|
+
// The runtime proxy is recreated for each worker invocation. Preserve
|
|
10738
|
+
// the policy that armed this boundary so its timeout finalizer and
|
|
10739
|
+
// parking behavior survive a durable replay.
|
|
10740
|
+
const timeoutPolicy =
|
|
10741
|
+
existing.timeoutPolicy ?? preparedBoundary.timeoutPolicy;
|
|
10742
|
+
// An explicit boundary policy may deliberately omit parking (the
|
|
10743
|
+
// new on_timeout: "return_null" escape hatch). Only fall back to the
|
|
10744
|
+
// handler's historical default when this is an old boundary with no
|
|
10745
|
+
// persisted policy at all.
|
|
10746
|
+
const suspendAfterTimeoutMs = timeoutPolicy
|
|
10747
|
+
? timeoutPolicy.suspendAfterTimeoutMs
|
|
10748
|
+
: handler.suspendAfterTimeoutMs;
|
|
10749
|
+
try {
|
|
10750
|
+
await handler.onTimeout?.({
|
|
10751
|
+
payload: input,
|
|
10752
|
+
context: eventContext,
|
|
10753
|
+
boundary: {
|
|
10754
|
+
boundaryId: preparedBoundary.boundaryId,
|
|
10755
|
+
eventKey: preparedBoundary.eventKey,
|
|
10756
|
+
timeoutMs: preparedBoundary.timeoutMs,
|
|
10757
|
+
provider: existing.provider ?? handler.provider,
|
|
10758
|
+
toolId: existing.toolId ?? handler.toolId,
|
|
10759
|
+
...(existing.messageRef
|
|
10760
|
+
? { messageRef: existing.messageRef }
|
|
10761
|
+
: {}),
|
|
10762
|
+
...(timeoutPolicy ? { timeoutPolicy } : {}),
|
|
10763
|
+
},
|
|
10764
|
+
});
|
|
10765
|
+
} catch (error) {
|
|
10766
|
+
this.recordSlackHitlTimeoutFailure(error);
|
|
10767
|
+
throw error;
|
|
10768
|
+
}
|
|
10769
|
+
if (timeoutPolicy?.timeoutOutcome === 'error') {
|
|
10770
|
+
// The provider finalizer above must complete first. That ordering
|
|
10771
|
+
// makes an expired Slack card and the terminal Play state agree,
|
|
10772
|
+
// while provider finalization failures stay retryable instead.
|
|
10773
|
+
const error = new SlackHitlTimeoutError(timeoutPolicy.timeoutError);
|
|
10774
|
+
this.recordSlackHitlTimeoutFailure(error);
|
|
10775
|
+
throw error;
|
|
10776
|
+
}
|
|
10777
|
+
if (suspendAfterTimeoutMs !== undefined) {
|
|
10778
|
+
this.runtimeLog(
|
|
10779
|
+
`Integration event ${preparedBoundary.boundaryId}: timeout finalized externally; keeping Play parked`,
|
|
10780
|
+
);
|
|
10781
|
+
this.suspendDurably({
|
|
10782
|
+
kind: 'integration_event',
|
|
10051
10783
|
boundaryId: preparedBoundary.boundaryId,
|
|
10052
10784
|
eventKey: preparedBoundary.eventKey,
|
|
10053
|
-
timeoutMs:
|
|
10054
|
-
provider: existing.provider ?? handler.provider,
|
|
10055
|
-
toolId: existing.toolId ?? handler.toolId,
|
|
10785
|
+
timeoutMs: suspendAfterTimeoutMs,
|
|
10056
10786
|
...(existing.messageRef ? { messageRef: existing.messageRef } : {}),
|
|
10057
|
-
}
|
|
10058
|
-
}
|
|
10787
|
+
});
|
|
10788
|
+
}
|
|
10059
10789
|
}
|
|
10060
10790
|
this.runtimeLog(
|
|
10061
10791
|
`Integration event ${preparedBoundary.boundaryId}: recovered response from checkpoint`,
|
|
@@ -10091,6 +10821,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10091
10821
|
timeoutMs: boundary.timeoutMs,
|
|
10092
10822
|
provider: boundary.provider,
|
|
10093
10823
|
toolId: boundary.toolId,
|
|
10824
|
+
...(boundary.timeoutPolicy
|
|
10825
|
+
? { timeoutPolicy: boundary.timeoutPolicy }
|
|
10826
|
+
: {}),
|
|
10094
10827
|
...(rowScope
|
|
10095
10828
|
? {
|
|
10096
10829
|
scope: {
|
|
@@ -10105,9 +10838,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10105
10838
|
...(boundary.messageRef ? { messageRef: boundary.messageRef } : {}),
|
|
10106
10839
|
},
|
|
10107
10840
|
};
|
|
10108
|
-
this.#options.onBatchComplete?.(this.checkpoint);
|
|
10109
|
-
|
|
10110
10841
|
if (rowScope) {
|
|
10842
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
10111
10843
|
throw new PlayRowExecutionSuspendedError({
|
|
10112
10844
|
boundaryId: boundary.boundaryId,
|
|
10113
10845
|
eventKey: boundary.eventKey,
|
|
@@ -10116,7 +10848,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10116
10848
|
});
|
|
10117
10849
|
}
|
|
10118
10850
|
|
|
10119
|
-
|
|
10851
|
+
this.suspendDurably({
|
|
10120
10852
|
kind: 'integration_event',
|
|
10121
10853
|
boundaryId: boundary.boundaryId,
|
|
10122
10854
|
eventKey: boundary.eventKey,
|
|
@@ -10131,6 +10863,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10131
10863
|
input: Record<string, unknown>,
|
|
10132
10864
|
options?: PlayCallOptions,
|
|
10133
10865
|
): Promise<TOutput> {
|
|
10866
|
+
this.assertNoPendingDurableSuspension();
|
|
10134
10867
|
if (
|
|
10135
10868
|
arguments.length === 3 &&
|
|
10136
10869
|
typeof playRef === 'object' &&
|
|
@@ -10140,19 +10873,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10140
10873
|
'ctx.runPlay(...) signature is ctx.runPlay(key, playRef, input, { description }). Add a stable call key as the first argument.',
|
|
10141
10874
|
);
|
|
10142
10875
|
}
|
|
10143
|
-
const resolvedName =
|
|
10144
|
-
typeof playRef === 'string'
|
|
10145
|
-
? playRef
|
|
10146
|
-
: playRef && typeof playRef.playName === 'string'
|
|
10147
|
-
? playRef.playName
|
|
10148
|
-
: playRef && typeof playRef.name === 'string'
|
|
10149
|
-
? playRef.name
|
|
10150
|
-
: '';
|
|
10876
|
+
const resolvedName = resolvePlayReferenceName(playRef);
|
|
10151
10877
|
|
|
10152
10878
|
if (!resolvedName.trim()) {
|
|
10153
10879
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
10154
10880
|
}
|
|
10155
|
-
|
|
10881
|
+
// Imported Play dependencies are callable proxy functions at runtime. The
|
|
10882
|
+
// authoring contract describes their durable identity, so validate the
|
|
10883
|
+
// normalized name rather than requiring the ephemeral handle itself to be
|
|
10884
|
+
// JSON-shaped.
|
|
10885
|
+
validatePlayAuthoringField('ctx.runPlay.playRef', resolvedName);
|
|
10156
10886
|
validatePlayAuthoringField('ctx.runPlay.input', input);
|
|
10157
10887
|
assertNoSecretTaint(input, 'ctx.runPlay input');
|
|
10158
10888
|
validatePlayAuthoringField('ctx.runPlay.key', key);
|
|
@@ -10258,12 +10988,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10258
10988
|
runtimeContract: resolvePlayArtifactRuntimeContract({
|
|
10259
10989
|
authoringContractEdition:
|
|
10260
10990
|
childCompatibility.authoringContractEdition,
|
|
10991
|
+
slackHitlTimeoutOutcome:
|
|
10992
|
+
childCompatibility.slackHitlTimeoutOutcome,
|
|
10261
10993
|
toolResponseContract: childToolResponseContract,
|
|
10262
10994
|
toolResponseReceiptRevision: childToolResponseReceiptRevision,
|
|
10263
10995
|
}),
|
|
10264
10996
|
},
|
|
10265
10997
|
() => this.executeResolvedPlay(resolvedPlay, this, input),
|
|
10266
10998
|
);
|
|
10999
|
+
// Inline composition shares this Play Invocation. A child may catch a
|
|
11000
|
+
// parent suspension, but it cannot publish child success while that
|
|
11001
|
+
// handoff is still pending.
|
|
11002
|
+
this.assertNoPendingDurableSuspension();
|
|
10267
11003
|
this.inlineChildAggregates.ok += 1;
|
|
10268
11004
|
if (rowStore) {
|
|
10269
11005
|
this.emitScopedFieldMetaUpdate({
|
|
@@ -10313,6 +11049,81 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10313
11049
|
}
|
|
10314
11050
|
}
|
|
10315
11051
|
|
|
11052
|
+
async runPlayAsync(
|
|
11053
|
+
key: string,
|
|
11054
|
+
playRef: string | { playName?: string; name?: string },
|
|
11055
|
+
input: Record<string, unknown>,
|
|
11056
|
+
options: PlayAsyncCallOptions,
|
|
11057
|
+
): Promise<AsyncPlayRunRef> {
|
|
11058
|
+
const resolvedName = resolvePlayReferenceName(playRef);
|
|
11059
|
+
if (!resolvedName.trim()) {
|
|
11060
|
+
throw new Error(
|
|
11061
|
+
'ctx.runPlayAsync(...) requires a resolvable Play name or definition handle.',
|
|
11062
|
+
);
|
|
11063
|
+
}
|
|
11064
|
+
validatePlayAuthoringField('ctx.runPlayAsync.playRef', resolvedName);
|
|
11065
|
+
validatePlayAuthoringField('ctx.runPlayAsync.input', input);
|
|
11066
|
+
assertNoSecretTaint(input, 'ctx.runPlayAsync input');
|
|
11067
|
+
validatePlayAuthoringField('ctx.runPlayAsync.key', key);
|
|
11068
|
+
validatePlayAuthoringField(
|
|
11069
|
+
'ctx.runPlayAsync.options.description',
|
|
11070
|
+
options?.description,
|
|
11071
|
+
);
|
|
11072
|
+
const normalizedKey = this.normalizeContextKey(key, 'runPlayAsync');
|
|
11073
|
+
if (!this.#options.launchPlayAsync) {
|
|
11074
|
+
throw new Error(
|
|
11075
|
+
'ctx.runPlayAsync(...) is unavailable because no async Play admission Adapter was configured.',
|
|
11076
|
+
);
|
|
11077
|
+
}
|
|
11078
|
+
const ancestryPlayIds =
|
|
11079
|
+
this.currentExecutionGovernor.snapshot().ancestryPlayIds;
|
|
11080
|
+
admitAsyncPlayLineage({
|
|
11081
|
+
ancestryPlayIds,
|
|
11082
|
+
childPlayName: resolvedName,
|
|
11083
|
+
});
|
|
11084
|
+
|
|
11085
|
+
const rowScope = rowContext.getStore();
|
|
11086
|
+
const invocationKey = sha256Hex(
|
|
11087
|
+
stableStringify({
|
|
11088
|
+
parentRunId: this.currentExecutionScope.logical.runId,
|
|
11089
|
+
callerScope: this.currentExecutionScope.receipt.namespace,
|
|
11090
|
+
key: normalizedKey,
|
|
11091
|
+
row: rowScope
|
|
11092
|
+
? {
|
|
11093
|
+
tableNamespace: rowScope.tableNamespace ?? null,
|
|
11094
|
+
rowKey: rowScope.rowKey ?? null,
|
|
11095
|
+
rowId: rowScope.rowKey ? null : rowScope.rowId,
|
|
11096
|
+
fieldName: rowScope.fieldName ?? null,
|
|
11097
|
+
}
|
|
11098
|
+
: null,
|
|
11099
|
+
}),
|
|
11100
|
+
);
|
|
11101
|
+
const launched = await this.#options.launchPlayAsync({
|
|
11102
|
+
key: normalizedKey,
|
|
11103
|
+
invocationKey,
|
|
11104
|
+
playRef: resolvedName,
|
|
11105
|
+
input,
|
|
11106
|
+
description: options.description,
|
|
11107
|
+
schedulerSchema: this.#options.runtimeSchedulerSchema ?? null,
|
|
11108
|
+
ancestryPlayIds,
|
|
11109
|
+
});
|
|
11110
|
+
const step = {
|
|
11111
|
+
type: 'play_launch' as const,
|
|
11112
|
+
playId: resolvedName,
|
|
11113
|
+
runId: launched.runId,
|
|
11114
|
+
description: normalizeStepDescription(options?.description),
|
|
11115
|
+
};
|
|
11116
|
+
if (this.activeDatasetStep) {
|
|
11117
|
+
this.activeDatasetStep.substeps.push(step);
|
|
11118
|
+
} else {
|
|
11119
|
+
this.steps.push(step);
|
|
11120
|
+
}
|
|
11121
|
+
this.log(
|
|
11122
|
+
`ctx.runPlayAsync(${normalizedKey}): admitted ${resolvedName} as ${launched.runId}`,
|
|
11123
|
+
);
|
|
11124
|
+
return launched;
|
|
11125
|
+
}
|
|
11126
|
+
|
|
10316
11127
|
/**
|
|
10317
11128
|
* Replay-stable receipt namespace for primitives executed by one inline
|
|
10318
11129
|
* `ctx.runPlay` invocation. Format
|
|
@@ -10577,7 +11388,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10577
11388
|
`sleep-${this.sleepBoundaryIndex}-${delayMs}`,
|
|
10578
11389
|
);
|
|
10579
11390
|
this.sleepBoundaryIndex += 1;
|
|
10580
|
-
const existing =
|
|
11391
|
+
const existing = assertKnownResolvedBoundary(
|
|
11392
|
+
boundaryId,
|
|
11393
|
+
this.checkpoint.resolvedBoundaries?.[boundaryId],
|
|
11394
|
+
);
|
|
10581
11395
|
if (existing?.kind === 'sleep' && existing.completedAt !== undefined) {
|
|
10582
11396
|
return;
|
|
10583
11397
|
}
|
|
@@ -10588,8 +11402,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10588
11402
|
delayMs,
|
|
10589
11403
|
},
|
|
10590
11404
|
};
|
|
10591
|
-
this
|
|
10592
|
-
throw new PlayExecutionSuspendedError({
|
|
11405
|
+
this.suspendDurably({
|
|
10593
11406
|
kind: 'sleep',
|
|
10594
11407
|
boundaryId,
|
|
10595
11408
|
delayMs,
|
|
@@ -10759,7 +11572,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10759
11572
|
)}`,
|
|
10760
11573
|
);
|
|
10761
11574
|
|
|
10762
|
-
const existing =
|
|
11575
|
+
const existing = assertKnownResolvedBoundary(
|
|
11576
|
+
boundaryId,
|
|
11577
|
+
this.checkpoint.resolvedBoundaries?.[boundaryId],
|
|
11578
|
+
);
|
|
10763
11579
|
// A transient request deliberately has no durable recovery path, so
|
|
10764
11580
|
// its response stays in the running Play rather than a checkpoint.
|
|
10765
11581
|
if (
|
|
@@ -10951,6 +11767,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10951
11767
|
options?.transient !== true &&
|
|
10952
11768
|
!this.durableDirectToolResultsBackedByReceipts
|
|
10953
11769
|
) {
|
|
11770
|
+
// Publishing a fetch boundary makes the result replay-visible.
|
|
11771
|
+
// A caught durable suspension is still the invocation's outcome,
|
|
11772
|
+
// so never checkpoint over a pending handoff.
|
|
11773
|
+
this.assertNoPendingDurableSuspension();
|
|
10954
11774
|
this.checkpoint.resolvedBoundaries = {
|
|
10955
11775
|
...(this.checkpoint.resolvedBoundaries ?? {}),
|
|
10956
11776
|
[boundaryId]: {
|
|
@@ -10998,7 +11818,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10998
11818
|
`step-${scope}:${normalizedKey}${boundarySuffix}`,
|
|
10999
11819
|
);
|
|
11000
11820
|
const executeStep = async (): Promise<T> => {
|
|
11001
|
-
const existing =
|
|
11821
|
+
const existing = assertKnownResolvedBoundary(
|
|
11822
|
+
boundaryId,
|
|
11823
|
+
this.checkpoint.resolvedBoundaries?.[boundaryId],
|
|
11824
|
+
);
|
|
11002
11825
|
if (existing?.kind === 'step' && 'output' in existing) {
|
|
11003
11826
|
this.runtimeLog(
|
|
11004
11827
|
`ctx.step(${normalizedKey}): recovered result from checkpoint`,
|
|
@@ -11008,6 +11831,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11008
11831
|
|
|
11009
11832
|
const output = await run();
|
|
11010
11833
|
assertJsonSerializableStepOutput(normalizedKey, output);
|
|
11834
|
+
// A caught durable suspension is still the invocation's outcome. Never
|
|
11835
|
+
// publish an enclosing step fallback that would make a later resume skip
|
|
11836
|
+
// the suspended work.
|
|
11837
|
+
this.assertNoPendingDurableSuspension();
|
|
11011
11838
|
this.checkpoint.resolvedBoundaries = {
|
|
11012
11839
|
...(this.checkpoint.resolvedBoundaries ?? {}),
|
|
11013
11840
|
[boundaryId]: {
|
|
@@ -11355,6 +12182,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11355
12182
|
output: serializeToolExecuteResult(receipt.result),
|
|
11356
12183
|
})),
|
|
11357
12184
|
);
|
|
12185
|
+
await this.persistSourceRunToolReceiptManifests(
|
|
12186
|
+
receipts.flatMap((receipt) => {
|
|
12187
|
+
const request = input.requestForReceipt.get(
|
|
12188
|
+
receipt.resultReceiptKey,
|
|
12189
|
+
);
|
|
12190
|
+
return request
|
|
12191
|
+
? [
|
|
12192
|
+
{
|
|
12193
|
+
toolId: input.toolId,
|
|
12194
|
+
logicalCallId: request.logicalCallId,
|
|
12195
|
+
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
12196
|
+
resultReceiptKey: receipt.resultReceiptKey,
|
|
12197
|
+
},
|
|
12198
|
+
]
|
|
12199
|
+
: [];
|
|
12200
|
+
}),
|
|
12201
|
+
);
|
|
11358
12202
|
return await recordsFor(
|
|
11359
12203
|
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
11360
12204
|
completed,
|
|
@@ -11566,19 +12410,38 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11566
12410
|
}
|
|
11567
12411
|
let releaseToolSlot: () => void = () => undefined;
|
|
11568
12412
|
try {
|
|
12413
|
+
const providerOperationKey = this.providerIdempotencyKeyForToolCall(
|
|
12414
|
+
{
|
|
12415
|
+
operationKey:
|
|
12416
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
12417
|
+
force: request.force === true,
|
|
12418
|
+
logicalCallId: request.logicalCallId,
|
|
12419
|
+
},
|
|
12420
|
+
);
|
|
11569
12421
|
const result = await this.executeMappedToolCallThroughModule({
|
|
11570
12422
|
request,
|
|
11571
12423
|
dispatchToolId: toolId,
|
|
11572
12424
|
dispatchPayload: request.input,
|
|
11573
12425
|
resultReceiptKey: request.receiptKey!,
|
|
11574
|
-
providerOperationKey
|
|
11575
|
-
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
12426
|
+
providerOperationKey,
|
|
11576
12427
|
resultReceipts: receipts,
|
|
11577
12428
|
apiOptions: {
|
|
11578
12429
|
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
11579
|
-
durableCallReceiptKey:
|
|
12430
|
+
durableCallReceiptKey: this.toolTransportReceiptKey({
|
|
12431
|
+
toolId,
|
|
12432
|
+
resultReceiptKey: request.receiptKey!,
|
|
12433
|
+
providerOperationKey,
|
|
12434
|
+
force: request.force,
|
|
12435
|
+
}),
|
|
11580
12436
|
completedResultCache: true,
|
|
11581
12437
|
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
12438
|
+
...this.providerForceRefreshOptionsForToolCall({
|
|
12439
|
+
operationKey:
|
|
12440
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
12441
|
+
force: request.force,
|
|
12442
|
+
requiresExecutionFence: request.requiresExecutionFence,
|
|
12443
|
+
logicalCallId: request.logicalCallId,
|
|
12444
|
+
}),
|
|
11582
12445
|
timeoutMs:
|
|
11583
12446
|
typeof request.timeoutMs === 'number' && request.timeoutMs > 0
|
|
11584
12447
|
? request.timeoutMs
|
|
@@ -11618,7 +12481,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11618
12481
|
if (requests.length === 0) return [];
|
|
11619
12482
|
|
|
11620
12483
|
const requestForReceipt = new Map<string, ToolCallRequest>();
|
|
11621
|
-
const requestByCallId = new Map<string, ToolCallRequest>();
|
|
11622
12484
|
for (const request of requests) {
|
|
11623
12485
|
const receiptKey = request.receiptKey?.trim();
|
|
11624
12486
|
if (!receiptKey) {
|
|
@@ -11627,13 +12489,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11627
12489
|
);
|
|
11628
12490
|
}
|
|
11629
12491
|
requestForReceipt.set(receiptKey, request);
|
|
11630
|
-
requestByCallId.set(request.callId, request);
|
|
11631
12492
|
}
|
|
11632
12493
|
const store = this.createRuntimeStepToolCallReceiptCohortStore({
|
|
11633
12494
|
toolId,
|
|
11634
12495
|
requestForReceipt,
|
|
11635
12496
|
});
|
|
11636
|
-
const
|
|
12497
|
+
const plannedBatches = compileRequestsWithStrategy({
|
|
11637
12498
|
requests,
|
|
11638
12499
|
strategy,
|
|
11639
12500
|
getPayload: (request: ToolCallRequest) => request.input,
|
|
@@ -11656,9 +12517,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11656
12517
|
const batchParallelismCeiling =
|
|
11657
12518
|
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
11658
12519
|
const batchSize =
|
|
11659
|
-
|
|
12520
|
+
plannedBatches.length > 0
|
|
11660
12521
|
? await this.resourceGovernor.suggestedToolParallelism(
|
|
11661
|
-
|
|
12522
|
+
plannedBatches[0]!.batchOperation,
|
|
11662
12523
|
batchParallelismCeiling,
|
|
11663
12524
|
)
|
|
11664
12525
|
: batchParallelismCeiling;
|
|
@@ -11697,27 +12558,36 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11697
12558
|
};
|
|
11698
12559
|
|
|
11699
12560
|
await executeChunkedRequests({
|
|
11700
|
-
requests:
|
|
12561
|
+
requests: plannedBatches,
|
|
11701
12562
|
batchSize,
|
|
11702
|
-
execute: async (
|
|
12563
|
+
execute: async (plannedBatch) => {
|
|
11703
12564
|
if (this.persistenceLatch.tripped) {
|
|
11704
12565
|
this.persistenceLatch.preventedCallCount +=
|
|
11705
|
-
|
|
12566
|
+
plannedBatch.memberRequests.length;
|
|
11706
12567
|
throw new RuntimePersistenceCircuitOpenError(this.persistenceLatch);
|
|
11707
12568
|
}
|
|
11708
|
-
const
|
|
11709
|
-
|
|
12569
|
+
const requestByCallId = new Map(
|
|
12570
|
+
plannedBatch.memberRequests.map((request) => [
|
|
12571
|
+
request.callId,
|
|
12572
|
+
request,
|
|
12573
|
+
]),
|
|
11710
12574
|
);
|
|
11711
|
-
|
|
12575
|
+
type NativeBatchPhysicalDispatchDetails = {
|
|
12576
|
+
members: ToolCallRequest[];
|
|
12577
|
+
compiled: CompiledRequestBatch<ToolCallRequest>;
|
|
12578
|
+
providerOperationKey: string;
|
|
12579
|
+
forceRefreshIdentities: string[];
|
|
12580
|
+
receiptLeaseExpiresAt: string | null;
|
|
12581
|
+
timeoutMs: number | undefined;
|
|
12582
|
+
};
|
|
12583
|
+
const nativeBatchPhysicalDispatchByAggregateReceipt = new Map<
|
|
11712
12584
|
string,
|
|
11713
|
-
|
|
11714
|
-
members: ToolCallRequest[];
|
|
11715
|
-
compiled: CompiledRequestBatch<ToolCallRequest>;
|
|
11716
|
-
providerOperationKey: string;
|
|
11717
|
-
receiptLeaseExpiresAt: string | null;
|
|
11718
|
-
timeoutMs: number | undefined;
|
|
11719
|
-
}
|
|
12585
|
+
NativeBatchPhysicalDispatchDetails
|
|
11720
12586
|
>();
|
|
12587
|
+
const aggregateReceiptKeyByOwnedMemberIds = new Map<string, string>();
|
|
12588
|
+
const ownedMemberIdsKey = (
|
|
12589
|
+
members: ReadonlyArray<{ memberId: string }>,
|
|
12590
|
+
): string => JSON.stringify(members.map((member) => member.memberId));
|
|
11721
12591
|
const physicalToolCall = {
|
|
11722
12592
|
call: async (
|
|
11723
12593
|
physical: Parameters<
|
|
@@ -11730,7 +12600,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11730
12600
|
'Native batch physical Tool Call requires compatibility identities.',
|
|
11731
12601
|
);
|
|
11732
12602
|
}
|
|
11733
|
-
const detail =
|
|
12603
|
+
const detail = nativeBatchPhysicalDispatchByAggregateReceipt.get(
|
|
12604
|
+
compatibility.resultReceiptKey,
|
|
12605
|
+
);
|
|
11734
12606
|
if (!detail) {
|
|
11735
12607
|
throw new Error(
|
|
11736
12608
|
'Native batch physical Tool Call lost its dispatch details.',
|
|
@@ -11752,7 +12624,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11752
12624
|
call.tool.id,
|
|
11753
12625
|
call.payload,
|
|
11754
12626
|
{
|
|
11755
|
-
durableCallReceiptKey:
|
|
12627
|
+
durableCallReceiptKey: this.toolTransportReceiptKey({
|
|
12628
|
+
toolId: call.tool.id,
|
|
12629
|
+
resultReceiptKey: compatibility.resultReceiptKey,
|
|
12630
|
+
providerOperationKey: detail.providerOperationKey,
|
|
12631
|
+
force: detail.members.some(
|
|
12632
|
+
(request) => request.force === true,
|
|
12633
|
+
),
|
|
12634
|
+
}),
|
|
11756
12635
|
completedResultCache: true,
|
|
11757
12636
|
playNodeScope: playNodeScopeForToolCallRequest(
|
|
11758
12637
|
detail.members[0]!,
|
|
@@ -11760,6 +12639,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11760
12639
|
),
|
|
11761
12640
|
executionAuthScopeDigest:
|
|
11762
12641
|
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
12642
|
+
...(detail.forceRefreshIdentities.length > 0
|
|
12643
|
+
? {
|
|
12644
|
+
providerOperationForceRefreshIdentities:
|
|
12645
|
+
detail.forceRefreshIdentities,
|
|
12646
|
+
}
|
|
12647
|
+
: {}),
|
|
11763
12648
|
receiptLeaseExpiresAt: detail.receiptLeaseExpiresAt,
|
|
11764
12649
|
timeoutMs: detail.timeoutMs,
|
|
11765
12650
|
},
|
|
@@ -11793,7 +12678,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11793
12678
|
runId: this.currentReceiptOwnerRunId,
|
|
11794
12679
|
attempt: this.currentRunAttempt,
|
|
11795
12680
|
},
|
|
11796
|
-
members:
|
|
12681
|
+
members: plannedBatch.memberRequests.map((request) => ({
|
|
11797
12682
|
memberId: request.callId,
|
|
11798
12683
|
forceFailedRefresh: request.forceFailedRefresh === true,
|
|
11799
12684
|
waitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
@@ -11831,53 +12716,92 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11831
12716
|
},
|
|
11832
12717
|
},
|
|
11833
12718
|
})),
|
|
11834
|
-
buildPhysicalCall: ({ members }) => {
|
|
11835
|
-
|
|
11836
|
-
|
|
12719
|
+
buildPhysicalCall: async ({ members: ownedMembers }) => {
|
|
12720
|
+
// `ownedMembers` are the rows we still need to execute.
|
|
12721
|
+
const ownedMemberRequests = ownedMembers.map(
|
|
12722
|
+
(member) => requestByCallId.get(member.memberId)!,
|
|
12723
|
+
);
|
|
12724
|
+
const compiled = compileOwnedPhysicalBatch(ownedMemberRequests);
|
|
12725
|
+
const receiptKeys = ownedMemberRequests.map(
|
|
12726
|
+
(request) => request.receiptKey!,
|
|
11837
12727
|
);
|
|
11838
|
-
const receiptKeys = owned.map((request) => request.receiptKey!);
|
|
11839
12728
|
const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
|
|
11840
12729
|
receiptKeys,
|
|
11841
12730
|
prefix: 'batch',
|
|
11842
12731
|
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
11843
12732
|
orgId: this.#options.orgId,
|
|
11844
|
-
toolId:
|
|
12733
|
+
toolId: compiled.batchOperation,
|
|
11845
12734
|
}),
|
|
11846
12735
|
});
|
|
11847
|
-
const
|
|
11848
|
-
|
|
11849
|
-
|
|
11850
|
-
|
|
11851
|
-
|
|
11852
|
-
|
|
11853
|
-
const
|
|
11854
|
-
|
|
11855
|
-
|
|
11856
|
-
|
|
11857
|
-
|
|
11858
|
-
|
|
11859
|
-
|
|
12736
|
+
const providerActionVersion =
|
|
12737
|
+
(
|
|
12738
|
+
await this.#options.getToolActionCacheVersion?.(
|
|
12739
|
+
compiled.batchOperation,
|
|
12740
|
+
)
|
|
12741
|
+
)?.trim() ?? '';
|
|
12742
|
+
const forceRefreshIdentities = ownedMemberRequests.flatMap(
|
|
12743
|
+
(request) =>
|
|
12744
|
+
this.providerForceRefreshIdentitiesForToolCall({
|
|
12745
|
+
operationKey:
|
|
12746
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
12747
|
+
force: request.force,
|
|
12748
|
+
requiresExecutionFence: request.requiresExecutionFence,
|
|
12749
|
+
logicalCallId: request.logicalCallId,
|
|
12750
|
+
}),
|
|
12751
|
+
);
|
|
12752
|
+
const providerOperationKey =
|
|
12753
|
+
buildDurableToolPayloadProviderIdempotencyKey({
|
|
12754
|
+
orgId: this.#options.orgId,
|
|
12755
|
+
toolId: compiled.batchOperation,
|
|
12756
|
+
executionAuthScopeDigest:
|
|
12757
|
+
ownedMemberRequests[0]?.executionAuthScopeDigest ?? null,
|
|
12758
|
+
providerActionVersion,
|
|
12759
|
+
payload: compiled.batchPayload,
|
|
12760
|
+
forceRefreshIdentities,
|
|
12761
|
+
});
|
|
12762
|
+
const aggregateDispatchDetails: NativeBatchPhysicalDispatchDetails =
|
|
12763
|
+
{
|
|
12764
|
+
members: ownedMemberRequests,
|
|
12765
|
+
compiled,
|
|
12766
|
+
providerOperationKey,
|
|
12767
|
+
forceRefreshIdentities,
|
|
12768
|
+
receiptLeaseExpiresAt: ownedMemberRequests.reduce<
|
|
12769
|
+
string | null
|
|
12770
|
+
>((earliest, request) => {
|
|
11860
12771
|
const expiresAt = request.receiptLeaseExpiresAt ?? null;
|
|
11861
12772
|
if (!expiresAt) return earliest;
|
|
11862
12773
|
if (!earliest) return expiresAt;
|
|
11863
12774
|
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
11864
12775
|
? expiresAt
|
|
11865
12776
|
: earliest;
|
|
11866
|
-
},
|
|
11867
|
-
|
|
11868
|
-
|
|
11869
|
-
|
|
11870
|
-
|
|
11871
|
-
|
|
11872
|
-
|
|
11873
|
-
|
|
11874
|
-
|
|
11875
|
-
|
|
11876
|
-
|
|
11877
|
-
|
|
12777
|
+
}, null),
|
|
12778
|
+
timeoutMs: ownedMemberRequests.every(
|
|
12779
|
+
(request) =>
|
|
12780
|
+
typeof request.timeoutMs === 'number' &&
|
|
12781
|
+
Number.isFinite(request.timeoutMs) &&
|
|
12782
|
+
request.timeoutMs > 0,
|
|
12783
|
+
)
|
|
12784
|
+
? Math.max(
|
|
12785
|
+
...ownedMemberRequests.map(
|
|
12786
|
+
(request) => request.timeoutMs!,
|
|
12787
|
+
),
|
|
12788
|
+
)
|
|
12789
|
+
: undefined,
|
|
12790
|
+
};
|
|
12791
|
+
nativeBatchPhysicalDispatchByAggregateReceipt.set(
|
|
12792
|
+
aggregateReceiptKey,
|
|
12793
|
+
aggregateDispatchDetails,
|
|
12794
|
+
);
|
|
12795
|
+
aggregateReceiptKeyByOwnedMemberIds.set(
|
|
12796
|
+
ownedMemberIdsKey(ownedMembers),
|
|
12797
|
+
aggregateReceiptKey,
|
|
12798
|
+
);
|
|
11878
12799
|
return {
|
|
11879
|
-
...
|
|
11880
|
-
tool: {
|
|
12800
|
+
...ownedMembers[0]!.call,
|
|
12801
|
+
tool: {
|
|
12802
|
+
id: compiled.batchOperation,
|
|
12803
|
+
providerActionVersion,
|
|
12804
|
+
},
|
|
11881
12805
|
payload: compiled.batchPayload,
|
|
11882
12806
|
compatibility: {
|
|
11883
12807
|
resultReceiptKey: aggregateReceiptKey,
|
|
@@ -11885,34 +12809,29 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11885
12809
|
},
|
|
11886
12810
|
};
|
|
11887
12811
|
},
|
|
11888
|
-
splitResult: async ({ result: aggregate, members }) => {
|
|
11889
|
-
const
|
|
11890
|
-
|
|
11891
|
-
|
|
11892
|
-
|
|
11893
|
-
|
|
11894
|
-
|
|
11895
|
-
|
|
11896
|
-
|
|
11897
|
-
|
|
11898
|
-
|
|
11899
|
-
|
|
11900
|
-
|
|
11901
|
-
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
11902
|
-
orgId: this.#options.orgId,
|
|
11903
|
-
toolId: batch.batchOperation,
|
|
11904
|
-
}),
|
|
11905
|
-
});
|
|
11906
|
-
const detail = dispatchDetails.get(aggregateReceiptKey);
|
|
12812
|
+
splitResult: async ({ result: aggregate, members: ownedMembers }) => {
|
|
12813
|
+
const aggregateReceiptKey = aggregateReceiptKeyByOwnedMemberIds.get(
|
|
12814
|
+
ownedMemberIdsKey(ownedMembers),
|
|
12815
|
+
);
|
|
12816
|
+
if (!aggregateReceiptKey) {
|
|
12817
|
+
throw new Error(
|
|
12818
|
+
'Native batch Tool Call lost its physical aggregate receipt identity.',
|
|
12819
|
+
);
|
|
12820
|
+
}
|
|
12821
|
+
const detail =
|
|
12822
|
+
nativeBatchPhysicalDispatchByAggregateReceipt.get(
|
|
12823
|
+
aggregateReceiptKey,
|
|
12824
|
+
);
|
|
11907
12825
|
if (!detail) {
|
|
11908
12826
|
throw new Error(
|
|
11909
12827
|
'Native batch Tool Call lost its split compilation details.',
|
|
11910
12828
|
);
|
|
11911
12829
|
}
|
|
11912
12830
|
if (
|
|
11913
|
-
detail.members.length !==
|
|
12831
|
+
detail.members.length !== ownedMembers.length ||
|
|
11914
12832
|
!detail.members.every(
|
|
11915
|
-
(request, index) =>
|
|
12833
|
+
(request, index) =>
|
|
12834
|
+
request.callId === ownedMembers[index]?.memberId,
|
|
11916
12835
|
)
|
|
11917
12836
|
) {
|
|
11918
12837
|
throw new Error(
|
|
@@ -11923,8 +12842,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11923
12842
|
legacyResultForBatchSplitter(aggregate),
|
|
11924
12843
|
);
|
|
11925
12844
|
return await Promise.all(
|
|
11926
|
-
|
|
11927
|
-
const request =
|
|
12845
|
+
ownedMembers.map(async (member, index) => {
|
|
12846
|
+
const request = requestByCallId.get(member.memberId);
|
|
11928
12847
|
if (!request) {
|
|
11929
12848
|
throw new Error(
|
|
11930
12849
|
`Native batch Tool Call lost member ${member.memberId}.`,
|
|
@@ -11954,7 +12873,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11954
12873
|
);
|
|
11955
12874
|
},
|
|
11956
12875
|
});
|
|
11957
|
-
return { batch, result };
|
|
12876
|
+
return { batch: plannedBatch, result };
|
|
11958
12877
|
},
|
|
11959
12878
|
onChunkComplete: async (entries) => {
|
|
11960
12879
|
for (const entry of entries) {
|
|
@@ -11964,6 +12883,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11964
12883
|
for (const request of entry.request.memberRequests) {
|
|
11965
12884
|
await reject(request, entry.error);
|
|
11966
12885
|
}
|
|
12886
|
+
// A provider-unavailable physical batch is a logical outcome for
|
|
12887
|
+
// each member, not a map-fatal scheduler failure. Each rejected
|
|
12888
|
+
// row resolver now receives the typed error above, where a
|
|
12889
|
+
// waterfall may persist the failed leg and advance to its next
|
|
12890
|
+
// provider. Keep infrastructure and contract failures loud.
|
|
12891
|
+
if (
|
|
12892
|
+
isProviderExhaustedFailure(entry.error) ||
|
|
12893
|
+
isProviderUnavailable(entry.error)
|
|
12894
|
+
) {
|
|
12895
|
+
continue;
|
|
12896
|
+
}
|
|
11967
12897
|
throw entry.error;
|
|
11968
12898
|
}
|
|
11969
12899
|
if (!entry.result) {
|
|
@@ -12618,6 +13548,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12618
13548
|
force: owner.force === true,
|
|
12619
13549
|
logicalCallId: owner.logicalCallId,
|
|
12620
13550
|
}),
|
|
13551
|
+
...this.providerForceRefreshOptionsForToolCall({
|
|
13552
|
+
operationKey:
|
|
13553
|
+
this.providerOperationKeyForQueuedToolCall(owner),
|
|
13554
|
+
force: owner.force,
|
|
13555
|
+
requiresExecutionFence: owner.requiresExecutionFence,
|
|
13556
|
+
logicalCallId: owner.logicalCallId,
|
|
13557
|
+
}),
|
|
12621
13558
|
timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
|
|
12622
13559
|
owner,
|
|
12623
13560
|
]),
|
|
@@ -13232,6 +14169,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13232
14169
|
force: request.force === true,
|
|
13233
14170
|
logicalCallId: request.logicalCallId,
|
|
13234
14171
|
}),
|
|
14172
|
+
...this.providerForceRefreshOptionsForToolCall({
|
|
14173
|
+
operationKey:
|
|
14174
|
+
this.providerOperationKeyForQueuedToolCall(
|
|
14175
|
+
request,
|
|
14176
|
+
),
|
|
14177
|
+
force: request.force,
|
|
14178
|
+
requiresExecutionFence:
|
|
14179
|
+
request.requiresExecutionFence,
|
|
14180
|
+
logicalCallId: request.logicalCallId,
|
|
14181
|
+
}),
|
|
13235
14182
|
receiptLeaseExpiresAt:
|
|
13236
14183
|
request.receiptLeaseExpiresAt,
|
|
13237
14184
|
heartbeatReceipt: () =>
|
|
@@ -13440,9 +14387,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13440
14387
|
const asyncContract = options?.skipAsyncOperationResolution
|
|
13441
14388
|
? null
|
|
13442
14389
|
: ((await this.#options.getToolAsyncOperation?.(toolId)) ?? null);
|
|
14390
|
+
const asyncOperationPoll = options?.skipAsyncOperationResolution
|
|
14391
|
+
? null
|
|
14392
|
+
: ((await this.#options.getToolAsyncOperationPoll?.(toolId)) ?? null);
|
|
13443
14393
|
const shouldWaitForAsyncOperation = Boolean(
|
|
13444
14394
|
asyncContract && asyncOperationShouldWait(asyncContract, input),
|
|
13445
14395
|
);
|
|
14396
|
+
// An explicitly opted-in caller-managed launch gives the Play author the
|
|
14397
|
+
// provider handle and leaves polling to the authored graph. Do not let a
|
|
14398
|
+
// provider's legacy synchronous helper turn a wide start wave into
|
|
14399
|
+
// long-lived HTTP calls before that graph can make its own poll.
|
|
14400
|
+
const shouldPreferAsyncProviderExecution =
|
|
14401
|
+
asyncContract?.wait.kind === 'caller_managed' &&
|
|
14402
|
+
asyncContract.lifecycle.startAction === toolId &&
|
|
14403
|
+
asyncContract.launch?.preferAsync === true;
|
|
13446
14404
|
const executionAuthScopeDigest = options?.executionAuthScopeDigest
|
|
13447
14405
|
? options.executionAuthScopeDigest.trim() || null
|
|
13448
14406
|
: ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null);
|
|
@@ -13635,13 +14593,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13635
14593
|
`Tool ${toolId} requires a provider execution fence but no provider operation identity was supplied.`,
|
|
13636
14594
|
);
|
|
13637
14595
|
}
|
|
13638
|
-
const canonicalOperationActionVersion =
|
|
13639
|
-
|
|
13640
|
-
|
|
13641
|
-
|
|
13642
|
-
|
|
14596
|
+
const canonicalOperationActionVersion =
|
|
14597
|
+
providerExecutionFenceKey ||
|
|
14598
|
+
options?.providerOperationForceRefreshIdentities?.length
|
|
14599
|
+
? (
|
|
14600
|
+
await this.#options.getToolActionCacheVersion?.(toolId)
|
|
14601
|
+
)?.trim() || null
|
|
14602
|
+
: null;
|
|
13643
14603
|
const canonicalOperation =
|
|
13644
|
-
providerExecutionFenceKey
|
|
14604
|
+
(providerExecutionFenceKey ||
|
|
14605
|
+
options?.providerOperationForceRefreshIdentities?.length) &&
|
|
14606
|
+
canonicalOperationActionVersion
|
|
13645
14607
|
? {
|
|
13646
14608
|
version: 1,
|
|
13647
14609
|
tool_id: toolId,
|
|
@@ -13672,6 +14634,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13672
14634
|
...(shouldWaitForAsyncOperation
|
|
13673
14635
|
? { async_operation_wait: true }
|
|
13674
14636
|
: {}),
|
|
14637
|
+
...(shouldPreferAsyncProviderExecution
|
|
14638
|
+
? { prefer_async: true }
|
|
14639
|
+
: {}),
|
|
14640
|
+
...(asyncOperationPoll
|
|
14641
|
+
? {
|
|
14642
|
+
async_operation_poll_timeout_ms:
|
|
14643
|
+
asyncOperationPoll.checkTimeoutMs,
|
|
14644
|
+
}
|
|
14645
|
+
: {}),
|
|
13675
14646
|
...(usesDurableInvocationFence
|
|
13676
14647
|
? {
|
|
13677
14648
|
invocation_fence_version:
|
|
@@ -14242,14 +15213,36 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14242
15213
|
toolId,
|
|
14243
15214
|
failure.backpressureDelayMs,
|
|
14244
15215
|
);
|
|
15216
|
+
// A Retry-After beyond the durable pacer's maximum wait has
|
|
15217
|
+
// no useful in-process retry. It is already an exhausted
|
|
15218
|
+
// provider outcome: surface the same typed error the next
|
|
15219
|
+
// provider admission would produce, without sleeping before a
|
|
15220
|
+
// waterfall can choose an independent fallback.
|
|
15221
|
+
if (
|
|
15222
|
+
failure.backpressureDelayMs > PROVIDER_EXHAUSTED_MAX_WAIT_MS
|
|
15223
|
+
) {
|
|
15224
|
+
throw new ProviderExhaustedError({
|
|
15225
|
+
provider,
|
|
15226
|
+
retryAtMs: Date.now() + failure.backpressureDelayMs,
|
|
15227
|
+
cause: failure.error,
|
|
15228
|
+
});
|
|
15229
|
+
}
|
|
14245
15230
|
}
|
|
14246
15231
|
if (
|
|
14247
15232
|
failure.shouldRetry &&
|
|
14248
|
-
(
|
|
15233
|
+
(failure.knownNotDispatched ||
|
|
15234
|
+
failure.reason === 'gateway_invocation_in_progress' ||
|
|
15235
|
+
ambiguousTransportReplayAllowed ||
|
|
14249
15236
|
retrySafeTransientHttp ||
|
|
14250
15237
|
fixtureReplaySafe)
|
|
14251
15238
|
) {
|
|
14252
|
-
|
|
15239
|
+
// A gateway follower is retrying transport delivery to the
|
|
15240
|
+
// same durable invocation. Advancing the logical attempt here
|
|
15241
|
+
// would mint a new invocation key while the original owner is
|
|
15242
|
+
// still running, defeating the gateway's replay fence.
|
|
15243
|
+
if (failure.reason !== 'gateway_invocation_in_progress') {
|
|
15244
|
+
invocationAttempt += 1;
|
|
15245
|
+
}
|
|
14253
15246
|
if (failure.chargeRetryBudget) {
|
|
14254
15247
|
await this.governor.chargeBudget('retry');
|
|
14255
15248
|
}
|