deepline 0.3.89 → 0.3.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/contracts/install-commands.json +29 -0
- package/dist/bundling-sources/sdk/src/client.ts +120 -37
- package/dist/bundling-sources/sdk/src/index.ts +4 -0
- package/dist/bundling-sources/sdk/src/pagination.ts +25 -0
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
- package/dist/bundling-sources/sdk/src/release.ts +16 -9
- package/dist/bundling-sources/sdk/src/types.ts +23 -0
- package/dist/bundling-sources/shared_libs/monitors/analytics-public.ts +10 -0
- package/dist/bundling-sources/shared_libs/monitors/http-error-response.ts +3 -0
- package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +4 -4
- package/dist/bundling-sources/shared_libs/monitors/inventory-reconciliation.ts +59 -0
- package/dist/bundling-sources/shared_libs/monitors/lifecycle-notifications.ts +7 -9
- package/dist/bundling-sources/shared_libs/monitors/package.json +7 -0
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +44 -2
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-consumer-health.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-identity-claims.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-public-aliases.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts +245 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-message.ts +38 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-ack-probe.ts +94 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-delivery.ts +60 -0
- package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue.ts +60 -0
- package/dist/bundling-sources/shared_libs/monitors/snitcher-install-validation.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/snitcher-ip2company.ts +2 -2
- package/dist/bundling-sources/shared_libs/monitors/snitcher-radar.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/tracking-provider-integrations.ts +1 -1
- package/dist/bundling-sources/shared_libs/observability/build-context.ts +13 -0
- package/dist/bundling-sources/shared_libs/observability/log-metadata.ts +160 -0
- package/dist/bundling-sources/shared_libs/observability/request-context.ts +68 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +654 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +35 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/neon-constants.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +272 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/async-play-invocation.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/bounceban-batching.ts +137 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +779 -140
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +46 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-payload-contract.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +725 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +442 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-protocol.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +66 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/ports.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/postgres-store.ts +170 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +1322 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/lane-retirement-gate.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +112 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/perf-trace.ts +127 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-artifact-runtime-contract.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +24 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/job.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/ports.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/contracts.ts +114 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/engine-suspension.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/index.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/job.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/ports.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/postgres-fence-adapter.ts +263 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/contracts.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/index.ts +15 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/job.ts +65 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/ports.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/index.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts +142 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/ports.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/postgres-adapter.ts +102 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +60 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/index.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +5 -8
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +3 -5
- package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +168 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +72 -9
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +6 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +14 -44
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +57 -205
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts +201 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-env.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +67 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-release-environment.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-contract.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +14 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-run-task.ts +68 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts +269 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-suffix.ts +73 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +6642 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +897 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-metrics.ts +120 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +76 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +476 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +147 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/index.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/lazy.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/play-run-recovery.ts +98 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts +260 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +1416 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +1255 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +942 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +14636 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +888 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +355 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +46 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runner-push.ts +60 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-api-origin.ts +33 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-gateway-health.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts +835 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts +1218 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-operation.ts +207 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-scheduler-schema.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts +123 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-launch-read-observability.ts +78 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +169 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/terminal-contract.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-queue-metrics.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +249 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +296 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +20 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +60 -15
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +14 -1
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +165 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +15 -23
- package/dist/bundling-sources/shared_libs/play-runtime/workflow-executor-auth.ts +169 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +122 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/deployment-status.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/docflow.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/play-grep-search.ts +39 -0
- package/dist/bundling-sources/shared_libs/plays/share/copy.ts +271 -0
- package/dist/bundling-sources/shared_libs/plays/share/metrics.ts +140 -0
- package/dist/bundling-sources/shared_libs/plays/share/public-read-model.ts +86 -0
- package/dist/bundling-sources/shared_libs/plays/share/seo-quality.ts +181 -0
- package/dist/bundling-sources/shared_libs/plays/share/static-pipeline-summary.ts +658 -0
- package/dist/bundling-sources/shared_libs/plays/share/types.ts +197 -0
- package/dist/bundling-sources/shared_libs/plays/slack-hitl-timeout-error.ts +37 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +6 -1
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/product-notifications/configuration.ts +7 -1
- package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +3 -2
- package/dist/bundling-sources/shared_libs/product-notifications/overflow.ts +18 -0
- package/dist/bundling-sources/shared_libs/product-notifications/rolling-limit.ts +24 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-delivery.ts +624 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-payload.ts +140 -0
- package/dist/bundling-sources/shared_libs/product-notifications/slack-transport.ts +75 -0
- package/dist/bundling-sources/shared_libs/runtime-env.ts +6 -0
- package/dist/bundling-sources/shared_libs/security/email-blacklist.ts +42 -0
- package/dist/bundling-sources/shared_libs/security/header-safe-env.ts +10 -3
- package/dist/bundling-sources/shared_libs/security/slack-sanitize.ts +133 -0
- package/dist/cli/getting-started.d.mts +52 -0
- package/dist/cli/getting-started.d.ts +52 -0
- package/dist/cli/getting-started.js +65 -0
- package/dist/cli/getting-started.mjs +36 -0
- package/dist/cli/index.d.mts +9 -0
- package/dist/cli/index.d.ts +9 -0
- package/dist/cli/index.js +545 -264
- package/dist/cli/index.mjs +543 -271
- package/dist/{compiler-manifest-DsLbQaew.d.mts → compiler-manifest-BaJfE-5H.d.mts} +5 -3
- package/dist/{compiler-manifest-DsLbQaew.d.ts → compiler-manifest-BaJfE-5H.d.ts} +5 -3
- package/dist/index.d.mts +81 -148
- package/dist/index.d.ts +81 -148
- package/dist/index.js +83 -26
- package/dist/index.mjs +83 -26
- package/dist/install-integrity.json +130 -2
- package/dist/monitor-fleet-contract.d.mts +138 -0
- package/dist/monitor-fleet-contract.d.ts +138 -0
- package/dist/monitor-fleet-contract.js +404 -0
- package/dist/monitor-fleet-contract.mjs +367 -0
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/release.d.mts +255 -0
- package/dist/release.d.ts +255 -0
- package/dist/release.js +215 -0
- package/dist/release.mjs +188 -0
- package/package.json +21 -4
|
@@ -126,6 +126,10 @@ import {
|
|
|
126
126
|
type ToolExecutionErrorSchemaVersion,
|
|
127
127
|
type ToolExecutionFailureV1,
|
|
128
128
|
} from '../plays/tool-execution-error';
|
|
129
|
+
import {
|
|
130
|
+
SLACK_HITL_TIMEOUT_ERROR_CODE,
|
|
131
|
+
SlackHitlTimeoutError,
|
|
132
|
+
} from '../plays/slack-hitl-timeout-error';
|
|
129
133
|
import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
|
|
130
134
|
import { pacingPolicyForTool } from './pacing';
|
|
131
135
|
import {
|
|
@@ -181,6 +185,7 @@ import {
|
|
|
181
185
|
buildDurableToolAggregateReceiptKey,
|
|
182
186
|
buildDurableToolCallAuthScopeDigest,
|
|
183
187
|
buildDurableToolCallCacheKey,
|
|
188
|
+
buildSourceRunToolReceiptManifestKey,
|
|
184
189
|
buildLegacyRevisionedDurableToolCallCacheKey,
|
|
185
190
|
buildDurableToolPayloadProviderIdempotencyKey,
|
|
186
191
|
buildProviderOperationIdempotencyKey,
|
|
@@ -2017,6 +2022,14 @@ function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
|
|
|
2017
2022
|
...frame,
|
|
2018
2023
|
completedRowKeys: [...frame.completedRowKeys],
|
|
2019
2024
|
pendingRowKeys: [...frame.pendingRowKeys],
|
|
2025
|
+
...(frame.suspendedInputRows
|
|
2026
|
+
? {
|
|
2027
|
+
suspendedInputRows: frame.suspendedInputRows.map((row) => ({
|
|
2028
|
+
inputIndex: row.inputIndex,
|
|
2029
|
+
data: cloneCsvAliasedRow(row.data),
|
|
2030
|
+
})),
|
|
2031
|
+
}
|
|
2032
|
+
: {}),
|
|
2020
2033
|
};
|
|
2021
2034
|
}
|
|
2022
2035
|
|
|
@@ -2110,6 +2123,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2110
2123
|
* waterfall miss.
|
|
2111
2124
|
*/
|
|
2112
2125
|
private deeplineInsufficientCreditsFailure: ToolExecutionError | null = null;
|
|
2126
|
+
/**
|
|
2127
|
+
* A Slack HITL timeout is terminal only after its card finalizer has proved
|
|
2128
|
+
* the Slack replacement. Keep that terminal result outside authored promise
|
|
2129
|
+
* control flow so catching the rejected tool call cannot settle the Play or
|
|
2130
|
+
* start a later durable operation.
|
|
2131
|
+
*/
|
|
2132
|
+
private slackHitlTimeoutFailure: ToolExecutionError | null = null;
|
|
2113
2133
|
private toolCallResolvers = new Map<
|
|
2114
2134
|
string,
|
|
2115
2135
|
{ resolve: (value: unknown) => void; reject: (reason: unknown) => void }
|
|
@@ -2309,6 +2329,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2309
2329
|
: undefined,
|
|
2310
2330
|
).catch((error: unknown) => {
|
|
2311
2331
|
this.recordDeeplineInsufficientCreditsFailure(error);
|
|
2332
|
+
this.recordSlackHitlTimeoutFailure(error);
|
|
2312
2333
|
throw error;
|
|
2313
2334
|
}) as Promise<TOutput>;
|
|
2314
2335
|
},
|
|
@@ -2331,12 +2352,46 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2331
2352
|
}
|
|
2332
2353
|
}
|
|
2333
2354
|
|
|
2355
|
+
private recordSlackHitlTimeoutFailure(error: unknown): void {
|
|
2356
|
+
if (
|
|
2357
|
+
error instanceof ToolExecutionError &&
|
|
2358
|
+
error.code === SLACK_HITL_TIMEOUT_ERROR_CODE
|
|
2359
|
+
) {
|
|
2360
|
+
if (!this.slackHitlTimeoutFailure) {
|
|
2361
|
+
this.slackHitlTimeoutFailure = error;
|
|
2362
|
+
this.runtimeLog(
|
|
2363
|
+
`[slack_hitl.timeout_terminal] code=${SLACK_HITL_TIMEOUT_ERROR_CODE} ` +
|
|
2364
|
+
'card_finalization=confirmed outcome=run_fatal',
|
|
2365
|
+
);
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
/**
|
|
2371
|
+
* Called by the runner before a normal result settles. This preserves a
|
|
2372
|
+
* terminal Slack timeout even if authored code catches it and returns a
|
|
2373
|
+
* value. It intentionally remains separate from provider errors, which are
|
|
2374
|
+
* ordinary customer-handled waterfall misses.
|
|
2375
|
+
*/
|
|
2376
|
+
assertNoSlackHitlTimeoutFailure(): void {
|
|
2377
|
+
if (this.slackHitlTimeoutFailure) {
|
|
2378
|
+
throw this.slackHitlTimeoutFailure;
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
/** Called by the runner after customer code returns, before a success settles. */
|
|
2383
|
+
assertNoRunFatalToolFailure(): void {
|
|
2384
|
+
this.assertNoSlackHitlTimeoutFailure();
|
|
2385
|
+
this.assertNoDeeplineInsufficientCreditsFailure();
|
|
2386
|
+
}
|
|
2387
|
+
|
|
2334
2388
|
/**
|
|
2335
2389
|
* Called at ctx effect boundaries and by the runner before it can settle a
|
|
2336
2390
|
* normal result. This makes suspension sticky even when authored code uses a
|
|
2337
2391
|
* broad catch around `await ctx.sleep(...)`.
|
|
2338
2392
|
*/
|
|
2339
2393
|
assertNoPendingDurableSuspension(): void {
|
|
2394
|
+
this.assertNoSlackHitlTimeoutFailure();
|
|
2340
2395
|
if (this.pendingDurableSuspension) {
|
|
2341
2396
|
throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
|
|
2342
2397
|
}
|
|
@@ -2466,6 +2521,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2466
2521
|
options.artifactRuntimeContract ??
|
|
2467
2522
|
resolvePlayArtifactRuntimeContract({
|
|
2468
2523
|
authoringContractEdition: options.authoringContractEdition,
|
|
2524
|
+
slackHitlTimeoutOutcome: options.slackHitlTimeoutOutcome,
|
|
2469
2525
|
toolResponseContract: options.toolResponseContract,
|
|
2470
2526
|
toolResponseReceiptRevision: options.toolResponseReceiptRevision,
|
|
2471
2527
|
});
|
|
@@ -3441,6 +3497,223 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3441
3497
|
};
|
|
3442
3498
|
}
|
|
3443
3499
|
|
|
3500
|
+
/**
|
|
3501
|
+
* Record tiny pointers after canonical tool receipts settle. A rerun can
|
|
3502
|
+
* then recover each exact source-run receipt before trying the ordinary
|
|
3503
|
+
* cross-run cache. The pointers never contain provider input, credentials,
|
|
3504
|
+
* or provider responses.
|
|
3505
|
+
*
|
|
3506
|
+
* This deliberately uses the same bulk receipt seam as canonical result
|
|
3507
|
+
* persistence. A map can settle hundreds of results at once; falling back
|
|
3508
|
+
* to one read/claim/complete triplet per pointer would make compatibility
|
|
3509
|
+
* bookkeeping contend with the customer's provider work.
|
|
3510
|
+
*/
|
|
3511
|
+
private async persistSourceRunToolReceiptManifests(
|
|
3512
|
+
inputs: Array<{
|
|
3513
|
+
toolId: string;
|
|
3514
|
+
logicalCallId?: string | null;
|
|
3515
|
+
executionAuthScopeDigest?: string | null;
|
|
3516
|
+
resultReceiptKey: string;
|
|
3517
|
+
}>,
|
|
3518
|
+
): Promise<void> {
|
|
3519
|
+
const sourceRunId = this.currentReceiptOwnerRunId.trim();
|
|
3520
|
+
if (
|
|
3521
|
+
!sourceRunId ||
|
|
3522
|
+
!(
|
|
3523
|
+
this.#options.claimRuntimeStepReceipt ||
|
|
3524
|
+
this.#options.claimRuntimeStepReceipts
|
|
3525
|
+
) ||
|
|
3526
|
+
!(
|
|
3527
|
+
this.#options.completeRuntimeStepReceipt ||
|
|
3528
|
+
this.#options.completeRuntimeStepReceipts
|
|
3529
|
+
)
|
|
3530
|
+
) {
|
|
3531
|
+
return;
|
|
3532
|
+
}
|
|
3533
|
+
try {
|
|
3534
|
+
const providerActionVersions = new Map<string, string>();
|
|
3535
|
+
await Promise.all(
|
|
3536
|
+
[...new Set(inputs.map((input) => input.toolId))].map(
|
|
3537
|
+
async (toolId) => {
|
|
3538
|
+
providerActionVersions.set(
|
|
3539
|
+
toolId,
|
|
3540
|
+
(
|
|
3541
|
+
await this.#options.getToolActionCacheVersion?.(toolId)
|
|
3542
|
+
)?.trim() ?? '',
|
|
3543
|
+
);
|
|
3544
|
+
},
|
|
3545
|
+
),
|
|
3546
|
+
);
|
|
3547
|
+
const manifests = inputs.flatMap((input) => {
|
|
3548
|
+
const logicalCallId = input.logicalCallId?.trim();
|
|
3549
|
+
const resultReceiptKey = input.resultReceiptKey.trim();
|
|
3550
|
+
if (!logicalCallId || !resultReceiptKey) return [];
|
|
3551
|
+
const providerActionVersion =
|
|
3552
|
+
providerActionVersions.get(input.toolId) ?? '';
|
|
3553
|
+
if (!providerActionVersion) return [];
|
|
3554
|
+
const key = buildSourceRunToolReceiptManifestKey({
|
|
3555
|
+
orgId: this.#options.orgId,
|
|
3556
|
+
sourceRunId,
|
|
3557
|
+
logicalCallId,
|
|
3558
|
+
});
|
|
3559
|
+
return [
|
|
3560
|
+
{
|
|
3561
|
+
key,
|
|
3562
|
+
output: {
|
|
3563
|
+
kind: 'source-run-tool-receipt-manifest-v1' as const,
|
|
3564
|
+
toolId: input.toolId,
|
|
3565
|
+
providerActionVersion,
|
|
3566
|
+
executionAuthScopeDigest:
|
|
3567
|
+
input.executionAuthScopeDigest?.trim() ?? null,
|
|
3568
|
+
resultReceiptKey,
|
|
3569
|
+
},
|
|
3570
|
+
},
|
|
3571
|
+
];
|
|
3572
|
+
});
|
|
3573
|
+
if (manifests.length === 0) return;
|
|
3574
|
+
|
|
3575
|
+
// Several row callbacks can refer to the same logical call. The first
|
|
3576
|
+
// canonical receipt is authoritative, so one idempotent pointer write is
|
|
3577
|
+
// enough.
|
|
3578
|
+
const uniqueManifests = [
|
|
3579
|
+
...new Map(
|
|
3580
|
+
manifests.map((manifest) => [manifest.key, manifest]),
|
|
3581
|
+
).values(),
|
|
3582
|
+
];
|
|
3583
|
+
const existing = await this.getRuntimeStepReceipts(
|
|
3584
|
+
uniqueManifests.map((manifest) => manifest.key),
|
|
3585
|
+
);
|
|
3586
|
+
const missing = uniqueManifests.filter((manifest) => {
|
|
3587
|
+
const receipt = existing.get(manifest.key);
|
|
3588
|
+
return receipt?.status !== 'completed' && receipt?.status !== 'skipped';
|
|
3589
|
+
});
|
|
3590
|
+
if (missing.length === 0) return;
|
|
3591
|
+
|
|
3592
|
+
const claims = await this.claimRuntimeStepReceipts(
|
|
3593
|
+
missing.map((manifest) => manifest.key),
|
|
3594
|
+
sourceRunId,
|
|
3595
|
+
);
|
|
3596
|
+
await this.completeRuntimeStepReceipts(
|
|
3597
|
+
missing.flatMap((manifest) => {
|
|
3598
|
+
const claim = claims.get(manifest.key);
|
|
3599
|
+
return claim?.leaseId
|
|
3600
|
+
? [
|
|
3601
|
+
{
|
|
3602
|
+
key: manifest.key,
|
|
3603
|
+
runId: sourceRunId,
|
|
3604
|
+
leaseId: claim.leaseId,
|
|
3605
|
+
output: manifest.output,
|
|
3606
|
+
},
|
|
3607
|
+
]
|
|
3608
|
+
: [];
|
|
3609
|
+
}),
|
|
3610
|
+
);
|
|
3611
|
+
} catch (error) {
|
|
3612
|
+
// The canonical result receipt has already settled. A telemetry pointer
|
|
3613
|
+
// must never turn that successful provider result into a failed run.
|
|
3614
|
+
this.runtimeLog(
|
|
3615
|
+
`Tool source-run receipt manifest write skipped: ${error instanceof Error ? error.message : String(error)}`,
|
|
3616
|
+
);
|
|
3617
|
+
}
|
|
3618
|
+
}
|
|
3619
|
+
|
|
3620
|
+
private async persistSourceRunToolReceiptManifest(input: {
|
|
3621
|
+
toolId: string;
|
|
3622
|
+
logicalCallId?: string | null;
|
|
3623
|
+
executionAuthScopeDigest?: string | null;
|
|
3624
|
+
resultReceiptKey: string;
|
|
3625
|
+
}): Promise<void> {
|
|
3626
|
+
await this.persistSourceRunToolReceiptManifests([input]);
|
|
3627
|
+
}
|
|
3628
|
+
|
|
3629
|
+
private async recoverSourceRunToolReceipt(input: {
|
|
3630
|
+
toolId: string;
|
|
3631
|
+
logicalCallId: string;
|
|
3632
|
+
executionAuthScopeDigest?: string | null;
|
|
3633
|
+
cacheKey: string;
|
|
3634
|
+
}): Promise<ToolExecuteResult | null> {
|
|
3635
|
+
const sourceRunId = this.#options.replayedFromRunId?.trim();
|
|
3636
|
+
if (!sourceRunId || !this.#options.getRuntimeStepReceipt) return null;
|
|
3637
|
+
|
|
3638
|
+
const manifestKey = buildSourceRunToolReceiptManifestKey({
|
|
3639
|
+
orgId: this.#options.orgId,
|
|
3640
|
+
sourceRunId,
|
|
3641
|
+
logicalCallId: input.logicalCallId,
|
|
3642
|
+
});
|
|
3643
|
+
const manifestReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
3644
|
+
async () => await this.getRuntimeStepReceipt(manifestKey),
|
|
3645
|
+
);
|
|
3646
|
+
if (
|
|
3647
|
+
!manifestReceipt ||
|
|
3648
|
+
(manifestReceipt.status !== 'completed' &&
|
|
3649
|
+
manifestReceipt.status !== 'skipped')
|
|
3650
|
+
) {
|
|
3651
|
+
return null;
|
|
3652
|
+
}
|
|
3653
|
+
const manifest = recordOrNull(this.runtimeReceiptOutput(manifestReceipt));
|
|
3654
|
+
if (
|
|
3655
|
+
!manifest ||
|
|
3656
|
+
manifest.kind !== 'source-run-tool-receipt-manifest-v1' ||
|
|
3657
|
+
manifest.toolId !== input.toolId ||
|
|
3658
|
+
typeof manifest.resultReceiptKey !== 'string' ||
|
|
3659
|
+
!manifest.resultReceiptKey.trim() ||
|
|
3660
|
+
typeof manifest.providerActionVersion !== 'string'
|
|
3661
|
+
) {
|
|
3662
|
+
return null;
|
|
3663
|
+
}
|
|
3664
|
+
const currentActionVersion =
|
|
3665
|
+
(await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
|
|
3666
|
+
'';
|
|
3667
|
+
const compatibleActionVersions = new Set([
|
|
3668
|
+
currentActionVersion,
|
|
3669
|
+
...(
|
|
3670
|
+
(await this.#options.getToolActionCacheReadVersions?.(input.toolId)) ??
|
|
3671
|
+
[]
|
|
3672
|
+
).map((version) => String(version || '').trim()),
|
|
3673
|
+
]);
|
|
3674
|
+
if (!compatibleActionVersions.has(manifest.providerActionVersion.trim())) {
|
|
3675
|
+
return null;
|
|
3676
|
+
}
|
|
3677
|
+
const manifestAuthScopeDigest =
|
|
3678
|
+
typeof manifest.executionAuthScopeDigest === 'string'
|
|
3679
|
+
? manifest.executionAuthScopeDigest.trim() || null
|
|
3680
|
+
: null;
|
|
3681
|
+
if (
|
|
3682
|
+
manifestAuthScopeDigest !==
|
|
3683
|
+
(input.executionAuthScopeDigest?.trim() || null)
|
|
3684
|
+
) {
|
|
3685
|
+
return null;
|
|
3686
|
+
}
|
|
3687
|
+
const resultReceiptKey = manifest.resultReceiptKey.trim();
|
|
3688
|
+
const sourceReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
3689
|
+
async () => await this.getRuntimeStepReceipt(resultReceiptKey),
|
|
3690
|
+
);
|
|
3691
|
+
if (
|
|
3692
|
+
!sourceReceipt ||
|
|
3693
|
+
(sourceReceipt.status !== 'completed' &&
|
|
3694
|
+
sourceReceipt.status !== 'skipped')
|
|
3695
|
+
) {
|
|
3696
|
+
return null;
|
|
3697
|
+
}
|
|
3698
|
+
const stored = this.runtimeReceiptOutput(sourceReceipt);
|
|
3699
|
+
const result = isSerializedToolExecuteResult(stored)
|
|
3700
|
+
? deserializeToolExecuteResult(stored)
|
|
3701
|
+
: isToolExecuteResult(stored)
|
|
3702
|
+
? stored
|
|
3703
|
+
: null;
|
|
3704
|
+
if (!result || !this.canReuseToolResultForCurrentResponseContract(result)) {
|
|
3705
|
+
return null;
|
|
3706
|
+
}
|
|
3707
|
+
return cloneToolExecuteResultWithExecution(
|
|
3708
|
+
result,
|
|
3709
|
+
toolExecutionMetadataForOutcome({
|
|
3710
|
+
kind: 'source_run',
|
|
3711
|
+
cacheKey: input.cacheKey,
|
|
3712
|
+
receiptKey: resultReceiptKey,
|
|
3713
|
+
}),
|
|
3714
|
+
);
|
|
3715
|
+
}
|
|
3716
|
+
|
|
3444
3717
|
private async releaseRuntimeStepReceipt(
|
|
3445
3718
|
key: string,
|
|
3446
3719
|
_runId: string,
|
|
@@ -4258,6 +4531,47 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4258
4531
|
return buildDurableToolCallCacheKey(identity);
|
|
4259
4532
|
}
|
|
4260
4533
|
|
|
4534
|
+
/**
|
|
4535
|
+
* The first key is the only write/provider identity. Later keys are explicit
|
|
4536
|
+
* action-local read fallbacks, used only when source metadata says their
|
|
4537
|
+
* completed result remains compatible. Deploy and generated-catalog state
|
|
4538
|
+
* never participate in this list.
|
|
4539
|
+
*/
|
|
4540
|
+
private async compatibleDurableToolCallCacheKeysForScope(input: {
|
|
4541
|
+
toolId: string;
|
|
4542
|
+
requestInput: Record<string, unknown>;
|
|
4543
|
+
executionAuthScopeDigest?: string | null;
|
|
4544
|
+
staleAfterSeconds?: number | null;
|
|
4545
|
+
playLocalScope?: string | null;
|
|
4546
|
+
}): Promise<string[]> {
|
|
4547
|
+
const identity = await this.durableToolCallCacheIdentity(input);
|
|
4548
|
+
const primaryVersion = identity.providerActionVersion?.trim();
|
|
4549
|
+
if (!primaryVersion) {
|
|
4550
|
+
throw new Error(
|
|
4551
|
+
`Tool ${input.toolId} is missing a provider action compatibility version.`,
|
|
4552
|
+
);
|
|
4553
|
+
}
|
|
4554
|
+
const readVersions = await this.#options.getToolActionCacheReadVersions?.(
|
|
4555
|
+
input.toolId,
|
|
4556
|
+
);
|
|
4557
|
+
const versions = [
|
|
4558
|
+
primaryVersion,
|
|
4559
|
+
...(readVersions ?? [])
|
|
4560
|
+
.map((version) => String(version || '').trim())
|
|
4561
|
+
.filter(Boolean),
|
|
4562
|
+
];
|
|
4563
|
+
return [
|
|
4564
|
+
...new Set(
|
|
4565
|
+
versions.map((providerActionVersion) =>
|
|
4566
|
+
buildDurableToolCallCacheKey({
|
|
4567
|
+
...identity,
|
|
4568
|
+
providerActionVersion,
|
|
4569
|
+
}),
|
|
4570
|
+
),
|
|
4571
|
+
),
|
|
4572
|
+
];
|
|
4573
|
+
}
|
|
4574
|
+
|
|
4261
4575
|
private async legacyDurableToolCallCacheKeyForScope(input: {
|
|
4262
4576
|
toolId: string;
|
|
4263
4577
|
requestInput: Record<string, unknown>;
|
|
@@ -4315,6 +4629,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4315
4629
|
});
|
|
4316
4630
|
}
|
|
4317
4631
|
|
|
4632
|
+
private async compatibleDurableToolCallCacheKeys(input: {
|
|
4633
|
+
toolId: string;
|
|
4634
|
+
requestInput: Record<string, unknown>;
|
|
4635
|
+
executionAuthScopeDigest?: string | null;
|
|
4636
|
+
staleAfterSeconds?: number | null;
|
|
4637
|
+
}): Promise<string[]> {
|
|
4638
|
+
return await this.compatibleDurableToolCallCacheKeysForScope({
|
|
4639
|
+
...input,
|
|
4640
|
+
playLocalScope: this.currentGovernance.currentPlayId,
|
|
4641
|
+
});
|
|
4642
|
+
}
|
|
4643
|
+
|
|
4318
4644
|
private async legacyDurableToolCallCacheKey(input: {
|
|
4319
4645
|
toolId: string;
|
|
4320
4646
|
requestInput: Record<string, unknown>;
|
|
@@ -6421,6 +6747,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6421
6747
|
this.#options.playName &&
|
|
6422
6748
|
this.#options.runId,
|
|
6423
6749
|
);
|
|
6750
|
+
// The invocation id is deterministic for a replay: mapInvocationIndex is
|
|
6751
|
+
// rebuilt in authored call order. Look up a previously suspended plan now,
|
|
6752
|
+
// before a stateful source query can decide that a sibling action is no
|
|
6753
|
+
// longer eligible because an earlier action already completed.
|
|
6754
|
+
const replayMapInvocationId = `${normalizedMapNamespace}:${this.mapInvocationIndex}`;
|
|
6755
|
+
const suspendedInputRows = shouldStreamRuntimeBackedDataset
|
|
6756
|
+
? null
|
|
6757
|
+
: (this.checkpoint.mapFrames?.[replayMapInvocationId]
|
|
6758
|
+
?.suspendedInputRows ?? null);
|
|
6424
6759
|
if (shouldStreamRuntimeBackedDataset) {
|
|
6425
6760
|
// Net-new admission happens one source page at a time. Its progress and
|
|
6426
6761
|
// returned dataset describe admitted rows, not every provider/CSV row
|
|
@@ -7070,11 +7405,24 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7070
7405
|
},
|
|
7071
7406
|
});
|
|
7072
7407
|
}
|
|
7073
|
-
const
|
|
7074
|
-
|
|
7075
|
-
|
|
7076
|
-
|
|
7077
|
-
|
|
7408
|
+
const rawMaterializedEntries = suspendedInputRows?.length
|
|
7409
|
+
? suspendedInputRows.map((row) => ({
|
|
7410
|
+
item: cloneCsvAliasedRow(row.data) as T,
|
|
7411
|
+
inputIndex: row.inputIndex,
|
|
7412
|
+
}))
|
|
7413
|
+
: (
|
|
7414
|
+
await materializePlayDatasetInput(items, {
|
|
7415
|
+
onRow: (row) => {
|
|
7416
|
+
materializationMemoryTracker.track(row);
|
|
7417
|
+
},
|
|
7418
|
+
})
|
|
7419
|
+
).map((item, inputIndex) => ({ item, inputIndex }));
|
|
7420
|
+
const rawMaterializedItems = rawMaterializedEntries.map(
|
|
7421
|
+
(entry) => entry.item,
|
|
7422
|
+
);
|
|
7423
|
+
const originalInputIndexByItem = new Map<T, number>(
|
|
7424
|
+
rawMaterializedEntries.map(({ item, inputIndex }) => [item, inputIndex]),
|
|
7425
|
+
);
|
|
7078
7426
|
// Silent dedupe of duplicate explicit map keys: keep the first row per
|
|
7079
7427
|
// canonical key and drop subsequent duplicates. Deduping the materialized
|
|
7080
7428
|
// input here keeps every downstream derivation (rawItems, itemsToProcess,
|
|
@@ -7087,7 +7435,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7087
7435
|
resolver: (item, index) =>
|
|
7088
7436
|
explicitKeyResolver(
|
|
7089
7437
|
this.toOutputRow(item as Record<string, unknown>),
|
|
7090
|
-
index,
|
|
7438
|
+
originalInputIndexByItem.get(item) ?? index,
|
|
7091
7439
|
),
|
|
7092
7440
|
})
|
|
7093
7441
|
: { rows: rawMaterializedItems, droppedCount: 0, duplicateKeys: [] };
|
|
@@ -7104,7 +7452,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7104
7452
|
itemsToProcess = materializedItems.map((item) =>
|
|
7105
7453
|
this.toOutputRow(item as Record<string, unknown>),
|
|
7106
7454
|
);
|
|
7107
|
-
itemOriginalIndexes = materializedItems.map(
|
|
7455
|
+
itemOriginalIndexes = materializedItems.map(
|
|
7456
|
+
(item, index) => originalInputIndexByItem.get(item) ?? index,
|
|
7457
|
+
);
|
|
7108
7458
|
const mapStartSeedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
7109
7459
|
if (this.#options.onMapStart) {
|
|
7110
7460
|
// ALWAYS stamp the canonical seed row key — including no-explicit-key
|
|
@@ -7492,6 +7842,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7492
7842
|
pendingRowsCount: 0,
|
|
7493
7843
|
failedRowsCount: durableFailedRows,
|
|
7494
7844
|
activeBoundaryId: null,
|
|
7845
|
+
suspendedInputRows: undefined,
|
|
7495
7846
|
updatedAt: Date.now(),
|
|
7496
7847
|
});
|
|
7497
7848
|
}
|
|
@@ -8340,6 +8691,67 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8340
8691
|
});
|
|
8341
8692
|
return WAITING_ROW;
|
|
8342
8693
|
}
|
|
8694
|
+
if (isPlayExecutionSuspendedError(error)) {
|
|
8695
|
+
const completedKeys = new Set(
|
|
8696
|
+
completedRowsToPersist.map((row) => row.key),
|
|
8697
|
+
);
|
|
8698
|
+
const suspendedInputRows = items.flatMap((item, index) => {
|
|
8699
|
+
const rowKey = executionRowKey(
|
|
8700
|
+
this.toOutputRow(item as Record<string, unknown>),
|
|
8701
|
+
index,
|
|
8702
|
+
);
|
|
8703
|
+
if (completedKeys.has(rowKey)) return [];
|
|
8704
|
+
return [
|
|
8705
|
+
{
|
|
8706
|
+
inputIndex: executionRowIndex(index),
|
|
8707
|
+
data: this.toPersistedOutputRow(
|
|
8708
|
+
this.toOutputRow(item as Record<string, unknown>),
|
|
8709
|
+
),
|
|
8710
|
+
},
|
|
8711
|
+
];
|
|
8712
|
+
});
|
|
8713
|
+
const existingFrame =
|
|
8714
|
+
this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
|
|
8715
|
+
this.setMapFrame({
|
|
8716
|
+
...(existingFrame ?? {
|
|
8717
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
8718
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
8719
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
8720
|
+
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
8721
|
+
totalRows,
|
|
8722
|
+
completedRowKeys: [],
|
|
8723
|
+
pendingRowKeys: [],
|
|
8724
|
+
startedAt: Date.now(),
|
|
8725
|
+
}),
|
|
8726
|
+
status: 'suspended',
|
|
8727
|
+
pendingRowKeys: suspendedInputRows.map(({ data, inputIndex }) =>
|
|
8728
|
+
executionRowKey(data, inputIndex),
|
|
8729
|
+
),
|
|
8730
|
+
suspendedInputRows,
|
|
8731
|
+
activeBoundaryId:
|
|
8732
|
+
error.suspension.kind === 'sleep'
|
|
8733
|
+
? error.suspension.boundaryId
|
|
8734
|
+
: null,
|
|
8735
|
+
updatedAt: Date.now(),
|
|
8736
|
+
});
|
|
8737
|
+
this.emitExecutionEvent({
|
|
8738
|
+
type: 'map.suspended',
|
|
8739
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
8740
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
8741
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
8742
|
+
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
8743
|
+
completedRows: completedKeys.size,
|
|
8744
|
+
failedRows: 0,
|
|
8745
|
+
totalRows,
|
|
8746
|
+
at: Date.now(),
|
|
8747
|
+
});
|
|
8748
|
+
// ctx.sleep() emitted its boundary checkpoint before control reached
|
|
8749
|
+
// this row catch. Emit once more after recording the admitted plan so
|
|
8750
|
+
// the scheduler can resume the same action set if the worker exits
|
|
8751
|
+
// immediately after handing off the suspension.
|
|
8752
|
+
this.#options.onBatchComplete?.(this.checkpoint);
|
|
8753
|
+
throw error;
|
|
8754
|
+
}
|
|
8343
8755
|
const fieldName = rowContext.getStore()?.fieldName;
|
|
8344
8756
|
updateMapFrameProgress({
|
|
8345
8757
|
status: 'failed',
|
|
@@ -8373,16 +8785,51 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8373
8785
|
try {
|
|
8374
8786
|
settledResults[idx] = await runRow(idx);
|
|
8375
8787
|
} catch (error) {
|
|
8376
|
-
firstRowError =
|
|
8788
|
+
firstRowError = this.pendingDurableSuspension
|
|
8789
|
+
? new PlayExecutionSuspendedError(this.pendingDurableSuspension)
|
|
8790
|
+
: (firstRowError ?? error);
|
|
8791
|
+
this.wakeToolDispatcher();
|
|
8377
8792
|
throw error;
|
|
8378
8793
|
}
|
|
8379
8794
|
}
|
|
8380
8795
|
});
|
|
8381
8796
|
|
|
8382
|
-
|
|
8797
|
+
// Drain queued work but exit early if a suspension error occurs.
|
|
8798
|
+
// When a worker suspends (e.g., ctx.sleep), we must propagate the
|
|
8799
|
+
// suspension immediately without waiting for other concurrent rows to
|
|
8800
|
+
// settle. This prevents restart loops and false completions.
|
|
8801
|
+
await this.drainQueuedWork(workers, () => {
|
|
8802
|
+
if (firstRowError != null) {
|
|
8803
|
+
if (
|
|
8804
|
+
isPlayExecutionSuspendedError(firstRowError) ||
|
|
8805
|
+
isPlayRowExecutionSuspendedError(firstRowError)
|
|
8806
|
+
)
|
|
8807
|
+
return firstRowError;
|
|
8808
|
+
}
|
|
8809
|
+
return null;
|
|
8810
|
+
});
|
|
8383
8811
|
this.lastDatasetStep = datasetStep;
|
|
8384
8812
|
this.activeDatasetStep = null;
|
|
8385
8813
|
|
|
8814
|
+
// If drainQueuedWork exited early due to suspension, throw it immediately
|
|
8815
|
+
// WITHOUT waiting for Promise.allSettled(workers). The worker promises
|
|
8816
|
+
// may never settle (e.g., sibling rows blocked on long-running operations),
|
|
8817
|
+
// and we must not block suspension propagation on them.
|
|
8818
|
+
if (firstRowError != null) {
|
|
8819
|
+
if (
|
|
8820
|
+
isPlayExecutionSuspendedError(firstRowError) ||
|
|
8821
|
+
isPlayRowExecutionSuspendedError(firstRowError)
|
|
8822
|
+
) {
|
|
8823
|
+
// Use the existing failure barrier to commit accepted terminal rows
|
|
8824
|
+
// before the caller hands this invocation back to the scheduler.
|
|
8825
|
+
throw new FailFastMapRowsError({
|
|
8826
|
+
cause: firstRowError,
|
|
8827
|
+
completedRows: completedRowsToPersist,
|
|
8828
|
+
failedRows: failedRowsToPersist,
|
|
8829
|
+
});
|
|
8830
|
+
}
|
|
8831
|
+
}
|
|
8832
|
+
|
|
8386
8833
|
const settled = await Promise.allSettled(workers);
|
|
8387
8834
|
const rejected = settled.find(
|
|
8388
8835
|
(result): result is PromiseRejectedResult => result.status === 'rejected',
|
|
@@ -8664,8 +9111,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8664
9111
|
// author hand-writing error handling.
|
|
8665
9112
|
if (
|
|
8666
9113
|
!continueOnProviderUnavailable ||
|
|
8667
|
-
(!isProviderExhaustedFailure(error) &&
|
|
8668
|
-
!isProviderUnavailable(error))
|
|
9114
|
+
(!isProviderExhaustedFailure(error) && !isProviderUnavailable(error))
|
|
8669
9115
|
) {
|
|
8670
9116
|
throw error;
|
|
8671
9117
|
}
|
|
@@ -9238,7 +9684,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9238
9684
|
});
|
|
9239
9685
|
}
|
|
9240
9686
|
|
|
9241
|
-
private async drainQueuedWork<T>(
|
|
9687
|
+
private async drainQueuedWork<T>(
|
|
9688
|
+
promises: Promise<T>[],
|
|
9689
|
+
earlyExitCheck?: () => unknown,
|
|
9690
|
+
): Promise<void> {
|
|
9242
9691
|
// One dispatcher owns ready work, fixed lane coalescing deadlines, and the
|
|
9243
9692
|
// bounded in-flight registry. A row resumes after its own receipt persists;
|
|
9244
9693
|
// it never waits for unrelated sibling calls from the previous column.
|
|
@@ -9270,6 +9719,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9270
9719
|
try {
|
|
9271
9720
|
let pass = 0;
|
|
9272
9721
|
while (true) {
|
|
9722
|
+
// Check for early exit FIRST, before any dispatch or wait logic.
|
|
9723
|
+
// When a row suspends, we must exit immediately without waiting for
|
|
9724
|
+
// sibling rows or tool dispatcher wake events.
|
|
9725
|
+
const suspension = earlyExitCheck?.();
|
|
9726
|
+
if (suspension != null) {
|
|
9727
|
+
// Queued calls have not started; reject them at the suspension fence.
|
|
9728
|
+
// Already-dispatched calls must settle while this attempt still owns
|
|
9729
|
+
// their receipts. Runner teardown releases leases, so detaching live
|
|
9730
|
+
// provider requests here could duplicate them on the next resume.
|
|
9731
|
+
this.rejectQueuedToolCalls(suspension);
|
|
9732
|
+
await Promise.allSettled([...inFlightToolExecutions]);
|
|
9733
|
+
break;
|
|
9734
|
+
}
|
|
9735
|
+
|
|
9273
9736
|
if (this.toolDispatcherFailure != null) {
|
|
9274
9737
|
this.rejectQueuedToolCalls(this.toolDispatcherFailure);
|
|
9275
9738
|
await Promise.allSettled([...inFlightToolExecutions]);
|
|
@@ -9484,12 +9947,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9484
9947
|
!eventWaitHandler &&
|
|
9485
9948
|
!isQueryResultDatasetReadRequest(toolId, input) &&
|
|
9486
9949
|
!isAlwaysFreshIntegrationTool(toolId);
|
|
9487
|
-
let
|
|
9488
|
-
|
|
9489
|
-
|
|
9490
|
-
|
|
9491
|
-
|
|
9492
|
-
|
|
9950
|
+
let compatibleDurableCacheKeys =
|
|
9951
|
+
await this.compatibleDurableToolCallCacheKeys({
|
|
9952
|
+
toolId,
|
|
9953
|
+
requestInput: input,
|
|
9954
|
+
executionAuthScopeDigest,
|
|
9955
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
9956
|
+
});
|
|
9957
|
+
let durableCacheKey = compatibleDurableCacheKeys[0];
|
|
9958
|
+
if (!durableCacheKey) {
|
|
9959
|
+
throw new Error(`Tool ${toolId} is missing a primary durable cache key.`);
|
|
9960
|
+
}
|
|
9493
9961
|
let legacyDurableCacheKey = await this.legacyDurableToolCallCacheKey({
|
|
9494
9962
|
toolId,
|
|
9495
9963
|
requestInput: input,
|
|
@@ -9516,7 +9984,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9516
9984
|
// incompatible state deliberately forgoes caching until it is repaired.
|
|
9517
9985
|
let bypassDirectResultCache = false;
|
|
9518
9986
|
const checkpointCacheKeys = [
|
|
9519
|
-
|
|
9987
|
+
...compatibleDurableCacheKeys,
|
|
9520
9988
|
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
9521
9989
|
];
|
|
9522
9990
|
|
|
@@ -9534,6 +10002,28 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9534
10002
|
if (toolId === 'run_javascript') {
|
|
9535
10003
|
throw new Error(DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE);
|
|
9536
10004
|
}
|
|
10005
|
+
// A user-requested rerun first asks its exact source run whether this
|
|
10006
|
+
// logical call already completed. This is stronger than the shared cache:
|
|
10007
|
+
// it is an explicit, authenticated pointer to one known source receipt.
|
|
10008
|
+
if (cacheableToolResult && !toolCachePolicy.force) {
|
|
10009
|
+
const sourceRunRecovered = await this.recoverSourceRunToolReceipt({
|
|
10010
|
+
toolId,
|
|
10011
|
+
logicalCallId,
|
|
10012
|
+
executionAuthScopeDigest,
|
|
10013
|
+
cacheKey: durableCacheKey,
|
|
10014
|
+
});
|
|
10015
|
+
if (sourceRunRecovered) {
|
|
10016
|
+
this.runtimeLog(
|
|
10017
|
+
`Calling tool: ${toolId} recovered from source run ${this.#options.replayedFromRunId}`,
|
|
10018
|
+
);
|
|
10019
|
+
this.emitToolCallLifecycle('started', toolId, normalizedKey);
|
|
10020
|
+
this.emitToolCallLifecycle('settled', toolId, normalizedKey, {
|
|
10021
|
+
outcome: 'cached',
|
|
10022
|
+
durationMs: 0,
|
|
10023
|
+
});
|
|
10024
|
+
return this.materializeToolExecutionResult(sourceRunRecovered);
|
|
10025
|
+
}
|
|
10026
|
+
}
|
|
9537
10027
|
// Tool-call budget is charged by the Governor's acquireToolSlot in
|
|
9538
10028
|
// callToolExecutionAPI, so each real outbound call is counted exactly once
|
|
9539
10029
|
// (cached/checkpoint recoveries never acquire a slot and so never charge).
|
|
@@ -9602,6 +10092,45 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9602
10092
|
// reader seam rather than teaching the deep Module an untyped result.
|
|
9603
10093
|
// New direct calls use the real Tool Result Receipt Adapter below.
|
|
9604
10094
|
if (cacheableToolResult && !toolCachePolicy.force) {
|
|
10095
|
+
// A source action can explicitly carry forward a previous result
|
|
10096
|
+
// contract after a local version bump. Read those exact older keys
|
|
10097
|
+
// before the canonical key; do not derive fallbacks from a deploy.
|
|
10098
|
+
for (const compatibleCacheKey of compatibleDurableCacheKeys.slice(
|
|
10099
|
+
1,
|
|
10100
|
+
)) {
|
|
10101
|
+
const compatibleReceipt =
|
|
10102
|
+
await this.withRuntimeReceiptHydrationTurn(
|
|
10103
|
+
async () =>
|
|
10104
|
+
await this.getRuntimeStepReceipt(compatibleCacheKey),
|
|
10105
|
+
);
|
|
10106
|
+
if (
|
|
10107
|
+
compatibleReceipt &&
|
|
10108
|
+
(compatibleReceipt.status === 'completed' ||
|
|
10109
|
+
compatibleReceipt.status === 'skipped') &&
|
|
10110
|
+
compatibleReceipt.output !== undefined
|
|
10111
|
+
) {
|
|
10112
|
+
const recoveredCompatible = markToolExecuteResultExecutionOutcome(
|
|
10113
|
+
this.runtimeReceiptOutput(compatibleReceipt),
|
|
10114
|
+
toolExecutionOutcomeForDurableReceipt({
|
|
10115
|
+
source: 'cache',
|
|
10116
|
+
receiptKey: compatibleCacheKey,
|
|
10117
|
+
cacheKey: directCacheKey,
|
|
10118
|
+
}),
|
|
10119
|
+
);
|
|
10120
|
+
if (
|
|
10121
|
+
this.canReuseToolResultForCurrentResponseContract(
|
|
10122
|
+
recoveredCompatible,
|
|
10123
|
+
)
|
|
10124
|
+
) {
|
|
10125
|
+
this.runtimeLog(
|
|
10126
|
+
`Calling tool: ${toolId} recovered from an explicit action compatibility receipt`,
|
|
10127
|
+
);
|
|
10128
|
+
return isToolExecuteResult(recoveredCompatible)
|
|
10129
|
+
? this.materializeToolExecutionResult(recoveredCompatible)
|
|
10130
|
+
: recoveredCompatible;
|
|
10131
|
+
}
|
|
10132
|
+
}
|
|
10133
|
+
}
|
|
9605
10134
|
const historicalReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
9606
10135
|
async () => await this.getRuntimeStepReceipt(directCacheKey),
|
|
9607
10136
|
);
|
|
@@ -9860,6 +10389,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9860
10389
|
: {}),
|
|
9861
10390
|
withCompletedReceiptHydration: (hydrate) =>
|
|
9862
10391
|
this.withRuntimeReceiptHydrationTurn(hydrate),
|
|
10392
|
+
onStored: async ({ resultReceiptKey }) => {
|
|
10393
|
+
await this.persistSourceRunToolReceiptManifest({
|
|
10394
|
+
toolId,
|
|
10395
|
+
logicalCallId,
|
|
10396
|
+
executionAuthScopeDigest,
|
|
10397
|
+
resultReceiptKey,
|
|
10398
|
+
});
|
|
10399
|
+
},
|
|
9863
10400
|
...(toolRetryPolicy?.requiresExecutionFence === true
|
|
9864
10401
|
? {
|
|
9865
10402
|
executionLock: {
|
|
@@ -10037,9 +10574,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10037
10574
|
providerIdempotencyKeyBase,
|
|
10038
10575
|
cacheable: cacheableToolResult,
|
|
10039
10576
|
receiptKey: cacheableToolResult ? durableCacheKey : null,
|
|
10040
|
-
legacyReceiptKeys:
|
|
10041
|
-
|
|
10042
|
-
: [],
|
|
10577
|
+
legacyReceiptKeys: [
|
|
10578
|
+
...compatibleDurableCacheKeys.slice(1),
|
|
10579
|
+
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
10580
|
+
],
|
|
10043
10581
|
executionAuthScopeDigest,
|
|
10044
10582
|
requiresExecutionFence:
|
|
10045
10583
|
rowToolRetryPolicy?.requiresExecutionFence === true,
|
|
@@ -10107,12 +10645,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10107
10645
|
this.invalidateToolAuthScopeDigest(toolId);
|
|
10108
10646
|
executionAuthScopeDigest =
|
|
10109
10647
|
(await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
|
|
10110
|
-
|
|
10111
|
-
|
|
10112
|
-
|
|
10113
|
-
|
|
10114
|
-
|
|
10115
|
-
|
|
10648
|
+
compatibleDurableCacheKeys =
|
|
10649
|
+
await this.compatibleDurableToolCallCacheKeys({
|
|
10650
|
+
toolId,
|
|
10651
|
+
requestInput: input,
|
|
10652
|
+
executionAuthScopeDigest,
|
|
10653
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
10654
|
+
});
|
|
10655
|
+
durableCacheKey = compatibleDurableCacheKeys[0];
|
|
10656
|
+
if (!durableCacheKey) {
|
|
10657
|
+
throw new Error(
|
|
10658
|
+
`Tool ${toolId} is missing a primary durable cache key after auth refresh.`,
|
|
10659
|
+
);
|
|
10660
|
+
}
|
|
10116
10661
|
legacyDurableCacheKey = await this.legacyDurableToolCallCacheKey({
|
|
10117
10662
|
toolId,
|
|
10118
10663
|
requestInput: input,
|
|
@@ -10132,7 +10677,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10132
10677
|
checkpointCacheKeys.splice(
|
|
10133
10678
|
0,
|
|
10134
10679
|
checkpointCacheKeys.length,
|
|
10135
|
-
|
|
10680
|
+
...compatibleDurableCacheKeys,
|
|
10136
10681
|
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
10137
10682
|
);
|
|
10138
10683
|
}
|
|
@@ -10160,6 +10705,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10160
10705
|
executorToken: this.#options.executorToken,
|
|
10161
10706
|
slackHitlDefaultTimeoutMs:
|
|
10162
10707
|
this.currentRuntimeSemantics.slackHitl.defaultTimeoutMs,
|
|
10708
|
+
slackHitlTimeoutOutcome:
|
|
10709
|
+
this.currentRuntimeContract.slackHitlTimeoutOutcome,
|
|
10163
10710
|
};
|
|
10164
10711
|
const basePreparedBoundary = await handler.prepare({
|
|
10165
10712
|
payload: input,
|
|
@@ -10187,19 +10734,47 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10187
10734
|
if (existing?.kind === 'integration_event' && 'output' in existing) {
|
|
10188
10735
|
const output = recordOrNull(existing.output);
|
|
10189
10736
|
if (output?.timed_out === true) {
|
|
10190
|
-
|
|
10191
|
-
|
|
10192
|
-
|
|
10193
|
-
|
|
10194
|
-
|
|
10195
|
-
|
|
10196
|
-
|
|
10197
|
-
|
|
10198
|
-
|
|
10199
|
-
|
|
10200
|
-
|
|
10201
|
-
|
|
10202
|
-
|
|
10737
|
+
// The runtime proxy is recreated for each worker invocation. Preserve
|
|
10738
|
+
// the policy that armed this boundary so its timeout finalizer and
|
|
10739
|
+
// parking behavior survive a durable replay.
|
|
10740
|
+
const timeoutPolicy =
|
|
10741
|
+
existing.timeoutPolicy ?? preparedBoundary.timeoutPolicy;
|
|
10742
|
+
// An explicit boundary policy may deliberately omit parking (the
|
|
10743
|
+
// new on_timeout: "return_null" escape hatch). Only fall back to the
|
|
10744
|
+
// handler's historical default when this is an old boundary with no
|
|
10745
|
+
// persisted policy at all.
|
|
10746
|
+
const suspendAfterTimeoutMs = timeoutPolicy
|
|
10747
|
+
? timeoutPolicy.suspendAfterTimeoutMs
|
|
10748
|
+
: handler.suspendAfterTimeoutMs;
|
|
10749
|
+
try {
|
|
10750
|
+
await handler.onTimeout?.({
|
|
10751
|
+
payload: input,
|
|
10752
|
+
context: eventContext,
|
|
10753
|
+
boundary: {
|
|
10754
|
+
boundaryId: preparedBoundary.boundaryId,
|
|
10755
|
+
eventKey: preparedBoundary.eventKey,
|
|
10756
|
+
timeoutMs: preparedBoundary.timeoutMs,
|
|
10757
|
+
provider: existing.provider ?? handler.provider,
|
|
10758
|
+
toolId: existing.toolId ?? handler.toolId,
|
|
10759
|
+
...(existing.messageRef
|
|
10760
|
+
? { messageRef: existing.messageRef }
|
|
10761
|
+
: {}),
|
|
10762
|
+
...(timeoutPolicy ? { timeoutPolicy } : {}),
|
|
10763
|
+
},
|
|
10764
|
+
});
|
|
10765
|
+
} catch (error) {
|
|
10766
|
+
this.recordSlackHitlTimeoutFailure(error);
|
|
10767
|
+
throw error;
|
|
10768
|
+
}
|
|
10769
|
+
if (timeoutPolicy?.timeoutOutcome === 'error') {
|
|
10770
|
+
// The provider finalizer above must complete first. That ordering
|
|
10771
|
+
// makes an expired Slack card and the terminal Play state agree,
|
|
10772
|
+
// while provider finalization failures stay retryable instead.
|
|
10773
|
+
const error = new SlackHitlTimeoutError(timeoutPolicy.timeoutError);
|
|
10774
|
+
this.recordSlackHitlTimeoutFailure(error);
|
|
10775
|
+
throw error;
|
|
10776
|
+
}
|
|
10777
|
+
if (suspendAfterTimeoutMs !== undefined) {
|
|
10203
10778
|
this.runtimeLog(
|
|
10204
10779
|
`Integration event ${preparedBoundary.boundaryId}: timeout finalized externally; keeping Play parked`,
|
|
10205
10780
|
);
|
|
@@ -10207,7 +10782,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10207
10782
|
kind: 'integration_event',
|
|
10208
10783
|
boundaryId: preparedBoundary.boundaryId,
|
|
10209
10784
|
eventKey: preparedBoundary.eventKey,
|
|
10210
|
-
timeoutMs:
|
|
10785
|
+
timeoutMs: suspendAfterTimeoutMs,
|
|
10211
10786
|
...(existing.messageRef ? { messageRef: existing.messageRef } : {}),
|
|
10212
10787
|
});
|
|
10213
10788
|
}
|
|
@@ -10246,6 +10821,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10246
10821
|
timeoutMs: boundary.timeoutMs,
|
|
10247
10822
|
provider: boundary.provider,
|
|
10248
10823
|
toolId: boundary.toolId,
|
|
10824
|
+
...(boundary.timeoutPolicy
|
|
10825
|
+
? { timeoutPolicy: boundary.timeoutPolicy }
|
|
10826
|
+
: {}),
|
|
10249
10827
|
...(rowScope
|
|
10250
10828
|
? {
|
|
10251
10829
|
scope: {
|
|
@@ -10410,6 +10988,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10410
10988
|
runtimeContract: resolvePlayArtifactRuntimeContract({
|
|
10411
10989
|
authoringContractEdition:
|
|
10412
10990
|
childCompatibility.authoringContractEdition,
|
|
10991
|
+
slackHitlTimeoutOutcome:
|
|
10992
|
+
childCompatibility.slackHitlTimeoutOutcome,
|
|
10413
10993
|
toolResponseContract: childToolResponseContract,
|
|
10414
10994
|
toolResponseReceiptRevision: childToolResponseReceiptRevision,
|
|
10415
10995
|
}),
|
|
@@ -11602,6 +12182,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11602
12182
|
output: serializeToolExecuteResult(receipt.result),
|
|
11603
12183
|
})),
|
|
11604
12184
|
);
|
|
12185
|
+
await this.persistSourceRunToolReceiptManifests(
|
|
12186
|
+
receipts.flatMap((receipt) => {
|
|
12187
|
+
const request = input.requestForReceipt.get(
|
|
12188
|
+
receipt.resultReceiptKey,
|
|
12189
|
+
);
|
|
12190
|
+
return request
|
|
12191
|
+
? [
|
|
12192
|
+
{
|
|
12193
|
+
toolId: input.toolId,
|
|
12194
|
+
logicalCallId: request.logicalCallId,
|
|
12195
|
+
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
12196
|
+
resultReceiptKey: receipt.resultReceiptKey,
|
|
12197
|
+
},
|
|
12198
|
+
]
|
|
12199
|
+
: [];
|
|
12200
|
+
}),
|
|
12201
|
+
);
|
|
11605
12202
|
return await recordsFor(
|
|
11606
12203
|
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
11607
12204
|
completed,
|
|
@@ -11884,7 +12481,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11884
12481
|
if (requests.length === 0) return [];
|
|
11885
12482
|
|
|
11886
12483
|
const requestForReceipt = new Map<string, ToolCallRequest>();
|
|
11887
|
-
const requestByCallId = new Map<string, ToolCallRequest>();
|
|
11888
12484
|
for (const request of requests) {
|
|
11889
12485
|
const receiptKey = request.receiptKey?.trim();
|
|
11890
12486
|
if (!receiptKey) {
|
|
@@ -11893,13 +12489,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11893
12489
|
);
|
|
11894
12490
|
}
|
|
11895
12491
|
requestForReceipt.set(receiptKey, request);
|
|
11896
|
-
requestByCallId.set(request.callId, request);
|
|
11897
12492
|
}
|
|
11898
12493
|
const store = this.createRuntimeStepToolCallReceiptCohortStore({
|
|
11899
12494
|
toolId,
|
|
11900
12495
|
requestForReceipt,
|
|
11901
12496
|
});
|
|
11902
|
-
const
|
|
12497
|
+
const plannedBatches = compileRequestsWithStrategy({
|
|
11903
12498
|
requests,
|
|
11904
12499
|
strategy,
|
|
11905
12500
|
getPayload: (request: ToolCallRequest) => request.input,
|
|
@@ -11922,9 +12517,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11922
12517
|
const batchParallelismCeiling =
|
|
11923
12518
|
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
11924
12519
|
const batchSize =
|
|
11925
|
-
|
|
12520
|
+
plannedBatches.length > 0
|
|
11926
12521
|
? await this.resourceGovernor.suggestedToolParallelism(
|
|
11927
|
-
|
|
12522
|
+
plannedBatches[0]!.batchOperation,
|
|
11928
12523
|
batchParallelismCeiling,
|
|
11929
12524
|
)
|
|
11930
12525
|
: batchParallelismCeiling;
|
|
@@ -11963,34 +12558,36 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11963
12558
|
};
|
|
11964
12559
|
|
|
11965
12560
|
await executeChunkedRequests({
|
|
11966
|
-
requests:
|
|
12561
|
+
requests: plannedBatches,
|
|
11967
12562
|
batchSize,
|
|
11968
|
-
execute: async (
|
|
12563
|
+
execute: async (plannedBatch) => {
|
|
11969
12564
|
if (this.persistenceLatch.tripped) {
|
|
11970
12565
|
this.persistenceLatch.preventedCallCount +=
|
|
11971
|
-
|
|
12566
|
+
plannedBatch.memberRequests.length;
|
|
11972
12567
|
throw new RuntimePersistenceCircuitOpenError(this.persistenceLatch);
|
|
11973
12568
|
}
|
|
11974
|
-
const
|
|
11975
|
-
(
|
|
11976
|
-
|
|
11977
|
-
|
|
11978
|
-
|
|
11979
|
-
)?.trim() ?? '';
|
|
11980
|
-
const batchRequestByCallId = new Map(
|
|
11981
|
-
batch.memberRequests.map((request) => [request.callId, request]),
|
|
12569
|
+
const requestByCallId = new Map(
|
|
12570
|
+
plannedBatch.memberRequests.map((request) => [
|
|
12571
|
+
request.callId,
|
|
12572
|
+
request,
|
|
12573
|
+
]),
|
|
11982
12574
|
);
|
|
11983
|
-
|
|
12575
|
+
type NativeBatchPhysicalDispatchDetails = {
|
|
12576
|
+
members: ToolCallRequest[];
|
|
12577
|
+
compiled: CompiledRequestBatch<ToolCallRequest>;
|
|
12578
|
+
providerOperationKey: string;
|
|
12579
|
+
forceRefreshIdentities: string[];
|
|
12580
|
+
receiptLeaseExpiresAt: string | null;
|
|
12581
|
+
timeoutMs: number | undefined;
|
|
12582
|
+
};
|
|
12583
|
+
const nativeBatchPhysicalDispatchByAggregateReceipt = new Map<
|
|
11984
12584
|
string,
|
|
11985
|
-
|
|
11986
|
-
members: ToolCallRequest[];
|
|
11987
|
-
compiled: CompiledRequestBatch<ToolCallRequest>;
|
|
11988
|
-
providerOperationKey: string;
|
|
11989
|
-
forceRefreshIdentities: string[];
|
|
11990
|
-
receiptLeaseExpiresAt: string | null;
|
|
11991
|
-
timeoutMs: number | undefined;
|
|
11992
|
-
}
|
|
12585
|
+
NativeBatchPhysicalDispatchDetails
|
|
11993
12586
|
>();
|
|
12587
|
+
const aggregateReceiptKeyByOwnedMemberIds = new Map<string, string>();
|
|
12588
|
+
const ownedMemberIdsKey = (
|
|
12589
|
+
members: ReadonlyArray<{ memberId: string }>,
|
|
12590
|
+
): string => JSON.stringify(members.map((member) => member.memberId));
|
|
11994
12591
|
const physicalToolCall = {
|
|
11995
12592
|
call: async (
|
|
11996
12593
|
physical: Parameters<
|
|
@@ -12003,7 +12600,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12003
12600
|
'Native batch physical Tool Call requires compatibility identities.',
|
|
12004
12601
|
);
|
|
12005
12602
|
}
|
|
12006
|
-
const detail =
|
|
12603
|
+
const detail = nativeBatchPhysicalDispatchByAggregateReceipt.get(
|
|
12604
|
+
compatibility.resultReceiptKey,
|
|
12605
|
+
);
|
|
12007
12606
|
if (!detail) {
|
|
12008
12607
|
throw new Error(
|
|
12009
12608
|
'Native batch physical Tool Call lost its dispatch details.',
|
|
@@ -12079,7 +12678,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12079
12678
|
runId: this.currentReceiptOwnerRunId,
|
|
12080
12679
|
attempt: this.currentRunAttempt,
|
|
12081
12680
|
},
|
|
12082
|
-
members:
|
|
12681
|
+
members: plannedBatch.memberRequests.map((request) => ({
|
|
12083
12682
|
memberId: request.callId,
|
|
12084
12683
|
forceFailedRefresh: request.forceFailedRefresh === true,
|
|
12085
12684
|
waitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
@@ -12117,74 +12716,91 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12117
12716
|
},
|
|
12118
12717
|
},
|
|
12119
12718
|
})),
|
|
12120
|
-
buildPhysicalCall: ({ members }) => {
|
|
12121
|
-
|
|
12122
|
-
|
|
12719
|
+
buildPhysicalCall: async ({ members: ownedMembers }) => {
|
|
12720
|
+
// `ownedMembers` are the rows we still need to execute.
|
|
12721
|
+
const ownedMemberRequests = ownedMembers.map(
|
|
12722
|
+
(member) => requestByCallId.get(member.memberId)!,
|
|
12723
|
+
);
|
|
12724
|
+
const compiled = compileOwnedPhysicalBatch(ownedMemberRequests);
|
|
12725
|
+
const receiptKeys = ownedMemberRequests.map(
|
|
12726
|
+
(request) => request.receiptKey!,
|
|
12123
12727
|
);
|
|
12124
|
-
const receiptKeys = owned.map((request) => request.receiptKey!);
|
|
12125
12728
|
const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
|
|
12126
12729
|
receiptKeys,
|
|
12127
12730
|
prefix: 'batch',
|
|
12128
12731
|
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
12129
12732
|
orgId: this.#options.orgId,
|
|
12130
|
-
toolId:
|
|
12733
|
+
toolId: compiled.batchOperation,
|
|
12131
12734
|
}),
|
|
12132
12735
|
});
|
|
12133
|
-
const
|
|
12134
|
-
|
|
12135
|
-
|
|
12136
|
-
|
|
12137
|
-
|
|
12138
|
-
|
|
12139
|
-
const forceRefreshIdentities =
|
|
12140
|
-
|
|
12141
|
-
|
|
12142
|
-
|
|
12143
|
-
|
|
12144
|
-
|
|
12145
|
-
|
|
12146
|
-
|
|
12736
|
+
const providerActionVersion =
|
|
12737
|
+
(
|
|
12738
|
+
await this.#options.getToolActionCacheVersion?.(
|
|
12739
|
+
compiled.batchOperation,
|
|
12740
|
+
)
|
|
12741
|
+
)?.trim() ?? '';
|
|
12742
|
+
const forceRefreshIdentities = ownedMemberRequests.flatMap(
|
|
12743
|
+
(request) =>
|
|
12744
|
+
this.providerForceRefreshIdentitiesForToolCall({
|
|
12745
|
+
operationKey:
|
|
12746
|
+
this.providerOperationKeyForQueuedToolCall(request),
|
|
12747
|
+
force: request.force,
|
|
12748
|
+
requiresExecutionFence: request.requiresExecutionFence,
|
|
12749
|
+
logicalCallId: request.logicalCallId,
|
|
12750
|
+
}),
|
|
12147
12751
|
);
|
|
12148
12752
|
const providerOperationKey =
|
|
12149
12753
|
buildDurableToolPayloadProviderIdempotencyKey({
|
|
12150
12754
|
orgId: this.#options.orgId,
|
|
12151
|
-
toolId:
|
|
12755
|
+
toolId: compiled.batchOperation,
|
|
12152
12756
|
executionAuthScopeDigest:
|
|
12153
|
-
|
|
12154
|
-
providerActionVersion
|
|
12757
|
+
ownedMemberRequests[0]?.executionAuthScopeDigest ?? null,
|
|
12758
|
+
providerActionVersion,
|
|
12155
12759
|
payload: compiled.batchPayload,
|
|
12156
12760
|
forceRefreshIdentities,
|
|
12157
12761
|
});
|
|
12158
|
-
|
|
12159
|
-
|
|
12160
|
-
|
|
12161
|
-
|
|
12162
|
-
|
|
12163
|
-
|
|
12164
|
-
|
|
12762
|
+
const aggregateDispatchDetails: NativeBatchPhysicalDispatchDetails =
|
|
12763
|
+
{
|
|
12764
|
+
members: ownedMemberRequests,
|
|
12765
|
+
compiled,
|
|
12766
|
+
providerOperationKey,
|
|
12767
|
+
forceRefreshIdentities,
|
|
12768
|
+
receiptLeaseExpiresAt: ownedMemberRequests.reduce<
|
|
12769
|
+
string | null
|
|
12770
|
+
>((earliest, request) => {
|
|
12165
12771
|
const expiresAt = request.receiptLeaseExpiresAt ?? null;
|
|
12166
12772
|
if (!expiresAt) return earliest;
|
|
12167
12773
|
if (!earliest) return expiresAt;
|
|
12168
12774
|
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
12169
12775
|
? expiresAt
|
|
12170
12776
|
: earliest;
|
|
12171
|
-
},
|
|
12172
|
-
|
|
12173
|
-
|
|
12174
|
-
|
|
12175
|
-
|
|
12176
|
-
|
|
12177
|
-
|
|
12178
|
-
|
|
12179
|
-
|
|
12180
|
-
|
|
12181
|
-
|
|
12182
|
-
|
|
12777
|
+
}, null),
|
|
12778
|
+
timeoutMs: ownedMemberRequests.every(
|
|
12779
|
+
(request) =>
|
|
12780
|
+
typeof request.timeoutMs === 'number' &&
|
|
12781
|
+
Number.isFinite(request.timeoutMs) &&
|
|
12782
|
+
request.timeoutMs > 0,
|
|
12783
|
+
)
|
|
12784
|
+
? Math.max(
|
|
12785
|
+
...ownedMemberRequests.map(
|
|
12786
|
+
(request) => request.timeoutMs!,
|
|
12787
|
+
),
|
|
12788
|
+
)
|
|
12789
|
+
: undefined,
|
|
12790
|
+
};
|
|
12791
|
+
nativeBatchPhysicalDispatchByAggregateReceipt.set(
|
|
12792
|
+
aggregateReceiptKey,
|
|
12793
|
+
aggregateDispatchDetails,
|
|
12794
|
+
);
|
|
12795
|
+
aggregateReceiptKeyByOwnedMemberIds.set(
|
|
12796
|
+
ownedMemberIdsKey(ownedMembers),
|
|
12797
|
+
aggregateReceiptKey,
|
|
12798
|
+
);
|
|
12183
12799
|
return {
|
|
12184
|
-
...
|
|
12800
|
+
...ownedMembers[0]!.call,
|
|
12185
12801
|
tool: {
|
|
12186
|
-
id:
|
|
12187
|
-
providerActionVersion
|
|
12802
|
+
id: compiled.batchOperation,
|
|
12803
|
+
providerActionVersion,
|
|
12188
12804
|
},
|
|
12189
12805
|
payload: compiled.batchPayload,
|
|
12190
12806
|
compatibility: {
|
|
@@ -12193,34 +12809,29 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12193
12809
|
},
|
|
12194
12810
|
};
|
|
12195
12811
|
},
|
|
12196
|
-
splitResult: async ({ result: aggregate, members }) => {
|
|
12197
|
-
const
|
|
12198
|
-
|
|
12199
|
-
|
|
12200
|
-
|
|
12201
|
-
|
|
12202
|
-
|
|
12203
|
-
|
|
12204
|
-
|
|
12205
|
-
|
|
12206
|
-
|
|
12207
|
-
|
|
12208
|
-
|
|
12209
|
-
aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
|
|
12210
|
-
orgId: this.#options.orgId,
|
|
12211
|
-
toolId: batch.batchOperation,
|
|
12212
|
-
}),
|
|
12213
|
-
});
|
|
12214
|
-
const detail = dispatchDetails.get(aggregateReceiptKey);
|
|
12812
|
+
splitResult: async ({ result: aggregate, members: ownedMembers }) => {
|
|
12813
|
+
const aggregateReceiptKey = aggregateReceiptKeyByOwnedMemberIds.get(
|
|
12814
|
+
ownedMemberIdsKey(ownedMembers),
|
|
12815
|
+
);
|
|
12816
|
+
if (!aggregateReceiptKey) {
|
|
12817
|
+
throw new Error(
|
|
12818
|
+
'Native batch Tool Call lost its physical aggregate receipt identity.',
|
|
12819
|
+
);
|
|
12820
|
+
}
|
|
12821
|
+
const detail =
|
|
12822
|
+
nativeBatchPhysicalDispatchByAggregateReceipt.get(
|
|
12823
|
+
aggregateReceiptKey,
|
|
12824
|
+
);
|
|
12215
12825
|
if (!detail) {
|
|
12216
12826
|
throw new Error(
|
|
12217
12827
|
'Native batch Tool Call lost its split compilation details.',
|
|
12218
12828
|
);
|
|
12219
12829
|
}
|
|
12220
12830
|
if (
|
|
12221
|
-
detail.members.length !==
|
|
12831
|
+
detail.members.length !== ownedMembers.length ||
|
|
12222
12832
|
!detail.members.every(
|
|
12223
|
-
(request, index) =>
|
|
12833
|
+
(request, index) =>
|
|
12834
|
+
request.callId === ownedMembers[index]?.memberId,
|
|
12224
12835
|
)
|
|
12225
12836
|
) {
|
|
12226
12837
|
throw new Error(
|
|
@@ -12231,8 +12842,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12231
12842
|
legacyResultForBatchSplitter(aggregate),
|
|
12232
12843
|
);
|
|
12233
12844
|
return await Promise.all(
|
|
12234
|
-
|
|
12235
|
-
const request =
|
|
12845
|
+
ownedMembers.map(async (member, index) => {
|
|
12846
|
+
const request = requestByCallId.get(member.memberId);
|
|
12236
12847
|
if (!request) {
|
|
12237
12848
|
throw new Error(
|
|
12238
12849
|
`Native batch Tool Call lost member ${member.memberId}.`,
|
|
@@ -12262,7 +12873,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12262
12873
|
);
|
|
12263
12874
|
},
|
|
12264
12875
|
});
|
|
12265
|
-
return { batch, result };
|
|
12876
|
+
return { batch: plannedBatch, result };
|
|
12266
12877
|
},
|
|
12267
12878
|
onChunkComplete: async (entries) => {
|
|
12268
12879
|
for (const entry of entries) {
|
|
@@ -13776,9 +14387,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13776
14387
|
const asyncContract = options?.skipAsyncOperationResolution
|
|
13777
14388
|
? null
|
|
13778
14389
|
: ((await this.#options.getToolAsyncOperation?.(toolId)) ?? null);
|
|
14390
|
+
const asyncOperationPoll = options?.skipAsyncOperationResolution
|
|
14391
|
+
? null
|
|
14392
|
+
: ((await this.#options.getToolAsyncOperationPoll?.(toolId)) ?? null);
|
|
13779
14393
|
const shouldWaitForAsyncOperation = Boolean(
|
|
13780
14394
|
asyncContract && asyncOperationShouldWait(asyncContract, input),
|
|
13781
14395
|
);
|
|
14396
|
+
// An explicitly opted-in caller-managed launch gives the Play author the
|
|
14397
|
+
// provider handle and leaves polling to the authored graph. Do not let a
|
|
14398
|
+
// provider's legacy synchronous helper turn a wide start wave into
|
|
14399
|
+
// long-lived HTTP calls before that graph can make its own poll.
|
|
14400
|
+
const shouldPreferAsyncProviderExecution =
|
|
14401
|
+
asyncContract?.wait.kind === 'caller_managed' &&
|
|
14402
|
+
asyncContract.lifecycle.startAction === toolId &&
|
|
14403
|
+
asyncContract.launch?.preferAsync === true;
|
|
13782
14404
|
const executionAuthScopeDigest = options?.executionAuthScopeDigest
|
|
13783
14405
|
? options.executionAuthScopeDigest.trim() || null
|
|
13784
14406
|
: ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null);
|
|
@@ -14012,6 +14634,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14012
14634
|
...(shouldWaitForAsyncOperation
|
|
14013
14635
|
? { async_operation_wait: true }
|
|
14014
14636
|
: {}),
|
|
14637
|
+
...(shouldPreferAsyncProviderExecution
|
|
14638
|
+
? { prefer_async: true }
|
|
14639
|
+
: {}),
|
|
14640
|
+
...(asyncOperationPoll
|
|
14641
|
+
? {
|
|
14642
|
+
async_operation_poll_timeout_ms:
|
|
14643
|
+
asyncOperationPoll.checkTimeoutMs,
|
|
14644
|
+
}
|
|
14645
|
+
: {}),
|
|
14015
14646
|
...(usesDurableInvocationFence
|
|
14016
14647
|
? {
|
|
14017
14648
|
invocation_fence_version:
|
|
@@ -14599,11 +15230,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14599
15230
|
}
|
|
14600
15231
|
if (
|
|
14601
15232
|
failure.shouldRetry &&
|
|
14602
|
-
(
|
|
15233
|
+
(failure.knownNotDispatched ||
|
|
15234
|
+
failure.reason === 'gateway_invocation_in_progress' ||
|
|
15235
|
+
ambiguousTransportReplayAllowed ||
|
|
14603
15236
|
retrySafeTransientHttp ||
|
|
14604
15237
|
fixtureReplaySafe)
|
|
14605
15238
|
) {
|
|
14606
|
-
|
|
15239
|
+
// A gateway follower is retrying transport delivery to the
|
|
15240
|
+
// same durable invocation. Advancing the logical attempt here
|
|
15241
|
+
// would mint a new invocation key while the original owner is
|
|
15242
|
+
// still running, defeating the gateway's replay fence.
|
|
15243
|
+
if (failure.reason !== 'gateway_invocation_in_progress') {
|
|
15244
|
+
invocationAttempt += 1;
|
|
15245
|
+
}
|
|
14607
15246
|
if (failure.chargeRetryBudget) {
|
|
14608
15247
|
await this.governor.chargeBudget('retry');
|
|
14609
15248
|
}
|