deepline 0.1.227 → 0.1.229

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +76 -1842
  2. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -113
  3. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +144 -1000
  4. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts +81 -0
  5. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +1 -50
  6. package/dist/bundling-sources/sdk/src/client.ts +24 -2
  7. package/dist/bundling-sources/sdk/src/play.ts +2 -2
  8. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  9. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +382 -31
  10. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +84 -79
  11. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +201 -705
  12. package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +3 -10
  13. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -2
  14. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -47
  15. package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +0 -2
  16. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +12 -200
  17. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +2 -17
  18. package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts +3 -9
  19. package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +6 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +28 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts +16 -64
  22. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +26 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +5 -2
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +0 -4
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +81 -18
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +0 -36
  27. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +3 -21
  28. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +9 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/transport-error-diagnostics.ts +63 -0
  30. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -0
  31. package/dist/cli/index.js +78 -11
  32. package/dist/cli/index.mjs +78 -11
  33. package/dist/{compiler-manifest-BhgZ23A4.d.ts → compiler-manifest-CYcwzSOJ.d.mts} +2 -0
  34. package/dist/{compiler-manifest-BhgZ23A4.d.mts → compiler-manifest-CYcwzSOJ.d.ts} +2 -0
  35. package/dist/index.d.mts +3 -3
  36. package/dist/index.d.ts +3 -3
  37. package/dist/index.js +14 -5
  38. package/dist/index.mjs +14 -5
  39. package/dist/plays/bundle-play-file.d.mts +2 -2
  40. package/dist/plays/bundle-play-file.d.ts +2 -2
  41. package/package.json +1 -1
  42. package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +0 -108
  43. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +0 -374
  44. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +0 -203
  45. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +0 -14
  46. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +0 -121
@@ -42,17 +42,7 @@ import {
42
42
  mapRowOutcomeRuntimeFields,
43
43
  resolveMapRowOutcomeKey,
44
44
  } from './map-row-outcome';
45
- import { buildChildRunId } from './child-run-id';
46
- import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
47
- import {
48
- ABSURD_RELEASE_OVERRIDE_HEADER,
49
- RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER,
50
- SYNTHETIC_RUN_HEADER,
51
- } from './coordinator-headers';
52
- import {
53
- PLAY_RUNTIME_API_COMPAT_PATH,
54
- PLAY_RUNTIME_API_CURRENT_PATH,
55
- } from './runtime-api-paths';
45
+ import { PLAY_RUNTIME_API_COMPAT_PATH } from './runtime-api-paths';
56
46
  import {
57
47
  createRootRunExecutionScope,
58
48
  deriveChildRunExecutionScope,
@@ -108,6 +98,10 @@ import {
108
98
  ToolExecuteAuthScopeChangedError,
109
99
  } from './tool-execute-retry-policy';
110
100
  import { ToolHttpError } from './tool-http-errors';
101
+ import {
102
+ describeTransportError,
103
+ transportGatewayOriginForDiagnostic,
104
+ } from './transport-error-diagnostics';
111
105
  import {
112
106
  buildDurableCtxCallCacheKey,
113
107
  buildDurableRunPlayInvocationScope,
@@ -241,8 +235,8 @@ import {
241
235
  } from './step-program-dataset-builder';
242
236
  import { readRuntimeSheetDatasetRows } from './runtime-api';
243
237
  import {
238
+ ctxRunPlayInlineOnlyMessage,
244
239
  resolveChildExecutionStrategy,
245
- inlineChildScheduledInRowMessage,
246
240
  } from './child-execution-strategy';
247
241
 
248
242
  type ResolvedPlayExecutor = (
@@ -263,6 +257,11 @@ const rowContext = new AsyncLocalStorage<{
263
257
  tableNamespace?: string;
264
258
  rowKey?: string;
265
259
  mapScope?: MapExecutionScope;
260
+ /**
261
+ * Per-resolver inline-child fan-width guard. This stays in the async row
262
+ * scope, so it becomes collectible as soon as the resolver settles.
263
+ */
264
+ inlineChildInvocationNamespaces?: Set<string>;
266
265
  }>();
267
266
  type InlineCompositionStore = {
268
267
  context: PlayContextImpl;
@@ -303,19 +302,11 @@ const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
303
302
  // provider 502; require both the branded page and our development tunnel host.
304
303
  const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
305
304
  /\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
306
- const CHILD_PLAY_LAUNCH_WRITE_CONFLICT_RETRY_DELAYS_MS = [
307
- 50, 100, 200, 400, 800,
308
- ] as const;
309
- const CHILD_PLAY_LAUNCH_ADMISSION_RETRY_DELAYS_MS = [
310
- 250, 500, 1_000, 2_000, 4_000, 8_000, 8_000, 8_000, 8_000, 8_000, 8_000,
311
- 8_000, 8_000, 8_000, 8_000,
312
- ] as const;
313
305
  const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
314
306
  const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
315
307
  // Per-row sanity cap on distinct inline child invocations. `maxPlayCallDepth`
316
- // (governor) already bounds nesting; this bounds fan-WIDTH from a single row so
317
- // a resolver that loops `ctx.runPlay` unbounded fails loudly instead of leaking
318
- // composition namespaces. Well above any legitimate authored per-row fan-out.
308
+ // (governor) already bounds nesting; this bounds fan-WIDTH from one active row
309
+ // resolver. The dedupe set is row-local, never retained for the full run.
319
310
  const MAX_INLINE_CHILD_INVOCATIONS_PER_ROW = 512;
320
311
  // Bound the retained failure detail carried in parent aggregates so a fully
321
312
  // failing large map cannot grow the progress event without limit.
@@ -350,44 +341,6 @@ export function chunkArrayForReceiptRequest<T>(items: readonly T[]): T[][] {
350
341
  type SafeFetchModule = typeof import('@shared_libs/security/safe-fetch');
351
342
  let safeFetchModule: Promise<SafeFetchModule> | null = null;
352
343
 
353
- function sleepTransientRuntimeMs(ms: number): Promise<void> {
354
- return new Promise((resolve) => setTimeout(resolve, ms));
355
- }
356
-
357
- function isConvexWriteConflictMessage(message: string): boolean {
358
- return (
359
- message.includes('OptimisticConcurrencyControlFailure') ||
360
- /changed while this mutation was being run/i.test(message) ||
361
- /write conflict/i.test(message)
362
- );
363
- }
364
-
365
- function isChildPlayLaunchAdmissionMessage(message: string): boolean {
366
- return /too many play runs (?:are )?(?:starting|active)/i.test(message);
367
- }
368
-
369
- function childPlayLaunchRetryDelayMs(input: {
370
- status: number;
371
- message: string;
372
- attempt: number;
373
- }): number | null {
374
- if (
375
- input.status === 429 &&
376
- isChildPlayLaunchAdmissionMessage(input.message)
377
- ) {
378
- return CHILD_PLAY_LAUNCH_ADMISSION_RETRY_DELAYS_MS[input.attempt] ?? null;
379
- }
380
- if (
381
- ![409, 429, 500, 502, 503, 504].includes(input.status) ||
382
- !isConvexWriteConflictMessage(input.message)
383
- ) {
384
- return null;
385
- }
386
- return (
387
- CHILD_PLAY_LAUNCH_WRITE_CONFLICT_RETRY_DELAYS_MS[input.attempt] ?? null
388
- );
389
- }
390
-
391
344
  export async function waitForNodeRuntimeMapRowsVisible(input: {
392
345
  mapName: string;
393
346
  tableNamespace: string;
@@ -1237,13 +1190,6 @@ export class PlayContextImpl {
1237
1190
  eventKey: string;
1238
1191
  timeoutMs: number;
1239
1192
  }> = [];
1240
- private pendingChildPlayBoundaries: Array<{
1241
- boundaryId: string;
1242
- childRunId: string;
1243
- childPlayName: string;
1244
- }> = [];
1245
- private activeScheduledChildPlayCalls = 0;
1246
- private scheduledChildPlayQuiescenceWaiters: Array<() => void> = [];
1247
1193
  private processedRowCount = 0;
1248
1194
  private sleepBoundaryIndex = 0;
1249
1195
  private fetchCallIndex = 0;
@@ -1251,19 +1197,6 @@ export class PlayContextImpl {
1251
1197
  createSecretRedactionContext();
1252
1198
  private mapInvocationIndex = 0;
1253
1199
  private readonly stepCallIndexByKey = new Map<string, number>();
1254
- /**
1255
- * Registry of distinct inline child invocation namespaces seen by this
1256
- * context. Inline child composition is a function call, not a run; each
1257
- * invocation's namespace (`child:<playName>#<callKey>[@<rowScope>]`) scopes
1258
- * its receipts under the parent run. See ADR 0013.
1259
- */
1260
- private readonly inlineChildInvocationOrdinals = new Map<string, number>();
1261
- /**
1262
- * Distinct inline child invocations per row scope (top-level counts as one
1263
- * scope). Backs the per-row fan-width sanity cap so a resolver looping
1264
- * `ctx.runPlay` unbounded fails loudly instead of leaking namespaces.
1265
- */
1266
- private readonly inlineChildInvocationsPerRow = new Map<string, number>();
1267
1200
  /**
1268
1201
  * Parent-level inline-child aggregates. Maintained only by the single-writer
1269
1202
  * progress path (never per child) so concurrent fan-out cannot contend. See
@@ -1474,6 +1407,15 @@ export class PlayContextImpl {
1474
1407
  return active?.context === this ? active : null;
1475
1408
  }
1476
1409
 
1410
+ /** Enforce the scalar inline-child contract at dynamic API boundaries. */
1411
+ private assertInlineChildContract(
1412
+ reason: 'dataset_child' | 'suspending_child',
1413
+ ): void {
1414
+ const composition = this.activeInlineComposition;
1415
+ if (!composition) return;
1416
+ throw new Error(ctxRunPlayInlineOnlyMessage(composition.playName, reason));
1417
+ }
1418
+
1477
1419
  private get currentExecutionScope(): RunExecutionScope {
1478
1420
  return this.activeInlineComposition?.executionScope ?? this.executionScope;
1479
1421
  }
@@ -1495,100 +1437,9 @@ export class PlayContextImpl {
1495
1437
  return active ? active.staticPipeline : this.#options.staticPipeline;
1496
1438
  }
1497
1439
 
1498
- private shouldLaunchChildPlaysThroughScheduler(): boolean {
1499
- return (
1500
- this.#options.requireSharedRateState === true &&
1501
- this.#options.durableBoundaries === true
1502
- );
1503
- }
1504
-
1505
- private childPlayBoundaryId(childRunId: string): string {
1506
- return this.durableBoundaryId(`runPlay:${childRunId}`);
1507
- }
1508
-
1509
- private recordPendingChildPlayBoundary(input: {
1510
- boundaryId: string;
1511
- childRunId: string;
1512
- childPlayName: string;
1513
- }): void {
1514
- this.pendingChildPlayBoundaries.push(input);
1515
- }
1516
-
1517
- private beginScheduledChildPlayCall(): { release: () => void } {
1518
- this.activeScheduledChildPlayCalls += 1;
1519
- let released = false;
1520
- return {
1521
- release: () => {
1522
- if (released) return;
1523
- released = true;
1524
- this.activeScheduledChildPlayCalls = Math.max(
1525
- 0,
1526
- this.activeScheduledChildPlayCalls - 1,
1527
- );
1528
- if (this.activeScheduledChildPlayCalls !== 0) return;
1529
- for (const resolve of this.scheduledChildPlayQuiescenceWaiters.splice(
1530
- 0,
1531
- )) {
1532
- resolve();
1533
- }
1534
- },
1535
- };
1536
- }
1537
-
1538
- private async waitForScheduledChildPlayQuiescence(): Promise<void> {
1539
- if (this.activeScheduledChildPlayCalls === 0) return;
1540
- await new Promise<void>((resolve) => {
1541
- this.scheduledChildPlayQuiescenceWaiters.push(resolve);
1542
- });
1543
- }
1544
-
1545
- private consumePendingChildPlaySuspension(): PlayExecutionSuspension | null {
1546
- if (this.pendingChildPlayBoundaries.length === 0) return null;
1547
- const boundaries = [
1548
- ...new Map(
1549
- this.pendingChildPlayBoundaries.map((boundary) => [
1550
- boundary.boundaryId,
1551
- boundary,
1552
- ]),
1553
- ).values(),
1554
- ];
1555
- this.pendingChildPlayBoundaries = [];
1556
- if (boundaries.length === 1) {
1557
- const boundary = boundaries[0]!;
1558
- return {
1559
- kind: 'child_play',
1560
- boundaryId: boundary.boundaryId,
1561
- childRunId: boundary.childRunId,
1562
- childPlayName: boundary.childPlayName,
1563
- };
1564
- }
1565
- return {
1566
- kind: 'child_play_batch',
1567
- boundaries,
1568
- };
1569
- }
1570
-
1571
- private childPlayBoundaryOutput<TOutput>(input: {
1572
- boundaryId: string;
1573
- childRunId: string;
1574
- childPlayName: string;
1575
- }): { found: true; output: TOutput } | { found: false } {
1576
- const existing = this.checkpoint.resolvedBoundaries?.[input.boundaryId];
1577
- if (!existing || existing.kind !== 'child_play') return { found: false };
1578
- if (existing.childRunId !== input.childRunId) return { found: false };
1579
- if (existing.status === 'completed') {
1580
- return { found: true, output: existing.output as TOutput };
1581
- }
1582
- const message =
1583
- typeof existing.error === 'string' && existing.error.trim()
1584
- ? existing.error.trim()
1585
- : `Child play ${input.childPlayName} (${input.childRunId}) ${existing.status}.`;
1586
- throw new Error(message);
1587
- }
1588
-
1589
1440
  private recordPlayCallStep(input: {
1590
1441
  playId: string;
1591
- execution?: 'inline' | 'child-workflow';
1442
+ execution?: 'inline';
1592
1443
  description?: string | null;
1593
1444
  nestedSteps?: PlayStep[];
1594
1445
  }): void {
@@ -1612,186 +1463,6 @@ export class PlayContextImpl {
1612
1463
  );
1613
1464
  }
1614
1465
 
1615
- private async runtimeApiHeaders(): Promise<Record<string, string>> {
1616
- if (!this.#options.executorToken?.trim()) {
1617
- throw new Error(
1618
- 'ctx.runPlay scheduled child launch requires an executor token.',
1619
- );
1620
- }
1621
- return {
1622
- Authorization: `Bearer ${this.#options.executorToken.trim()}`,
1623
- 'Content-Type': 'application/json',
1624
- ...(this.#options.runtimeSchedulerSchema?.trim()
1625
- ? {
1626
- [RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER]:
1627
- this.#options.runtimeSchedulerSchema.trim(),
1628
- [SYNTHETIC_RUN_HEADER]: '1',
1629
- }
1630
- : {}),
1631
- // Defense-in-depth: pin the child to the parent worker's release lane so
1632
- // a child always finishes on the release that launched its parent, even
1633
- // if the resolver default (schema-derived) would drift. The run route
1634
- // honors this only for internal ctx.runPlay child launches.
1635
- ...(this.#options.absurdReleaseId?.trim()
1636
- ? {
1637
- [ABSURD_RELEASE_OVERRIDE_HEADER]:
1638
- this.#options.absurdReleaseId.trim(),
1639
- }
1640
- : {}),
1641
- ...(await this.vercelProtectionHeaders()),
1642
- };
1643
- }
1644
-
1645
- private runtimeApiBaseUrl(): string {
1646
- const baseUrl = this.#options.baseUrl?.trim();
1647
- if (!baseUrl) {
1648
- throw new Error('ctx.runPlay scheduled child launch requires baseUrl.');
1649
- }
1650
- return baseUrl.replace(/\/$/, '');
1651
- }
1652
-
1653
- private async launchScheduledChildPlay(input: {
1654
- key: string;
1655
- childRunId: string;
1656
- childPlayName: string;
1657
- childGovernance: GovernanceSnapshot;
1658
- childInput: Record<string, unknown>;
1659
- description?: string | null;
1660
- }): Promise<void> {
1661
- const parentGovernance = this.currentGovernance;
1662
- const parentPlayName =
1663
- parentGovernance.currentPlayId ||
1664
- this.currentPlayName ||
1665
- this.#options.playId ||
1666
- 'anonymous-play';
1667
- const governance: PlayCallGovernanceSnapshot = {
1668
- rootRunId: parentGovernance.rootRunId,
1669
- parentRunId: parentGovernance.currentRunId,
1670
- parentPlayName,
1671
- key: input.key,
1672
- ancestryPlayIds: [...parentGovernance.ancestryPlayIds],
1673
- callDepth: input.childGovernance.callDepth,
1674
- playCallCount: input.childGovernance.playCallCount,
1675
- toolCallCount: input.childGovernance.toolCallCount,
1676
- retryCount: input.childGovernance.retryCount,
1677
- descendantCount: input.childGovernance.descendantCount,
1678
- waterfallStepExecutions: input.childGovernance.waterfallStepExecutions,
1679
- };
1680
- const requestBody = JSON.stringify({
1681
- name: input.childPlayName,
1682
- workflowId: input.childRunId,
1683
- input: input.childInput,
1684
- // Inherit the parent's scheduler profile so an `absurd` parent launches
1685
- // its child on the same scheduler. Absent threading defaults to absurd.
1686
- profile: this.#options.childRunProfile?.trim() || 'absurd',
1687
- waitForCompletionMs: 0,
1688
- internalRunPlay: {
1689
- ...governance,
1690
- description: input.description ?? null,
1691
- },
1692
- });
1693
- for (let attempt = 0; ; attempt += 1) {
1694
- const response = await fetch(
1695
- `${this.runtimeApiBaseUrl()}/api/v2/plays/run`,
1696
- {
1697
- method: 'POST',
1698
- headers: await this.runtimeApiHeaders(),
1699
- body: requestBody,
1700
- },
1701
- );
1702
- if (response.ok) {
1703
- if (attempt > 0) {
1704
- this.log(
1705
- `ctx.runPlay(${input.key}): child launch recovered after ${attempt} retry(s)`,
1706
- );
1707
- }
1708
- return;
1709
- }
1710
- const body = (await response.json().catch(() => null)) as {
1711
- error?: unknown;
1712
- } | null;
1713
- const message =
1714
- typeof body?.error === 'string' && body.error.trim()
1715
- ? body.error.trim()
1716
- : response.statusText;
1717
- const retryDelayMs = childPlayLaunchRetryDelayMs({
1718
- status: response.status,
1719
- message,
1720
- attempt,
1721
- });
1722
- if (retryDelayMs === null) {
1723
- throw new Error(
1724
- `ctx.runPlay(${input.key}) failed to launch child play "${input.childPlayName}": ${message} (status ${response.status}).`,
1725
- );
1726
- }
1727
- const retryReason =
1728
- response.status === 429 && isChildPlayLaunchAdmissionMessage(message)
1729
- ? 'start admission backpressure'
1730
- : 'transient write conflict';
1731
- this.log(
1732
- `ctx.runPlay(${input.key}): retrying child launch after ${retryReason} (attempt ${attempt + 1})`,
1733
- );
1734
- await sleepTransientRuntimeMs(retryDelayMs);
1735
- }
1736
- }
1737
-
1738
- private async readScheduledChildTerminal<TOutput>(input: {
1739
- childRunId: string;
1740
- childPlayName: string;
1741
- }): Promise<
1742
- | { status: 'completed'; output: TOutput }
1743
- | { status: 'failed' | 'cancelled'; error: string }
1744
- | null
1745
- > {
1746
- const response = await fetch(
1747
- `${this.runtimeApiBaseUrl()}${PLAY_RUNTIME_API_COMPAT_PATH}`,
1748
- {
1749
- method: 'POST',
1750
- headers: await this.runtimeApiHeaders(),
1751
- body: JSON.stringify({
1752
- action: 'read_child_run_terminal_snapshot',
1753
- parentRunId: this.currentGovernance.currentRunId,
1754
- childRunId: input.childRunId,
1755
- childPlayName: input.childPlayName,
1756
- }),
1757
- },
1758
- );
1759
- const body = (await response.json().catch(() => null)) as {
1760
- state?: {
1761
- data?: {
1762
- status?: unknown;
1763
- result?: unknown;
1764
- error?: unknown;
1765
- };
1766
- };
1767
- error?: unknown;
1768
- } | null;
1769
- if (!response.ok) {
1770
- const message =
1771
- typeof body?.error === 'string' && body.error.trim()
1772
- ? body.error.trim()
1773
- : response.statusText;
1774
- throw new Error(
1775
- `ctx.runPlay failed to read child terminal snapshot for "${input.childPlayName}": ${message} (status ${response.status}).`,
1776
- );
1777
- }
1778
- const data = body?.state?.data;
1779
- const status = String(data?.status ?? '').toLowerCase();
1780
- if (status === 'completed') {
1781
- return { status, output: data?.result as TOutput };
1782
- }
1783
- if (status === 'failed' || status === 'cancelled') {
1784
- return {
1785
- status,
1786
- error:
1787
- typeof data?.error === 'string' && data.error.trim()
1788
- ? data.error.trim()
1789
- : `Child play ${input.childPlayName} (${input.childRunId}) ${status}.`,
1790
- };
1791
- }
1792
- return null;
1793
- }
1794
-
1795
1466
  private emitScopedRowUpdate(
1796
1467
  key: string | null,
1797
1468
  tableNamespace: string | null,
@@ -3699,6 +3370,7 @@ export class PlayContextImpl {
3699
3370
  path: string,
3700
3371
  _options?: CsvOptions,
3701
3372
  ): Promise<PlayDataset<Record<string, unknown>>> {
3373
+ this.assertInlineChildContract('dataset_child');
3702
3374
  void _options;
3703
3375
  // In cloud mode, CSV data is passed in — path is just a label
3704
3376
  // The activity loads the actual data before creating the ctx
@@ -3727,6 +3399,7 @@ export class PlayContextImpl {
3727
3399
  input?: MapFieldDefinition<T, TColumns> | RuntimeStepProgram,
3728
3400
  options?: DatasetOptions<T>,
3729
3401
  ): RuntimeDatasetBuilder<T> | Promise<PlayDataset<Record<string, unknown>>> {
3402
+ this.assertInlineChildContract('dataset_child');
3730
3403
  if (rowContext.getStore()) {
3731
3404
  throw new Error(
3732
3405
  'Nested ctx.dataset() is not supported. Flatten your fields into one dataset, or keep custom per-row logic inside a single column.',
@@ -6315,13 +5988,6 @@ export class PlayContextImpl {
6315
5988
  this.toolCallQueue.length === 0 &&
6316
5989
  inFlightToolExecutions.size === 0
6317
5990
  ) {
6318
- if (!this.activeDatasetStep) {
6319
- const childPlaySuspension =
6320
- this.consumePendingChildPlaySuspension();
6321
- if (childPlaySuspension) {
6322
- throw new PlayExecutionSuspendedError(childPlaySuspension);
6323
- }
6324
- }
6325
5991
  break;
6326
5992
  }
6327
5993
 
@@ -6723,6 +6389,7 @@ export class PlayContextImpl {
6723
6389
  input: Record<string, unknown>,
6724
6390
  handler: IntegrationEventWaitHandler,
6725
6391
  ): Promise<unknown> {
6392
+ this.assertInlineChildContract('suspending_child');
6726
6393
  if (!this.#options.durableBoundaries) {
6727
6394
  throw new Error(`${toolId} requires durable play boundaries.`);
6728
6395
  }
@@ -6841,328 +6508,136 @@ export class PlayContextImpl {
6841
6508
  if (!resolvedName.trim()) {
6842
6509
  throw new Error('ctx.runPlay(...) requires a resolvable play name.');
6843
6510
  }
6844
- let scheduledChildPlayCall: { release(): void } | null = null;
6511
+ if (!this.#options.resolvePlay) {
6512
+ throw new Error(
6513
+ 'ctx.runPlay(...) is unavailable because no play resolver was configured.',
6514
+ );
6515
+ }
6516
+ const resolvedPlay = await this.#options.resolvePlay(resolvedName);
6517
+ if (!resolvedPlay) {
6518
+ throw new Error(
6519
+ `Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
6520
+ );
6521
+ }
6522
+ const childExecutionDecision = resolveChildExecutionStrategy({
6523
+ pipeline: resolvedPlay.staticPipeline,
6524
+ timeoutMs: options?.timeoutMs,
6525
+ hasExplicitTimeout:
6526
+ options != null && Object.hasOwn(options, 'timeoutMs'),
6527
+ execution: options?.execution,
6528
+ childPlayName: resolvedName,
6529
+ });
6530
+ this.log(
6531
+ `ctx.runPlay(${normalizedKey}): ${childExecutionDecision.strategy} (${childExecutionDecision.reason})`,
6532
+ );
6845
6533
 
6846
- try {
6847
- if (!this.#options.resolvePlay) {
6848
- throw new Error(
6849
- 'ctx.runPlay(...) is unavailable because no play resolver was configured.',
6850
- );
6851
- }
6852
- const resolvePlay = this.#options.resolvePlay;
6853
- const resolvedPlay = await resolvePlay(resolvedName);
6854
- if (!resolvedPlay) {
6855
- throw new Error(
6856
- `Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
6857
- );
6858
- }
6859
- const childExecutionDecision = resolveChildExecutionStrategy({
6860
- pipeline: resolvedPlay.staticPipeline,
6861
- execution: options?.execution,
6534
+ const compositionNamespace = this.inlineChildCompositionNamespace(
6535
+ resolvedName,
6536
+ normalizedKey,
6537
+ );
6538
+ const inlineChildGovernor =
6539
+ await this.currentExecutionGovernor.forkInlineChild({
6862
6540
  childPlayName: resolvedName,
6541
+ childRunId: compositionNamespace,
6863
6542
  });
6864
- const launchThroughScheduler =
6865
- this.shouldLaunchChildPlaysThroughScheduler() &&
6866
- childExecutionDecision.strategy === 'scheduled';
6867
- scheduledChildPlayCall = launchThroughScheduler
6868
- ? this.beginScheduledChildPlayCall()
6869
- : null;
6870
- this.log(
6871
- `ctx.runPlay(${normalizedKey}): ${childExecutionDecision.strategy} (${childExecutionDecision.reason})`,
6872
- );
6873
- const runPlayRowScope = rowContext.getStore();
6874
- // Runtime enforcement of the row-scoped-scheduled-child rule for children
6875
- // whose name is only known at runtime (static preflight cannot see them).
6876
- // A scheduling-required child fanned out per row is the 100,000-run-doc
6877
- // failure mode the inline-child cutover removes. Fail on the FIRST row,
6878
- // loud, with the SAME message the static preflight emits. See ADR 0013.
6879
- //
6880
- // Mirrors the static check's proof standard: reject when the child's
6881
- // resolved contract PROVES it needs scheduling (dataset/csv, event-wait,
6882
- // explicit timeout), or when this call would actually launch a durable
6883
- // scheduled child run per row. A missing static contract alone is not
6884
- // proof — in-process fallback runtimes legitimately run contract-less
6885
- // scalar children inline.
6886
- const provablySchedulingRequired =
6887
- childExecutionDecision.strategy === 'scheduled' &&
6888
- childExecutionDecision.reason !== 'missing_static_contract';
6889
- if (
6890
- runPlayRowScope &&
6891
- (provablySchedulingRequired || launchThroughScheduler)
6892
- ) {
6893
- throw new Error(
6894
- inlineChildScheduledInRowMessage(
6895
- resolvedName,
6896
- runPlayRowScope.tableNamespace ??
6897
- runPlayRowScope.fieldName ??
6898
- 'dataset',
6899
- ),
6900
- );
6543
+ const rowStore = rowContext.getStore();
6544
+ const producer = {
6545
+ kind: 'play' as const,
6546
+ id: normalizedKey,
6547
+ playId: resolvedName,
6548
+ displayName: displayNameFromProducerId(resolvedName),
6549
+ runId: compositionNamespace,
6550
+ };
6551
+ try {
6552
+ if (rowStore) {
6553
+ this.emitScopedFieldMetaUpdate({
6554
+ rowId: rowStore.rowId,
6555
+ key: rowStore.rowKey ?? null,
6556
+ tableNamespace: rowStore.tableNamespace ?? null,
6557
+ fieldName: rowStore.fieldName,
6558
+ status: 'running',
6559
+ rowStatus: 'running',
6560
+ stage: resolvedName,
6561
+ provider: 'deepline_native',
6562
+ error: null,
6563
+ producer,
6564
+ dataPatch: {},
6565
+ });
6901
6566
  }
6902
- const runPlayInvocationScope = buildDurableRunPlayInvocationScope({
6903
- childPlayName: resolvedName,
6904
- input,
6905
- rowScope: runPlayRowScope
6906
- ? {
6907
- fieldName: runPlayRowScope.fieldName,
6908
- rowKey: runPlayRowScope.rowKey ?? null,
6909
- tableNamespace: runPlayRowScope.tableNamespace ?? null,
6910
- }
6911
- : null,
6912
- });
6913
-
6914
- const executePlayCall = async (): Promise<TOutput> => {
6915
- // Scheduled children reserve the parent's durable play/descendant
6916
- // budgets. Inline scalar children are function composition: their
6917
- // Governor view changes only local lineage for recursion/depth checks
6918
- // and delegates tool admission, pacing, and tool budgets to the parent.
6919
- //
6920
- // The SCHEDULED substrate still allocates a durable child run id (through
6921
- // the shared canonical builder so in-process and the workers_edge
6922
- // coordinator produce the same FORMAT `play/<slug>/run/child-<digest>`).
6923
- // INLINE scalar children are pure function composition (ADR 0013): they
6924
- // carry NO durable run identity — no child run id, no playRuns doc, no
6925
- // settlement. Their only identity is a replay-stable, call-site scoped
6926
- // composition namespace (`child:<playName>#<ordinal>`) used to scope tool
6927
- // receipts under the PARENT run. See child-composition-namespace below.
6928
- const parentGovernor = this.currentExecutionGovernor;
6929
- const inlineChildGovernor = launchThroughScheduler
6930
- ? null
6931
- : await parentGovernor.forkInlineChild({
6932
- childPlayName: resolvedName,
6933
- childRunId: this.inlineChildCompositionNamespace(
6934
- resolvedName,
6935
- normalizedKey,
6936
- ),
6937
- });
6938
- const childGovernance = launchThroughScheduler
6939
- ? await parentGovernor.forkChild({
6940
- childPlayName: resolvedName,
6941
- childRunId: buildChildRunId({
6942
- childPlayName: resolvedName,
6943
- parentRunId: this.currentRunId,
6944
- parentPlayName: this.#options.playName,
6945
- key: normalizedKey,
6946
- runPlaySemanticKey: runPlayInvocationScope,
6947
- input,
6948
- graphHash: null,
6949
- }),
6950
- })
6951
- : inlineChildGovernor!.snapshot();
6952
- const childPlaySlot = launchThroughScheduler
6953
- ? await parentGovernor.acquireChildSubmitSlot()
6954
- : null;
6955
- const rowStore = rowContext.getStore();
6956
- const producer = {
6957
- kind: 'play' as const,
6958
- id: normalizedKey,
6567
+ // Inline composition has no child run, launch request, terminal poll, or
6568
+ // scheduler slot. It shares the caller's execution and tool governors.
6569
+ const childExecutionScope = deriveChildRunExecutionScope(
6570
+ this.currentExecutionScope,
6571
+ {
6572
+ placement: 'inline',
6573
+ runId: compositionNamespace,
6959
6574
  playId: resolvedName,
6960
- displayName: displayNameFromProducerId(resolvedName),
6961
- // For scheduled children this is the durable child run id; for inline
6962
- // children it is the composition namespace (`child:<playName>#<ord>`).
6963
- runId: childGovernance.currentRunId,
6964
- };
6965
- try {
6966
- if (rowStore) {
6967
- this.emitScopedFieldMetaUpdate({
6968
- rowId: rowStore.rowId,
6969
- key: rowStore.rowKey ?? null,
6970
- tableNamespace: rowStore.tableNamespace ?? null,
6971
- fieldName: rowStore.fieldName,
6972
- status: 'running',
6973
- rowStatus: 'running',
6974
- stage: resolvedName,
6975
- provider: 'deepline_native',
6976
- error: null,
6977
- producer,
6978
- dataPatch: {},
6979
- });
6980
- }
6981
- if (launchThroughScheduler) {
6982
- const boundaryId = this.childPlayBoundaryId(
6983
- childGovernance.currentRunId,
6984
- );
6985
- const checkpointResult = this.childPlayBoundaryOutput<TOutput>({
6986
- boundaryId,
6987
- childRunId: childGovernance.currentRunId,
6988
- childPlayName: resolvedName,
6989
- });
6990
- if (checkpointResult.found) {
6991
- if (rowStore) {
6992
- this.emitScopedFieldMetaUpdate({
6993
- rowId: rowStore.rowId,
6994
- key: rowStore.rowKey ?? null,
6995
- tableNamespace: rowStore.tableNamespace ?? null,
6996
- fieldName: rowStore.fieldName,
6997
- status: 'completed',
6998
- rowStatus: 'running',
6999
- stage: resolvedName,
7000
- provider: 'deepline_native',
7001
- error: null,
7002
- producer,
7003
- dataPatch: {},
7004
- });
7005
- }
7006
- this.recordPlayCallStep({
7007
- playId: resolvedName,
7008
- execution: options?.execution,
7009
- description: options?.description,
7010
- });
7011
- return checkpointResult.output;
7012
- }
7013
-
7014
- await this.launchScheduledChildPlay({
7015
- key: normalizedKey,
7016
- childRunId: childGovernance.currentRunId,
7017
- childPlayName: resolvedName,
7018
- childGovernance,
7019
- childInput: input,
7020
- description: options?.description ?? null,
7021
- });
7022
- childPlaySlot?.release();
7023
- const terminal = await this.readScheduledChildTerminal<TOutput>({
7024
- childRunId: childGovernance.currentRunId,
7025
- childPlayName: resolvedName,
7026
- });
7027
- if (terminal?.status === 'completed') {
7028
- this.checkpoint.resolvedBoundaries = {
7029
- ...(this.checkpoint.resolvedBoundaries ?? {}),
7030
- [boundaryId]: {
7031
- kind: 'child_play',
7032
- childRunId: childGovernance.currentRunId,
7033
- childPlayName: resolvedName,
7034
- status: 'completed',
7035
- output: terminal.output,
7036
- completedAt: Date.now(),
7037
- },
7038
- };
7039
- if (rowStore) {
7040
- this.emitScopedFieldMetaUpdate({
7041
- rowId: rowStore.rowId,
7042
- key: rowStore.rowKey ?? null,
7043
- tableNamespace: rowStore.tableNamespace ?? null,
7044
- fieldName: rowStore.fieldName,
7045
- status: 'completed',
7046
- rowStatus: 'running',
7047
- stage: resolvedName,
7048
- provider: 'deepline_native',
7049
- error: null,
7050
- producer,
7051
- dataPatch: {},
7052
- });
7053
- }
7054
- this.recordPlayCallStep({
7055
- playId: resolvedName,
7056
- execution: options?.execution,
7057
- description: options?.description,
7058
- });
7059
- return terminal.output;
7060
- }
7061
- if (terminal) {
7062
- throw new Error(terminal.error);
7063
- }
7064
-
7065
- const suspension = {
7066
- kind: 'child_play',
7067
- boundaryId,
7068
- childRunId: childGovernance.currentRunId,
7069
- childPlayName: resolvedName,
7070
- } as const;
7071
- this.recordPendingChildPlayBoundary(suspension);
7072
- scheduledChildPlayCall?.release();
7073
- await this.waitForScheduledChildPlayQuiescence();
7074
- throw new PlayExecutionSuspendedError(
7075
- this.consumePendingChildPlaySuspension() ?? suspension,
7076
- );
7077
- }
7078
- // Inline child = pure function composition (ADR 0013). No durable
7079
- // child run: no playRuns doc, no start/settle round-trip, no
7080
- // OptimisticConcurrencyControlFailure fan-out. The child's tool
7081
- // receipts still land durably under the PARENT run (the inline
7082
- // execution scope keeps the parent's receipt.ownerRunId) scoped by the
7083
- // replay-stable composition namespace (which also enforces the
7084
- // per-row inline-invocation sanity cap at creation).
7085
- const compositionNamespace = childGovernance.currentRunId;
7086
- const childExecutionScope = deriveChildRunExecutionScope(
7087
- this.currentExecutionScope,
7088
- {
7089
- placement: 'inline',
7090
- runId: compositionNamespace,
7091
- playId: resolvedName,
7092
- receiptNamespace: compositionNamespace,
7093
- },
7094
- );
7095
- this.inlineChildAggregates.total += 1;
7096
- try {
7097
- // A scalar child is a normal function call on the parent's context.
7098
- // Async-local identity changes receipt and recursion scope without
7099
- // allocating another PlayContext, tool queue, drain loop, resource
7100
- // governor, or receipt client. Calls from every concurrent child
7101
- // therefore reach the same proven parent batching path.
7102
- const result = await inlineCompositionContext.run(
7103
- {
7104
- context: this,
7105
- executionScope: childExecutionScope,
7106
- governor: inlineChildGovernor!,
7107
- playName: resolvedName,
7108
- staticPipeline: resolvedPlay.staticPipeline ?? null,
7109
- },
7110
- () => this.executeResolvedPlay(resolvedPlay, this, input),
7111
- );
7112
- this.inlineChildAggregates.ok += 1;
7113
- if (rowStore) {
7114
- this.emitScopedFieldMetaUpdate({
7115
- rowId: rowStore.rowId,
7116
- key: rowStore.rowKey ?? null,
7117
- tableNamespace: rowStore.tableNamespace ?? null,
7118
- fieldName: rowStore.fieldName,
7119
- status: 'completed',
7120
- rowStatus: 'running',
7121
- stage: resolvedName,
7122
- provider: 'deepline_native',
7123
- error: null,
7124
- producer,
7125
- dataPatch: {},
7126
- });
7127
- }
7128
- this.recordPlayCallStep({
7129
- playId: resolvedName,
7130
- execution: options?.execution,
7131
- description: options?.description,
7132
- });
7133
- return result as TOutput;
7134
- } catch (childError) {
7135
- this.recordInlineChildFailure(resolvedName, childError);
7136
- throw childError;
7137
- }
7138
- } catch (error) {
7139
- if (isPlayExecutionSuspendedError(error)) {
7140
- throw error;
7141
- }
7142
- if (rowStore) {
7143
- this.emitScopedFieldMetaUpdate({
7144
- rowId: rowStore.rowId,
7145
- key: rowStore.rowKey ?? null,
7146
- tableNamespace: rowStore.tableNamespace ?? null,
7147
- fieldName: rowStore.fieldName,
7148
- status: 'failed',
7149
- rowStatus: 'running',
7150
- stage: resolvedName,
7151
- provider: 'deepline_native',
7152
- error: this.formatRuntimeError(error),
7153
- producer,
7154
- dataPatch: {},
7155
- });
7156
- }
7157
- throw error;
7158
- } finally {
7159
- childPlaySlot?.release();
6575
+ receiptNamespace: compositionNamespace,
6576
+ },
6577
+ );
6578
+ this.inlineChildAggregates.total += 1;
6579
+ try {
6580
+ // A scalar child is a normal function call on the parent's context.
6581
+ // Async-local identity changes receipt and recursion scope without
6582
+ // allocating another PlayContext, tool queue, drain loop, resource
6583
+ // governor, or receipt client. Calls from every concurrent child
6584
+ // therefore reach the same proven parent batching path.
6585
+ const result = await inlineCompositionContext.run(
6586
+ {
6587
+ context: this,
6588
+ executionScope: childExecutionScope,
6589
+ governor: inlineChildGovernor,
6590
+ playName: resolvedName,
6591
+ staticPipeline: resolvedPlay.staticPipeline ?? null,
6592
+ },
6593
+ () => this.executeResolvedPlay(resolvedPlay, this, input),
6594
+ );
6595
+ this.inlineChildAggregates.ok += 1;
6596
+ if (rowStore) {
6597
+ this.emitScopedFieldMetaUpdate({
6598
+ rowId: rowStore.rowId,
6599
+ key: rowStore.rowKey ?? null,
6600
+ tableNamespace: rowStore.tableNamespace ?? null,
6601
+ fieldName: rowStore.fieldName,
6602
+ status: 'completed',
6603
+ rowStatus: 'running',
6604
+ stage: resolvedName,
6605
+ provider: 'deepline_native',
6606
+ error: null,
6607
+ producer,
6608
+ dataPatch: {},
6609
+ });
7160
6610
  }
7161
- };
7162
-
7163
- return await executePlayCall();
7164
- } finally {
7165
- scheduledChildPlayCall?.release();
6611
+ this.recordPlayCallStep({
6612
+ playId: resolvedName,
6613
+ execution: options?.execution === 'inline' ? 'inline' : undefined,
6614
+ description: options?.description,
6615
+ });
6616
+ return result as TOutput;
6617
+ } catch (childError) {
6618
+ this.recordInlineChildFailure(resolvedName, childError);
6619
+ throw childError;
6620
+ }
6621
+ } catch (error) {
6622
+ if (isPlayExecutionSuspendedError(error)) {
6623
+ throw error;
6624
+ }
6625
+ if (rowStore) {
6626
+ this.emitScopedFieldMetaUpdate({
6627
+ rowId: rowStore.rowId,
6628
+ key: rowStore.rowKey ?? null,
6629
+ tableNamespace: rowStore.tableNamespace ?? null,
6630
+ fieldName: rowStore.fieldName,
6631
+ status: 'failed',
6632
+ rowStatus: 'running',
6633
+ stage: resolvedName,
6634
+ provider: 'deepline_native',
6635
+ error: this.formatRuntimeError(error),
6636
+ producer,
6637
+ dataPatch: {},
6638
+ });
6639
+ }
6640
+ throw error;
7166
6641
  }
7167
6642
  }
7168
6643
 
@@ -7196,25 +6671,17 @@ export class PlayContextImpl {
7196
6671
  const namespace = `child:${childPlayName}#${normalizedKey}${
7197
6672
  rowScopeKey ? `@${rowScopeKey}` : ''
7198
6673
  }`;
7199
- if (!this.inlineChildInvocationOrdinals.has(namespace)) {
7200
- // Per-row fan-width guard. `maxPlayCallDepth` (governor) bounds nesting;
7201
- // this bounds distinct inline invocations from ONE row scope so a
7202
- // resolver looping runPlay unbounded fails loudly on that row instead of
7203
- // leaking namespaces.
7204
- const rowInvocationCount =
7205
- (this.inlineChildInvocationsPerRow.get(rowScopeKey) ?? 0) + 1;
7206
- if (rowInvocationCount > MAX_INLINE_CHILD_INVOCATIONS_PER_ROW) {
6674
+ if (rowScope) {
6675
+ const namespaces =
6676
+ rowScope.inlineChildInvocationNamespaces ??= new Set<string>();
6677
+ namespaces.add(namespace);
6678
+ if (namespaces.size > MAX_INLINE_CHILD_INVOCATIONS_PER_ROW) {
7207
6679
  throw new Error(
7208
6680
  `ctx.runPlay("${childPlayName}") exceeded the inline child invocation cap ` +
7209
6681
  `(${MAX_INLINE_CHILD_INVOCATIONS_PER_ROW} per row). A resolver is calling ` +
7210
6682
  'runPlay in an unbounded loop; give the batch ONE dataset child at play level instead.',
7211
6683
  );
7212
6684
  }
7213
- this.inlineChildInvocationsPerRow.set(rowScopeKey, rowInvocationCount);
7214
- this.inlineChildInvocationOrdinals.set(
7215
- namespace,
7216
- this.inlineChildInvocationOrdinals.size,
7217
- );
7218
6685
  }
7219
6686
  return namespace;
7220
6687
  }
@@ -7302,6 +6769,7 @@ export class PlayContextImpl {
7302
6769
  }
7303
6770
 
7304
6771
  async sleep(ms: number): Promise<void> {
6772
+ this.assertInlineChildContract('suspending_child');
7305
6773
  if (this.#options.durableBoundaries) {
7306
6774
  const delayMs = Math.max(0, Math.round(ms));
7307
6775
  const boundaryId = this.durableBoundaryId(
@@ -8725,10 +8193,26 @@ export class PlayContextImpl {
8725
8193
  const retrySafeTransientHttp =
8726
8194
  retryPolicy?.retrySafeTransientHttp === true;
8727
8195
  let transportAttempt = 0;
8728
- const retryToolTransportFailure = async (
8729
- message: string,
8730
- ): Promise<void> => {
8196
+ const retryToolTransportFailure = async (input: {
8197
+ error: unknown;
8198
+ elapsedMs: number;
8199
+ requestId: string | null;
8200
+ aborted: boolean;
8201
+ }): Promise<void> => {
8731
8202
  transportAttempt += 1;
8203
+ const diagnostic = describeTransportError(input.error);
8204
+ this.log(
8205
+ `[runtime.transport_failure] ${JSON.stringify({
8206
+ tool_id: toolId,
8207
+ gateway_origin: transportGatewayOriginForDiagnostic(url),
8208
+ attempt: transportAttempt,
8209
+ max_attempts: TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
8210
+ elapsed_ms: input.elapsedMs,
8211
+ request_id: input.requestId,
8212
+ aborted: input.aborted,
8213
+ error: diagnostic,
8214
+ })}`,
8215
+ );
8732
8216
  if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
8733
8217
  await this.governor.chargeBudget('retry');
8734
8218
  const retryAfterMs =
@@ -8738,14 +8222,14 @@ export class PlayContextImpl {
8738
8222
  transportAttempt,
8739
8223
  );
8740
8224
  this.log(
8741
- `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
8225
+ `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${diagnostic.message ?? 'unknown transport error'}`,
8742
8226
  );
8743
8227
  await options?.parkProviderCall?.();
8744
8228
  await this.sleepWithCheckpointHeartbeat(retryAfterMs);
8745
8229
  return;
8746
8230
  }
8747
8231
  throw new ToolHttpError(
8748
- `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
8232
+ `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${diagnostic.message ?? 'unknown transport error'}`,
8749
8233
  null,
8750
8234
  0,
8751
8235
  'repairable',
@@ -8767,9 +8251,12 @@ export class PlayContextImpl {
8767
8251
  : null;
8768
8252
  const providerIdempotencyKey =
8769
8253
  options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
8254
+ // Every gateway request needs a correlation ID. When the call has
8255
+ // an idempotency key, keep its stable ID across retries; otherwise
8256
+ // give this individual attempt a fresh ID for gateway pairing.
8770
8257
  const deeplineRequestId = providerIdempotencyKey
8771
8258
  ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
8772
- : null;
8259
+ : `ctx-tool-${crypto.randomUUID()}`;
8773
8260
  // Receipt ownership is a liveness contract, not a one-time check.
8774
8261
  // Keep it alive for the whole provider HTTP request. The cadence
8775
8262
  // comes from the store-issued expiry because a remote runner may
@@ -8837,10 +8324,10 @@ export class PlayContextImpl {
8837
8324
  V2_EXECUTE_RESPONSE_CONTRACT,
8838
8325
  [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
8839
8326
  [EXECUTE_TOOL_METADATA_HEADER]: 'true',
8840
- ...(deeplineRequestId
8327
+ 'x-deepline-request-id': deeplineRequestId,
8328
+ ...(providerIdempotencyKey
8841
8329
  ? {
8842
- 'x-deepline-request-id': deeplineRequestId,
8843
- 'x-deepline-idempotency-key': providerIdempotencyKey!,
8330
+ 'x-deepline-idempotency-key': providerIdempotencyKey,
8844
8331
  }
8845
8332
  : {}),
8846
8333
  ...(await this.vercelProtectionHeaders()),
@@ -8918,11 +8405,15 @@ export class PlayContextImpl {
8918
8405
  `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
8919
8406
  )
8920
8407
  : error;
8921
- const message =
8922
- transportError instanceof Error
8923
- ? transportError.message
8924
- : String(transportError);
8925
- await retryToolTransportFailure(message);
8408
+ await retryToolTransportFailure({
8409
+ error: transportError,
8410
+ elapsedMs:
8411
+ providerCallStartedAt === null
8412
+ ? 0
8413
+ : Date.now() - providerCallStartedAt,
8414
+ requestId: deeplineRequestId,
8415
+ aborted: abortController?.signal.aborted === true,
8416
+ });
8926
8417
  continue;
8927
8418
  } finally {
8928
8419
  if (timeoutHandle) {
@@ -8941,9 +8432,14 @@ export class PlayContextImpl {
8941
8432
  bodyText: text,
8942
8433
  })
8943
8434
  ) {
8944
- await retryToolTransportFailure(
8945
- 'the Deepline development tunnel returned a branded 502 before reaching the app origin',
8946
- );
8435
+ await retryToolTransportFailure({
8436
+ error: new Error(
8437
+ 'the Deepline development tunnel returned a branded 502 before reaching the app origin',
8438
+ ),
8439
+ elapsedMs: providerCallElapsedMs ?? 0,
8440
+ requestId: deeplineRequestId,
8441
+ aborted: false,
8442
+ });
8947
8443
  continue;
8948
8444
  }
8949
8445
  const authScopeChanged = parseToolExecuteAuthScopeChangedError({