deepline 0.1.212 → 0.1.214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +167 -329
  2. package/dist/bundling-sources/sdk/src/client.ts +2 -2
  3. package/dist/bundling-sources/sdk/src/index.ts +2 -0
  4. package/dist/bundling-sources/sdk/src/play.ts +8 -1
  5. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
  6. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  7. package/dist/bundling-sources/sdk/src/types.ts +3 -3
  8. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +49 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
  11. package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +209 -123
  13. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +30 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +160 -2
  16. package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
  22. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +54 -1
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +477 -85
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -2
  27. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
  28. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
  29. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
  30. package/dist/cli/index.js +50 -436
  31. package/dist/cli/index.mjs +50 -436
  32. package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
  33. package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
  34. package/dist/index.d.mts +11 -8
  35. package/dist/index.d.ts +11 -8
  36. package/dist/index.js +4 -4
  37. package/dist/index.mjs +4 -4
  38. package/dist/plays/bundle-play-file.d.mts +2 -2
  39. package/dist/plays/bundle-play-file.d.ts +2 -2
  40. package/dist/plays/bundle-play-file.mjs +1 -1
  41. package/package.json +1 -1
@@ -40,7 +40,6 @@ import {
40
40
  mapRowOutcomeRuntimeFields,
41
41
  resolveMapRowOutcomeKey,
42
42
  } from './map-row-outcome';
43
- import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
44
43
  import { buildChildRunId } from './child-run-id';
45
44
  import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
46
45
  import {
@@ -285,6 +284,11 @@ const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
285
284
  const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
286
285
  const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
287
286
  const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
287
+ // cloudflared returns this branded HTML when its connection to the local app
288
+ // resets. It is a transport failure before the app can reach a provider, not a
289
+ // provider 502; require both the branded page and our development tunnel host.
290
+ const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
291
+ /\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
288
292
  const CHILD_PLAY_LAUNCH_WRITE_CONFLICT_RETRY_DELAYS_MS = [
289
293
  50, 100, 200, 400, 800,
290
294
  ] as const;
@@ -519,6 +523,28 @@ export function resolveToolRuntimeTimeoutMs(
519
523
  : undefined;
520
524
  }
521
525
 
526
+ function isDeeplineDeveloperTunnelOrigin502(input: {
527
+ url: string;
528
+ status: number;
529
+ bodyText: string;
530
+ }): boolean {
531
+ if (
532
+ input.status !== 502 ||
533
+ !DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN.test(input.bodyText)
534
+ ) {
535
+ return false;
536
+ }
537
+ try {
538
+ const hostname = new URL(input.url).hostname.toLowerCase();
539
+ return (
540
+ hostname === 'deeplinedeveloper.com' ||
541
+ hostname.endsWith('.deeplinedeveloper.com')
542
+ );
543
+ } catch {
544
+ return false;
545
+ }
546
+ }
547
+
522
548
  function loadSafeFetch(): Promise<SafeFetchModule> {
523
549
  safeFetchModule ??= import('@shared_libs/security/safe-fetch');
524
550
  return safeFetchModule;
@@ -1521,12 +1547,14 @@ export class PlayContextImpl {
1521
1547
 
1522
1548
  private recordPlayCallStep(input: {
1523
1549
  playId: string;
1550
+ execution?: 'inline' | 'child-workflow';
1524
1551
  description?: string | null;
1525
1552
  nestedSteps?: PlayStep[];
1526
1553
  }): void {
1527
1554
  const step = {
1528
1555
  type: 'play_call' as const,
1529
1556
  playId: input.playId,
1557
+ ...(input.execution ? { execution: input.execution } : {}),
1530
1558
  nestedSteps: input.nestedSteps ?? [],
1531
1559
  description: normalizeStepDescription(input.description ?? undefined),
1532
1560
  };
@@ -1903,6 +1931,7 @@ export class PlayContextImpl {
1903
1931
  }
1904
1932
  const claimed = await this.#options.claimRuntimeStepReceipt({
1905
1933
  key,
1934
+ leaseId: `receipt-lease:${crypto.randomUUID()}`,
1906
1935
  runId: this.currentReceiptOwnerRunId,
1907
1936
  runAttempt: this.currentRunAttempt,
1908
1937
  leaseAware: true,
@@ -1979,30 +2008,6 @@ export class PlayContextImpl {
1979
2008
  };
1980
2009
  }
1981
2010
 
1982
- private async skipRuntimeStepReceipt(
1983
- key: string,
1984
- runId: string,
1985
- leaseId?: string | null,
1986
- ): Promise<RuntimeStepReceipt | null> {
1987
- if (!this.#options.skipRuntimeStepReceipt) {
1988
- return null;
1989
- }
1990
- const skipped = await this.#options.skipRuntimeStepReceipt({
1991
- key,
1992
- runId,
1993
- runAttempt: this.currentRunAttempt,
1994
- ...(leaseId ? { leaseId } : {}),
1995
- });
1996
- if (!skipped || typeof skipped.key !== 'string' || !skipped.key.trim()) {
1997
- return null;
1998
- }
1999
- return {
2000
- ...skipped,
2001
- key: skipped.key.trim(),
2002
- runId: skipped.runId ?? null,
2003
- };
2004
- }
2005
-
2006
2011
  private async heartbeatRuntimeStepReceipt(
2007
2012
  key: string,
2008
2013
  _runId: string,
@@ -2310,6 +2315,11 @@ export class PlayContextImpl {
2310
2315
  ? await this.dispatchChunkedRuntimeReceiptRequest(uniqueKeys, (chunk) =>
2311
2316
  claimReceipts({
2312
2317
  keys: chunk,
2318
+ // A transport retry can replay this mutation after the store
2319
+ // committed but before the response body was consumed. Stable
2320
+ // caller-owned tokens distinguish that replay from a concurrent
2321
+ // claimant without weakening the provider-call execution fence.
2322
+ leaseIds: chunk.map(() => `receipt-lease:${crypto.randomUUID()}`),
2313
2323
  runId: this.currentReceiptOwnerRunId,
2314
2324
  runAttempt: this.currentRunAttempt,
2315
2325
  leaseAware: true,
@@ -2777,6 +2787,24 @@ export class PlayContextImpl {
2777
2787
  this.#options.completeRuntimeStepReceipt ||
2778
2788
  this.#options.completeRuntimeStepReceipts,
2779
2789
  ),
2790
+ ...(this.#options.acquireRuntimeReceiptExecutionLock &&
2791
+ this.#options.releaseRuntimeReceiptExecutionLock
2792
+ ? {
2793
+ acquireExecutionLock: ({ receiptKey, ownerExecutionId, ttlMs }) =>
2794
+ this.#options.acquireRuntimeReceiptExecutionLock!({
2795
+ key: receiptKey,
2796
+ runId: this.currentReceiptOwnerRunId,
2797
+ ownerExecutionId,
2798
+ ttlMs,
2799
+ }),
2800
+ releaseExecutionLock: ({ receiptKey, ownerExecutionId }) =>
2801
+ this.#options.releaseRuntimeReceiptExecutionLock!({
2802
+ key: receiptKey,
2803
+ runId: this.currentReceiptOwnerRunId,
2804
+ ownerExecutionId,
2805
+ }),
2806
+ }
2807
+ : {}),
2780
2808
  };
2781
2809
  }
2782
2810
 
@@ -2834,6 +2862,8 @@ export class PlayContextImpl {
2834
2862
  onClaimedResult?: (output: T, receiptKey: string) => T;
2835
2863
  shouldPersistFailure?: (error: unknown) => boolean;
2836
2864
  markRunningBeforeExecute?: boolean;
2865
+ requiresExecutionLock?: boolean;
2866
+ executionLockTtlMs?: number;
2837
2867
  execute: (context: { leaseId: string | null }) => Promise<T>;
2838
2868
  },
2839
2869
  ): Promise<T> {
@@ -2865,6 +2895,9 @@ export class PlayContextImpl {
2865
2895
  onClaimedResult: opts.onClaimedResult,
2866
2896
  shouldPersistFailure: opts.shouldPersistFailure,
2867
2897
  markRunningBeforeExecute: opts.markRunningBeforeExecute,
2898
+ completedCacheOnly: operation === 'tool',
2899
+ requiresExecutionLock: opts.requiresExecutionLock,
2900
+ executionLockTtlMs: opts.executionLockTtlMs,
2868
2901
  formatError: (error) => this.formatRuntimeError(error),
2869
2902
  log: (message) => this.log(message),
2870
2903
  execute: opts.execute,
@@ -4016,12 +4049,6 @@ export class PlayContextImpl {
4016
4049
  resolvedTableNamespace = normalizeTableNamespace(
4017
4050
  mapStartResult.tableNamespace,
4018
4051
  );
4019
- if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
4020
- throw new RuntimeSheetRowsBlockedError({
4021
- tableNamespace: resolvedTableNamespace,
4022
- blockedRows: mapStartResult.blockedRows ?? [],
4023
- });
4024
- }
4025
4052
 
4026
4053
  const persistedRowIdentity = (
4027
4054
  row: Record<string, unknown>,
@@ -4456,12 +4483,6 @@ export class PlayContextImpl {
4456
4483
  resolvedTableNamespace = normalizeTableNamespace(
4457
4484
  mapStartResult.tableNamespace,
4458
4485
  );
4459
- if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
4460
- throw new RuntimeSheetRowsBlockedError({
4461
- tableNamespace: resolvedTableNamespace,
4462
- blockedRows: mapStartResult.blockedRows ?? [],
4463
- });
4464
- }
4465
4486
  const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
4466
4487
  resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
4467
4488
  const pendingRowsByKey = new Map<string, Record<string, unknown>>();
@@ -4574,6 +4595,7 @@ export class PlayContextImpl {
4574
4595
  );
4575
4596
 
4576
4597
  this.activeMapCellMeta = new Map();
4598
+ const staleCompletionKeys = new Set<string>();
4577
4599
  const persistMapRows = async (rows: PersistableMapRow[]) => {
4578
4600
  if (!this.#options.onMapRowsCompleted || rows.length === 0) {
4579
4601
  return;
@@ -4597,7 +4619,7 @@ export class PlayContextImpl {
4597
4619
  );
4598
4620
  const flushStartedAt = Date.now();
4599
4621
  try {
4600
- await this.#options.onMapRowsCompleted!({
4622
+ const writeResult = await this.#options.onMapRowsCompleted!({
4601
4623
  playName: this.#options.playName,
4602
4624
  playId: this.#options.playId,
4603
4625
  runId: this.#options.runId,
@@ -4609,6 +4631,11 @@ export class PlayContextImpl {
4609
4631
  ),
4610
4632
  staticPipeline: this.#options.staticPipeline ?? null,
4611
4633
  });
4634
+ if (writeResult) {
4635
+ for (const key of writeResult.staleDroppedKeys ?? []) {
4636
+ staleCompletionKeys.add(key);
4637
+ }
4638
+ }
4612
4639
  this.resourceGovernor.observe({
4613
4640
  sheetFlushBytes: chunkBytes,
4614
4641
  sheetFlushLatencyMs: Date.now() - flushStartedAt,
@@ -4654,6 +4681,7 @@ export class PlayContextImpl {
4654
4681
  completedRows: duplicateReuseCount,
4655
4682
  },
4656
4683
  {
4684
+ emitTerminalEvent: false,
4657
4685
  onRowError: options?.onRowError,
4658
4686
  executionRowKeys: rowsToExecuteEntries.map((entry) => entry.rowKey),
4659
4687
  executionRowIndexes: rowsToExecuteEntries.map(
@@ -4751,7 +4779,49 @@ export class PlayContextImpl {
4751
4779
  }
4752
4780
  this.activeMapCellMeta = null;
4753
4781
 
4754
- if (this.#options.onMapRowsCompleted) {
4782
+ if (staleCompletionKeys.size > 0) {
4783
+ results = results.filter((row, index) => {
4784
+ const key = rowIdentity(row, itemOriginalIndexes[index] ?? index);
4785
+ return !staleCompletionKeys.has(key);
4786
+ });
4787
+ }
4788
+
4789
+ const durableCompletedRows = Math.max(
4790
+ 0,
4791
+ reusedCount + mapResult.completedRows.length - staleCompletionKeys.size,
4792
+ );
4793
+ const durableFailedRows = mapResult.failedRows.length;
4794
+ const terminalFrame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
4795
+ if (terminalFrame) {
4796
+ this.setMapFrame({
4797
+ ...terminalFrame,
4798
+ status: 'completed',
4799
+ // Incremental rows add keys after their commit. Pure maps use the
4800
+ // durable aggregate count below instead of materializing up to millions
4801
+ // of keys only to mark the terminal frame.
4802
+ completedRowKeys: terminalFrame.completedRowKeys,
4803
+ pendingRowKeys: [],
4804
+ completedRowsCount: durableCompletedRows,
4805
+ pendingRowsCount: 0,
4806
+ failedRowsCount: durableFailedRows,
4807
+ activeBoundaryId: null,
4808
+ updatedAt: Date.now(),
4809
+ });
4810
+ }
4811
+ this.emitExecutionEvent({
4812
+ type: 'map.completed',
4813
+ mapInvocationId: mapScope.mapInvocationId,
4814
+ mapNodeId: mapScope.mapNodeId ?? null,
4815
+ logicalNamespace: mapScope.logicalNamespace,
4816
+ artifactTableNamespace: resolvedTableNamespace,
4817
+ completedRows: durableCompletedRows,
4818
+ failedRows: durableFailedRows,
4819
+ totalRows: totalInputCount,
4820
+ ...this.inlineChildAggregateEventFields(),
4821
+ at: Date.now(),
4822
+ });
4823
+
4824
+ if (this.#options.onMapRowsCompleted && staleCompletionKeys.size === 0) {
4755
4825
  results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
4756
4826
  mapName: normalizedMapNamespace,
4757
4827
  tableNamespace: resolvedTableNamespace,
@@ -4988,11 +5058,21 @@ export class PlayContextImpl {
4988
5058
  mapName: normalizedTableNamespace,
4989
5059
  budgetBytes: runtimeOptions?.retainedRowsMemoryBudgetBytes,
4990
5060
  });
4991
- const enqueueIncrementalPersist = (row: PersistableMapRow): void => {
4992
- void incrementalPersistence?.persistRows([row]).catch(() => {
4993
- // The final map-level flush awaits the same chain and surfaces the
4994
- // persistence failure loudly without holding a completed row slot open.
4995
- });
5061
+ const enqueueIncrementalPersist = (
5062
+ row: PersistableMapRow,
5063
+ onCommitted: () => void,
5064
+ ): void => {
5065
+ if (!incrementalPersistence) {
5066
+ onCommitted();
5067
+ return;
5068
+ }
5069
+ void incrementalPersistence
5070
+ .persistRows([row])
5071
+ .then(onCommitted)
5072
+ .catch(() => {
5073
+ // The final map-level flush awaits the same chain and surfaces the
5074
+ // persistence failure loudly without holding a completed row slot open.
5075
+ });
4996
5076
  };
4997
5077
 
4998
5078
  if (completedRows > 0 || pendingRows !== totalRows) {
@@ -5158,14 +5238,16 @@ export class PlayContextImpl {
5158
5238
  // this replaces ran AFTER every row had already computed, emitting 150k
5159
5239
  // post-hoc map.progress events and re-copying the completed-keys array
5160
5240
  // per row — all theater, no live progress value.
5161
- updateMapFrameProgress({
5162
- completedRowKeys: results.map((_row, index) =>
5163
- executionRowKey(
5164
- this.toOutputRow(items[index] as Record<string, unknown>),
5165
- index,
5241
+ if (!incrementalPersistence) {
5242
+ updateMapFrameProgress({
5243
+ completedRowKeys: results.map((_row, index) =>
5244
+ executionRowKey(
5245
+ this.toOutputRow(items[index] as Record<string, unknown>),
5246
+ index,
5247
+ ),
5166
5248
  ),
5167
- ),
5168
- });
5249
+ });
5250
+ }
5169
5251
  if (emitTerminalEvent) {
5170
5252
  updateMapFrameProgress({
5171
5253
  status: 'completed',
@@ -5346,9 +5428,6 @@ export class PlayContextImpl {
5346
5428
  });
5347
5429
  retainedRowsMemoryTracker.track(failedRow);
5348
5430
  failedRowsToPersist.push(failedRow);
5349
- updateMapFrameProgress({
5350
- failedRowKey: rowKey,
5351
- });
5352
5431
  this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
5353
5432
  rowId: idx,
5354
5433
  status: 'failed',
@@ -5357,7 +5436,9 @@ export class PlayContextImpl {
5357
5436
  error: formattedError,
5358
5437
  dataPatch: {},
5359
5438
  });
5360
- enqueueIncrementalPersist(failedRow);
5439
+ enqueueIncrementalPersist(failedRow, () => {
5440
+ updateMapFrameProgress({ failedRowKey: rowKey });
5441
+ });
5361
5442
  return FAILED_ROW;
5362
5443
  }
5363
5444
  const cellValue = this.serializeCellValue(value);
@@ -5404,9 +5485,6 @@ export class PlayContextImpl {
5404
5485
 
5405
5486
  const merged = cloneCsvAliasedRow(baseRow, computedFields);
5406
5487
  activeFieldName = null;
5407
- updateMapFrameProgress({
5408
- completedRowKey: rowKey,
5409
- });
5410
5488
  this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
5411
5489
  rowId: idx,
5412
5490
  status: 'completed',
@@ -5426,7 +5504,9 @@ export class PlayContextImpl {
5426
5504
  });
5427
5505
  retainedRowsMemoryTracker.track(completedRow);
5428
5506
  completedRowsToPersist.push(completedRow);
5429
- enqueueIncrementalPersist(completedRow);
5507
+ enqueueIncrementalPersist(completedRow, () => {
5508
+ updateMapFrameProgress({ completedRowKey: rowKey });
5509
+ });
5430
5510
  return publicRow;
5431
5511
  } catch (error) {
5432
5512
  if (isPlayRowExecutionSuspendedError(error)) {
@@ -5597,6 +5677,11 @@ export class PlayContextImpl {
5597
5677
  (result): result is Record<string, unknown> =>
5598
5678
  result !== WAITING_ROW && result !== FAILED_ROW,
5599
5679
  );
5680
+ // A row is acknowledged only after its terminal sheet batch commits. The
5681
+ // incremental promises publish their frame updates from the commit
5682
+ // continuation, so this barrier also makes the terminal map event derive
5683
+ // from durable row truth instead of resolver completion.
5684
+ await incrementalPersistence?.flush();
5600
5685
  if (emitTerminalEvent) {
5601
5686
  updateMapFrameProgress({
5602
5687
  status: 'completed',
@@ -6081,7 +6166,10 @@ export class PlayContextImpl {
6081
6166
  });
6082
6167
  let executionAuthScopeDigest =
6083
6168
  (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
6084
- const cacheableToolResult = !isQueryResultDatasetReadRequest(toolId, input);
6169
+ const eventWaitHandler =
6170
+ (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
6171
+ const cacheableToolResult =
6172
+ !eventWaitHandler && !isQueryResultDatasetReadRequest(toolId, input);
6085
6173
  let durableCacheKey = await this.durableToolCallCacheKey({
6086
6174
  toolId,
6087
6175
  requestInput: input,
@@ -6089,8 +6177,6 @@ export class PlayContextImpl {
6089
6177
  staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
6090
6178
  });
6091
6179
  const checkpointCacheKeys = [durableCacheKey];
6092
- const eventWaitHandler =
6093
- (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
6094
6180
  const store = rowContext.getStore();
6095
6181
 
6096
6182
  const executeTool = async (context?: {
@@ -6306,12 +6392,10 @@ export class PlayContextImpl {
6306
6392
  authScopeAttempt < 2;
6307
6393
  authScopeAttempt += 1
6308
6394
  ) {
6309
- // Capture the lease the receipt store hands to `execute` for THIS
6310
- // attempt's key. On an auth-scope re-claim we mark that old-scope receipt
6311
- // `skipped` (below), and the skip is fenced on this run+attempt+lease so
6312
- // it can only supersede the receipt this attempt owns.
6313
- let claimedReceiptLeaseId: string | null = null;
6314
6395
  try {
6396
+ const toolRetryPolicy = await this.#options
6397
+ .getToolRetryPolicy?.(toolId)
6398
+ .catch(() => null);
6315
6399
  return await this.executeWithRuntimeReceipt(
6316
6400
  'tool',
6317
6401
  normalizedKey,
@@ -6320,6 +6404,15 @@ export class PlayContextImpl {
6320
6404
  receiptKey: durableCacheKey,
6321
6405
  semanticKey: toolRequestIdentity,
6322
6406
  force: toolCachePolicy.force,
6407
+ // Missing metadata is intentionally lock-free. Only explicitly
6408
+ // dangerous non-idempotent side effects enter the fence path.
6409
+ requiresExecutionLock:
6410
+ toolRetryPolicy?.requiresExecutionFence === true,
6411
+ executionLockTtlMs: Math.min(
6412
+ 600_000,
6413
+ resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs) ??
6414
+ 300_000,
6415
+ ),
6323
6416
  staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
6324
6417
  onClaimedResult: (output, receiptKey) =>
6325
6418
  markToolExecuteResultExecutionOutcome(output, {
@@ -6337,10 +6430,7 @@ export class PlayContextImpl {
6337
6430
  shouldPersistFailure: (error) =>
6338
6431
  !(error instanceof ToolExecuteAuthScopeChangedError),
6339
6432
  markRunningBeforeExecute: false,
6340
- execute: ({ leaseId }) => {
6341
- claimedReceiptLeaseId = leaseId;
6342
- return executeTool({ leaseId });
6343
- },
6433
+ execute: ({ leaseId }) => executeTool({ leaseId }),
6344
6434
  runningReceiptWaitMaxAttempts:
6345
6435
  resolveRuntimeToolReceiptWaitMaxAttempts(
6346
6436
  typeof options?.receiptWaitMs === 'number'
@@ -6356,36 +6446,9 @@ export class PlayContextImpl {
6356
6446
  ) {
6357
6447
  throw error;
6358
6448
  }
6359
- // Supersede the old-scope receipt so the run's node projection does not
6360
- // surface it as `failed`. shouldPersistFailure kept it out of the failed
6361
- // state, but the claim left it `running` with a now-orphaned lease under
6362
- // the OLD cache key; without an explicit terminal it projects as a failed
6363
- // node even though the work is about to be re-done under the new scope.
6364
- // `skipped` is the terminal for superseded work; the skip is fenced on
6365
- // this run+attempt+lease so it can only touch the receipt this attempt
6366
- // owns. Best-effort: a skip that cannot fence (store returns null) must
6367
- // not mask the reclaim itself.
6368
- const supersededReceiptKey = durableCacheKey;
6369
- try {
6370
- const skipped = await this.skipRuntimeStepReceipt(
6371
- supersededReceiptKey,
6372
- this.currentRunId,
6373
- claimedReceiptLeaseId,
6374
- );
6375
- if (skipped) {
6376
- this.log(
6377
- `ctx.tools.execute(${toolId}): auth_scope_changed_superseded ` +
6378
- `old-scope receipt ${toolRequestIdentity} (label: ${normalizedKey}) ` +
6379
- `marked skipped so the reclaim does not project as a failed node.`,
6380
- );
6381
- }
6382
- } catch (skipError) {
6383
- this.log(
6384
- `ctx.tools.execute(${toolId}): auth_scope_changed_supersede_skip_failed ` +
6385
- `(label: ${normalizedKey}); could not mark the old-scope receipt ` +
6386
- `skipped: ${this.formatRuntimeError(skipError)}`,
6387
- );
6388
- }
6449
+ // Completed-receipt cache misses never publish a running receipt. The
6450
+ // old auth-scope attempt therefore has no ownership metadata to clean
6451
+ // up; re-key under the refreshed credential scope and retry once.
6389
6452
  // Make the bounded auth-scope re-claim observable in run logs instead
6390
6453
  // of a silent continuation: the credential identity changed after the
6391
6454
  // receipt key was prepared, so we evict the cached digest, re-resolve,
@@ -6552,6 +6615,8 @@ export class PlayContextImpl {
6552
6615
  }
6553
6616
  const childExecutionDecision = resolveChildExecutionStrategy({
6554
6617
  pipeline: resolvedPlay.staticPipeline,
6618
+ execution: options?.execution,
6619
+ childPlayName: resolvedName,
6555
6620
  });
6556
6621
  const launchThroughScheduler =
6557
6622
  this.shouldLaunchChildPlaysThroughScheduler() &&
@@ -6696,6 +6761,7 @@ export class PlayContextImpl {
6696
6761
  }
6697
6762
  this.recordPlayCallStep({
6698
6763
  playId: resolvedName,
6764
+ execution: options?.execution,
6699
6765
  description: options?.description,
6700
6766
  });
6701
6767
  return checkpointResult.output;
@@ -6743,6 +6809,7 @@ export class PlayContextImpl {
6743
6809
  }
6744
6810
  this.recordPlayCallStep({
6745
6811
  playId: resolvedName,
6812
+ execution: options?.execution,
6746
6813
  description: options?.description,
6747
6814
  });
6748
6815
  return terminal.output;
@@ -6843,6 +6910,7 @@ export class PlayContextImpl {
6843
6910
  }
6844
6911
  this.recordPlayCallStep({
6845
6912
  playId: resolvedName,
6913
+ execution: options?.execution,
6846
6914
  description: options?.description,
6847
6915
  nestedSteps: childContext.getSteps(),
6848
6916
  });
@@ -8488,6 +8556,32 @@ export class PlayContextImpl {
8488
8556
  const retrySafeTransientHttp =
8489
8557
  retryPolicy?.retrySafeTransientHttp === true;
8490
8558
  let transportAttempt = 0;
8559
+ const retryToolTransportFailure = async (
8560
+ message: string,
8561
+ ): Promise<void> => {
8562
+ transportAttempt += 1;
8563
+ if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
8564
+ await this.governor.chargeBudget('retry');
8565
+ const retryAfterMs =
8566
+ TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
8567
+ span.setAttribute(
8568
+ 'plays.transport_retry_attempt',
8569
+ transportAttempt,
8570
+ );
8571
+ this.log(
8572
+ `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
8573
+ );
8574
+ await options?.parkProviderCall?.();
8575
+ await this.sleepWithCheckpointHeartbeat(retryAfterMs);
8576
+ return;
8577
+ }
8578
+ throw new ToolHttpError(
8579
+ `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
8580
+ null,
8581
+ 0,
8582
+ 'repairable',
8583
+ );
8584
+ };
8491
8585
 
8492
8586
  while (true) {
8493
8587
  let response: Response;
@@ -8655,32 +8749,12 @@ export class PlayContextImpl {
8655
8749
  `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
8656
8750
  )
8657
8751
  : error;
8658
- transportAttempt += 1;
8659
8752
  const message =
8660
8753
  transportError instanceof Error
8661
8754
  ? transportError.message
8662
8755
  : String(transportError);
8663
- if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
8664
- await this.governor.chargeBudget('retry');
8665
- const retryAfterMs =
8666
- TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
8667
- span.setAttribute(
8668
- 'plays.transport_retry_attempt',
8669
- transportAttempt,
8670
- );
8671
- this.log(
8672
- `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
8673
- );
8674
- await options?.parkProviderCall?.();
8675
- await this.sleepWithCheckpointHeartbeat(retryAfterMs);
8676
- continue;
8677
- }
8678
- throw new ToolHttpError(
8679
- `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
8680
- null,
8681
- 0,
8682
- 'repairable',
8683
- );
8756
+ await retryToolTransportFailure(message);
8757
+ continue;
8684
8758
  } finally {
8685
8759
  if (timeoutHandle) {
8686
8760
  clearTimeout(timeoutHandle);
@@ -8691,6 +8765,18 @@ export class PlayContextImpl {
8691
8765
 
8692
8766
  if (!response.ok) {
8693
8767
  const text = await response.text();
8768
+ if (
8769
+ isDeeplineDeveloperTunnelOrigin502({
8770
+ url,
8771
+ status: response.status,
8772
+ bodyText: text,
8773
+ })
8774
+ ) {
8775
+ await retryToolTransportFailure(
8776
+ 'the Deepline development tunnel returned a branded 502 before reaching the app origin',
8777
+ );
8778
+ continue;
8779
+ }
8694
8780
  const authScopeChanged = parseToolExecuteAuthScopeChangedError({
8695
8781
  status: response.status,
8696
8782
  bodyText: text,
@@ -214,6 +214,7 @@ export type PlayStep =
214
214
  | {
215
215
  type: 'play_call';
216
216
  playId: string;
217
+ execution?: 'inline' | 'child-workflow';
217
218
  results?: PlayStepRowResult[];
218
219
  nestedSteps: PlayStep[];
219
220
  description?: string;
@@ -252,6 +253,7 @@ export type PlayDatasetSubstep =
252
253
  | {
253
254
  type: 'play_call';
254
255
  playId: string;
256
+ execution?: 'inline' | 'child-workflow';
255
257
  results?: PlayStepRowResult[];
256
258
  nestedSteps: PlayStep[];
257
259
  description?: string;