deepline 0.1.211 → 0.1.213

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +317 -340
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +1 -1
  4. package/dist/bundling-sources/sdk/src/client.ts +118 -3
  5. package/dist/bundling-sources/sdk/src/index.ts +2 -0
  6. package/dist/bundling-sources/sdk/src/play.ts +8 -1
  7. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
  8. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
  9. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  10. package/dist/bundling-sources/sdk/src/types.ts +47 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +302 -30
  12. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
  14. package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +164 -121
  16. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +42 -1
  18. package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
  19. package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +163 -7
  21. package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
  23. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
  24. package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
  26. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
  27. package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
  29. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
  30. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +91 -6
  33. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
  34. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +540 -92
  35. package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
  36. package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
  37. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
  38. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
  39. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
  40. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
  41. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
  42. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
  43. package/dist/cli/index.js +637 -229
  44. package/dist/cli/index.mjs +637 -229
  45. package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
  46. package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
  47. package/dist/index.d.mts +84 -11
  48. package/dist/index.d.ts +84 -11
  49. package/dist/index.js +225 -43
  50. package/dist/index.mjs +225 -43
  51. package/dist/plays/bundle-play-file.d.mts +2 -2
  52. package/dist/plays/bundle-play-file.d.ts +2 -2
  53. package/dist/plays/bundle-play-file.mjs +65 -4
  54. package/package.json +1 -1
@@ -40,7 +40,6 @@ import {
40
40
  mapRowOutcomeRuntimeFields,
41
41
  resolveMapRowOutcomeKey,
42
42
  } from './map-row-outcome';
43
- import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
44
43
  import { buildChildRunId } from './child-run-id';
45
44
  import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
46
45
  import {
@@ -106,6 +105,7 @@ import {
106
105
  parseToolExecuteAuthScopeChangedError,
107
106
  ToolExecuteAuthScopeChangedError,
108
107
  } from './tool-execute-retry-policy';
108
+ import { ToolHttpError } from './tool-http-errors';
109
109
  import {
110
110
  buildDurableCtxCallCacheKey,
111
111
  buildDurableRunPlayInvocationScope,
@@ -1520,12 +1520,14 @@ export class PlayContextImpl {
1520
1520
 
1521
1521
  private recordPlayCallStep(input: {
1522
1522
  playId: string;
1523
+ execution?: 'inline' | 'child-workflow';
1523
1524
  description?: string | null;
1524
1525
  nestedSteps?: PlayStep[];
1525
1526
  }): void {
1526
1527
  const step = {
1527
1528
  type: 'play_call' as const,
1528
1529
  playId: input.playId,
1530
+ ...(input.execution ? { execution: input.execution } : {}),
1529
1531
  nestedSteps: input.nestedSteps ?? [],
1530
1532
  description: normalizeStepDescription(input.description ?? undefined),
1531
1533
  };
@@ -1902,6 +1904,7 @@ export class PlayContextImpl {
1902
1904
  }
1903
1905
  const claimed = await this.#options.claimRuntimeStepReceipt({
1904
1906
  key,
1907
+ leaseId: `receipt-lease:${crypto.randomUUID()}`,
1905
1908
  runId: this.currentReceiptOwnerRunId,
1906
1909
  runAttempt: this.currentRunAttempt,
1907
1910
  leaseAware: true,
@@ -1978,30 +1981,6 @@ export class PlayContextImpl {
1978
1981
  };
1979
1982
  }
1980
1983
 
1981
- private async skipRuntimeStepReceipt(
1982
- key: string,
1983
- runId: string,
1984
- leaseId?: string | null,
1985
- ): Promise<RuntimeStepReceipt | null> {
1986
- if (!this.#options.skipRuntimeStepReceipt) {
1987
- return null;
1988
- }
1989
- const skipped = await this.#options.skipRuntimeStepReceipt({
1990
- key,
1991
- runId,
1992
- runAttempt: this.currentRunAttempt,
1993
- ...(leaseId ? { leaseId } : {}),
1994
- });
1995
- if (!skipped || typeof skipped.key !== 'string' || !skipped.key.trim()) {
1996
- return null;
1997
- }
1998
- return {
1999
- ...skipped,
2000
- key: skipped.key.trim(),
2001
- runId: skipped.runId ?? null,
2002
- };
2003
- }
2004
-
2005
1984
  private async heartbeatRuntimeStepReceipt(
2006
1985
  key: string,
2007
1986
  _runId: string,
@@ -2309,6 +2288,11 @@ export class PlayContextImpl {
2309
2288
  ? await this.dispatchChunkedRuntimeReceiptRequest(uniqueKeys, (chunk) =>
2310
2289
  claimReceipts({
2311
2290
  keys: chunk,
2291
+ // A transport retry can replay this mutation after the store
2292
+ // committed but before the response body was consumed. Stable
2293
+ // caller-owned tokens distinguish that replay from a concurrent
2294
+ // claimant without weakening the provider-call execution fence.
2295
+ leaseIds: chunk.map(() => `receipt-lease:${crypto.randomUUID()}`),
2312
2296
  runId: this.currentReceiptOwnerRunId,
2313
2297
  runAttempt: this.currentRunAttempt,
2314
2298
  leaseAware: true,
@@ -2776,6 +2760,24 @@ export class PlayContextImpl {
2776
2760
  this.#options.completeRuntimeStepReceipt ||
2777
2761
  this.#options.completeRuntimeStepReceipts,
2778
2762
  ),
2763
+ ...(this.#options.acquireRuntimeReceiptExecutionLock &&
2764
+ this.#options.releaseRuntimeReceiptExecutionLock
2765
+ ? {
2766
+ acquireExecutionLock: ({ receiptKey, ownerExecutionId, ttlMs }) =>
2767
+ this.#options.acquireRuntimeReceiptExecutionLock!({
2768
+ key: receiptKey,
2769
+ runId: this.currentReceiptOwnerRunId,
2770
+ ownerExecutionId,
2771
+ ttlMs,
2772
+ }),
2773
+ releaseExecutionLock: ({ receiptKey, ownerExecutionId }) =>
2774
+ this.#options.releaseRuntimeReceiptExecutionLock!({
2775
+ key: receiptKey,
2776
+ runId: this.currentReceiptOwnerRunId,
2777
+ ownerExecutionId,
2778
+ }),
2779
+ }
2780
+ : {}),
2779
2781
  };
2780
2782
  }
2781
2783
 
@@ -2833,6 +2835,8 @@ export class PlayContextImpl {
2833
2835
  onClaimedResult?: (output: T, receiptKey: string) => T;
2834
2836
  shouldPersistFailure?: (error: unknown) => boolean;
2835
2837
  markRunningBeforeExecute?: boolean;
2838
+ requiresExecutionLock?: boolean;
2839
+ executionLockTtlMs?: number;
2836
2840
  execute: (context: { leaseId: string | null }) => Promise<T>;
2837
2841
  },
2838
2842
  ): Promise<T> {
@@ -2864,6 +2868,9 @@ export class PlayContextImpl {
2864
2868
  onClaimedResult: opts.onClaimedResult,
2865
2869
  shouldPersistFailure: opts.shouldPersistFailure,
2866
2870
  markRunningBeforeExecute: opts.markRunningBeforeExecute,
2871
+ completedCacheOnly: operation === 'tool',
2872
+ requiresExecutionLock: opts.requiresExecutionLock,
2873
+ executionLockTtlMs: opts.executionLockTtlMs,
2867
2874
  formatError: (error) => this.formatRuntimeError(error),
2868
2875
  log: (message) => this.log(message),
2869
2876
  execute: opts.execute,
@@ -3486,7 +3493,8 @@ export class PlayContextImpl {
3486
3493
  );
3487
3494
  }
3488
3495
  const finalWrapped =
3489
- completed?.status === 'completed' || completed?.status === 'skipped'
3496
+ (completed?.status === 'completed' || completed?.status === 'skipped') &&
3497
+ completed.output !== undefined
3490
3498
  ? await this.wrapToolExecutionResult({
3491
3499
  toolId,
3492
3500
  status:
@@ -4014,12 +4022,6 @@ export class PlayContextImpl {
4014
4022
  resolvedTableNamespace = normalizeTableNamespace(
4015
4023
  mapStartResult.tableNamespace,
4016
4024
  );
4017
- if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
4018
- throw new RuntimeSheetRowsBlockedError({
4019
- tableNamespace: resolvedTableNamespace,
4020
- blockedRows: mapStartResult.blockedRows ?? [],
4021
- });
4022
- }
4023
4025
 
4024
4026
  const persistedRowIdentity = (
4025
4027
  row: Record<string, unknown>,
@@ -4454,12 +4456,6 @@ export class PlayContextImpl {
4454
4456
  resolvedTableNamespace = normalizeTableNamespace(
4455
4457
  mapStartResult.tableNamespace,
4456
4458
  );
4457
- if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
4458
- throw new RuntimeSheetRowsBlockedError({
4459
- tableNamespace: resolvedTableNamespace,
4460
- blockedRows: mapStartResult.blockedRows ?? [],
4461
- });
4462
- }
4463
4459
  const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
4464
4460
  resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
4465
4461
  const pendingRowsByKey = new Map<string, Record<string, unknown>>();
@@ -4572,6 +4568,7 @@ export class PlayContextImpl {
4572
4568
  );
4573
4569
 
4574
4570
  this.activeMapCellMeta = new Map();
4571
+ const staleCompletionKeys = new Set<string>();
4575
4572
  const persistMapRows = async (rows: PersistableMapRow[]) => {
4576
4573
  if (!this.#options.onMapRowsCompleted || rows.length === 0) {
4577
4574
  return;
@@ -4595,7 +4592,7 @@ export class PlayContextImpl {
4595
4592
  );
4596
4593
  const flushStartedAt = Date.now();
4597
4594
  try {
4598
- await this.#options.onMapRowsCompleted!({
4595
+ const writeResult = await this.#options.onMapRowsCompleted!({
4599
4596
  playName: this.#options.playName,
4600
4597
  playId: this.#options.playId,
4601
4598
  runId: this.#options.runId,
@@ -4607,6 +4604,11 @@ export class PlayContextImpl {
4607
4604
  ),
4608
4605
  staticPipeline: this.#options.staticPipeline ?? null,
4609
4606
  });
4607
+ if (writeResult) {
4608
+ for (const key of writeResult.staleDroppedKeys ?? []) {
4609
+ staleCompletionKeys.add(key);
4610
+ }
4611
+ }
4610
4612
  this.resourceGovernor.observe({
4611
4613
  sheetFlushBytes: chunkBytes,
4612
4614
  sheetFlushLatencyMs: Date.now() - flushStartedAt,
@@ -4652,6 +4654,7 @@ export class PlayContextImpl {
4652
4654
  completedRows: duplicateReuseCount,
4653
4655
  },
4654
4656
  {
4657
+ emitTerminalEvent: false,
4655
4658
  onRowError: options?.onRowError,
4656
4659
  executionRowKeys: rowsToExecuteEntries.map((entry) => entry.rowKey),
4657
4660
  executionRowIndexes: rowsToExecuteEntries.map(
@@ -4749,7 +4752,49 @@ export class PlayContextImpl {
4749
4752
  }
4750
4753
  this.activeMapCellMeta = null;
4751
4754
 
4752
- if (this.#options.onMapRowsCompleted) {
4755
+ if (staleCompletionKeys.size > 0) {
4756
+ results = results.filter((row, index) => {
4757
+ const key = rowIdentity(row, itemOriginalIndexes[index] ?? index);
4758
+ return !staleCompletionKeys.has(key);
4759
+ });
4760
+ }
4761
+
4762
+ const durableCompletedRows = Math.max(
4763
+ 0,
4764
+ reusedCount + mapResult.completedRows.length - staleCompletionKeys.size,
4765
+ );
4766
+ const durableFailedRows = mapResult.failedRows.length;
4767
+ const terminalFrame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
4768
+ if (terminalFrame) {
4769
+ this.setMapFrame({
4770
+ ...terminalFrame,
4771
+ status: 'completed',
4772
+ // Incremental rows add keys after their commit. Pure maps use the
4773
+ // durable aggregate count below instead of materializing up to millions
4774
+ // of keys only to mark the terminal frame.
4775
+ completedRowKeys: terminalFrame.completedRowKeys,
4776
+ pendingRowKeys: [],
4777
+ completedRowsCount: durableCompletedRows,
4778
+ pendingRowsCount: 0,
4779
+ failedRowsCount: durableFailedRows,
4780
+ activeBoundaryId: null,
4781
+ updatedAt: Date.now(),
4782
+ });
4783
+ }
4784
+ this.emitExecutionEvent({
4785
+ type: 'map.completed',
4786
+ mapInvocationId: mapScope.mapInvocationId,
4787
+ mapNodeId: mapScope.mapNodeId ?? null,
4788
+ logicalNamespace: mapScope.logicalNamespace,
4789
+ artifactTableNamespace: resolvedTableNamespace,
4790
+ completedRows: durableCompletedRows,
4791
+ failedRows: durableFailedRows,
4792
+ totalRows: totalInputCount,
4793
+ ...this.inlineChildAggregateEventFields(),
4794
+ at: Date.now(),
4795
+ });
4796
+
4797
+ if (this.#options.onMapRowsCompleted && staleCompletionKeys.size === 0) {
4753
4798
  results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
4754
4799
  mapName: normalizedMapNamespace,
4755
4800
  tableNamespace: resolvedTableNamespace,
@@ -4986,11 +5031,21 @@ export class PlayContextImpl {
4986
5031
  mapName: normalizedTableNamespace,
4987
5032
  budgetBytes: runtimeOptions?.retainedRowsMemoryBudgetBytes,
4988
5033
  });
4989
- const enqueueIncrementalPersist = (row: PersistableMapRow): void => {
4990
- void incrementalPersistence?.persistRows([row]).catch(() => {
4991
- // The final map-level flush awaits the same chain and surfaces the
4992
- // persistence failure loudly without holding a completed row slot open.
4993
- });
5034
+ const enqueueIncrementalPersist = (
5035
+ row: PersistableMapRow,
5036
+ onCommitted: () => void,
5037
+ ): void => {
5038
+ if (!incrementalPersistence) {
5039
+ onCommitted();
5040
+ return;
5041
+ }
5042
+ void incrementalPersistence
5043
+ .persistRows([row])
5044
+ .then(onCommitted)
5045
+ .catch(() => {
5046
+ // The final map-level flush awaits the same chain and surfaces the
5047
+ // persistence failure loudly without holding a completed row slot open.
5048
+ });
4994
5049
  };
4995
5050
 
4996
5051
  if (completedRows > 0 || pendingRows !== totalRows) {
@@ -5156,14 +5211,16 @@ export class PlayContextImpl {
5156
5211
  // this replaces ran AFTER every row had already computed, emitting 150k
5157
5212
  // post-hoc map.progress events and re-copying the completed-keys array
5158
5213
  // per row — all theater, no live progress value.
5159
- updateMapFrameProgress({
5160
- completedRowKeys: results.map((_row, index) =>
5161
- executionRowKey(
5162
- this.toOutputRow(items[index] as Record<string, unknown>),
5163
- index,
5214
+ if (!incrementalPersistence) {
5215
+ updateMapFrameProgress({
5216
+ completedRowKeys: results.map((_row, index) =>
5217
+ executionRowKey(
5218
+ this.toOutputRow(items[index] as Record<string, unknown>),
5219
+ index,
5220
+ ),
5164
5221
  ),
5165
- ),
5166
- });
5222
+ });
5223
+ }
5167
5224
  if (emitTerminalEvent) {
5168
5225
  updateMapFrameProgress({
5169
5226
  status: 'completed',
@@ -5344,9 +5401,6 @@ export class PlayContextImpl {
5344
5401
  });
5345
5402
  retainedRowsMemoryTracker.track(failedRow);
5346
5403
  failedRowsToPersist.push(failedRow);
5347
- updateMapFrameProgress({
5348
- failedRowKey: rowKey,
5349
- });
5350
5404
  this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
5351
5405
  rowId: idx,
5352
5406
  status: 'failed',
@@ -5355,7 +5409,9 @@ export class PlayContextImpl {
5355
5409
  error: formattedError,
5356
5410
  dataPatch: {},
5357
5411
  });
5358
- enqueueIncrementalPersist(failedRow);
5412
+ enqueueIncrementalPersist(failedRow, () => {
5413
+ updateMapFrameProgress({ failedRowKey: rowKey });
5414
+ });
5359
5415
  return FAILED_ROW;
5360
5416
  }
5361
5417
  const cellValue = this.serializeCellValue(value);
@@ -5402,9 +5458,6 @@ export class PlayContextImpl {
5402
5458
 
5403
5459
  const merged = cloneCsvAliasedRow(baseRow, computedFields);
5404
5460
  activeFieldName = null;
5405
- updateMapFrameProgress({
5406
- completedRowKey: rowKey,
5407
- });
5408
5461
  this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
5409
5462
  rowId: idx,
5410
5463
  status: 'completed',
@@ -5424,7 +5477,9 @@ export class PlayContextImpl {
5424
5477
  });
5425
5478
  retainedRowsMemoryTracker.track(completedRow);
5426
5479
  completedRowsToPersist.push(completedRow);
5427
- enqueueIncrementalPersist(completedRow);
5480
+ enqueueIncrementalPersist(completedRow, () => {
5481
+ updateMapFrameProgress({ completedRowKey: rowKey });
5482
+ });
5428
5483
  return publicRow;
5429
5484
  } catch (error) {
5430
5485
  if (isPlayRowExecutionSuspendedError(error)) {
@@ -5595,6 +5650,11 @@ export class PlayContextImpl {
5595
5650
  (result): result is Record<string, unknown> =>
5596
5651
  result !== WAITING_ROW && result !== FAILED_ROW,
5597
5652
  );
5653
+ // A row is acknowledged only after its terminal sheet batch commits. The
5654
+ // incremental promises publish their frame updates from the commit
5655
+ // continuation, so this barrier also makes the terminal map event derive
5656
+ // from durable row truth instead of resolver completion.
5657
+ await incrementalPersistence?.flush();
5598
5658
  if (emitTerminalEvent) {
5599
5659
  updateMapFrameProgress({
5600
5660
  status: 'completed',
@@ -6079,7 +6139,10 @@ export class PlayContextImpl {
6079
6139
  });
6080
6140
  let executionAuthScopeDigest =
6081
6141
  (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
6082
- const cacheableToolResult = !isQueryResultDatasetReadRequest(toolId, input);
6142
+ const eventWaitHandler =
6143
+ (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
6144
+ const cacheableToolResult =
6145
+ !eventWaitHandler && !isQueryResultDatasetReadRequest(toolId, input);
6083
6146
  let durableCacheKey = await this.durableToolCallCacheKey({
6084
6147
  toolId,
6085
6148
  requestInput: input,
@@ -6087,8 +6150,6 @@ export class PlayContextImpl {
6087
6150
  staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
6088
6151
  });
6089
6152
  const checkpointCacheKeys = [durableCacheKey];
6090
- const eventWaitHandler =
6091
- (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
6092
6153
  const store = rowContext.getStore();
6093
6154
 
6094
6155
  const executeTool = async (context?: {
@@ -6304,12 +6365,10 @@ export class PlayContextImpl {
6304
6365
  authScopeAttempt < 2;
6305
6366
  authScopeAttempt += 1
6306
6367
  ) {
6307
- // Capture the lease the receipt store hands to `execute` for THIS
6308
- // attempt's key. On an auth-scope re-claim we mark that old-scope receipt
6309
- // `skipped` (below), and the skip is fenced on this run+attempt+lease so
6310
- // it can only supersede the receipt this attempt owns.
6311
- let claimedReceiptLeaseId: string | null = null;
6312
6368
  try {
6369
+ const toolRetryPolicy = await this.#options
6370
+ .getToolRetryPolicy?.(toolId)
6371
+ .catch(() => null);
6313
6372
  return await this.executeWithRuntimeReceipt(
6314
6373
  'tool',
6315
6374
  normalizedKey,
@@ -6318,12 +6377,16 @@ export class PlayContextImpl {
6318
6377
  receiptKey: durableCacheKey,
6319
6378
  semanticKey: toolRequestIdentity,
6320
6379
  force: toolCachePolicy.force,
6380
+ // Missing metadata is intentionally lock-free. Only explicitly
6381
+ // dangerous non-idempotent side effects enter the fence path.
6382
+ requiresExecutionLock:
6383
+ toolRetryPolicy?.requiresExecutionFence === true,
6384
+ executionLockTtlMs: Math.min(
6385
+ 600_000,
6386
+ resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs) ??
6387
+ 300_000,
6388
+ ),
6321
6389
  staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
6322
- markSkipped: () => {
6323
- this.log(
6324
- `ctx.tools.execute(${toolId}): no-op due completed receipt ${toolRequestIdentity} (label: ${normalizedKey})`,
6325
- );
6326
- },
6327
6390
  onClaimedResult: (output, receiptKey) =>
6328
6391
  markToolExecuteResultExecutionOutcome(output, {
6329
6392
  kind: 'live',
@@ -6340,10 +6403,7 @@ export class PlayContextImpl {
6340
6403
  shouldPersistFailure: (error) =>
6341
6404
  !(error instanceof ToolExecuteAuthScopeChangedError),
6342
6405
  markRunningBeforeExecute: false,
6343
- execute: ({ leaseId }) => {
6344
- claimedReceiptLeaseId = leaseId;
6345
- return executeTool({ leaseId });
6346
- },
6406
+ execute: ({ leaseId }) => executeTool({ leaseId }),
6347
6407
  runningReceiptWaitMaxAttempts:
6348
6408
  resolveRuntimeToolReceiptWaitMaxAttempts(
6349
6409
  typeof options?.receiptWaitMs === 'number'
@@ -6359,36 +6419,9 @@ export class PlayContextImpl {
6359
6419
  ) {
6360
6420
  throw error;
6361
6421
  }
6362
- // Supersede the old-scope receipt so the run's node projection does not
6363
- // surface it as `failed`. shouldPersistFailure kept it out of the failed
6364
- // state, but the claim left it `running` with a now-orphaned lease under
6365
- // the OLD cache key; without an explicit terminal it projects as a failed
6366
- // node even though the work is about to be re-done under the new scope.
6367
- // `skipped` is the terminal for superseded work; the skip is fenced on
6368
- // this run+attempt+lease so it can only touch the receipt this attempt
6369
- // owns. Best-effort: a skip that cannot fence (store returns null) must
6370
- // not mask the reclaim itself.
6371
- const supersededReceiptKey = durableCacheKey;
6372
- try {
6373
- const skipped = await this.skipRuntimeStepReceipt(
6374
- supersededReceiptKey,
6375
- this.currentRunId,
6376
- claimedReceiptLeaseId,
6377
- );
6378
- if (skipped) {
6379
- this.log(
6380
- `ctx.tools.execute(${toolId}): auth_scope_changed_superseded ` +
6381
- `old-scope receipt ${toolRequestIdentity} (label: ${normalizedKey}) ` +
6382
- `marked skipped so the reclaim does not project as a failed node.`,
6383
- );
6384
- }
6385
- } catch (skipError) {
6386
- this.log(
6387
- `ctx.tools.execute(${toolId}): auth_scope_changed_supersede_skip_failed ` +
6388
- `(label: ${normalizedKey}); could not mark the old-scope receipt ` +
6389
- `skipped: ${this.formatRuntimeError(skipError)}`,
6390
- );
6391
- }
6422
+ // Completed-receipt cache misses never publish a running receipt. The
6423
+ // old auth-scope attempt therefore has no ownership metadata to clean
6424
+ // up; re-key under the refreshed credential scope and retry once.
6392
6425
  // Make the bounded auth-scope re-claim observable in run logs instead
6393
6426
  // of a silent continuation: the credential identity changed after the
6394
6427
  // receipt key was prepared, so we evict the cached digest, re-resolve,
@@ -6555,6 +6588,8 @@ export class PlayContextImpl {
6555
6588
  }
6556
6589
  const childExecutionDecision = resolveChildExecutionStrategy({
6557
6590
  pipeline: resolvedPlay.staticPipeline,
6591
+ execution: options?.execution,
6592
+ childPlayName: resolvedName,
6558
6593
  });
6559
6594
  const launchThroughScheduler =
6560
6595
  this.shouldLaunchChildPlaysThroughScheduler() &&
@@ -6699,6 +6734,7 @@ export class PlayContextImpl {
6699
6734
  }
6700
6735
  this.recordPlayCallStep({
6701
6736
  playId: resolvedName,
6737
+ execution: options?.execution,
6702
6738
  description: options?.description,
6703
6739
  });
6704
6740
  return checkpointResult.output;
@@ -6746,6 +6782,7 @@ export class PlayContextImpl {
6746
6782
  }
6747
6783
  this.recordPlayCallStep({
6748
6784
  playId: resolvedName,
6785
+ execution: options?.execution,
6749
6786
  description: options?.description,
6750
6787
  });
6751
6788
  return terminal.output;
@@ -6846,6 +6883,7 @@ export class PlayContextImpl {
6846
6883
  }
6847
6884
  this.recordPlayCallStep({
6848
6885
  playId: resolvedName,
6886
+ execution: options?.execution,
6849
6887
  description: options?.description,
6850
6888
  nestedSteps: childContext.getSteps(),
6851
6889
  });
@@ -7277,11 +7315,6 @@ export class PlayContextImpl {
7277
7315
  egressSlot.release();
7278
7316
  }
7279
7317
  },
7280
- markSkipped: (output) => {
7281
- this.log(
7282
- `ctx.fetch(${output?.url ?? ''}): no-op due completed receipt`,
7283
- );
7284
- },
7285
7318
  },
7286
7319
  );
7287
7320
  }
@@ -7348,7 +7381,6 @@ export class PlayContextImpl {
7348
7381
  : options?.semanticKey,
7349
7382
  staleAfterSeconds: options?.staleAfterSeconds,
7350
7383
  markSkipped: (output) => {
7351
- this.log(`ctx.step(${normalizedKey}): no-op due completed receipt`);
7352
7384
  assertJsonSerializableStepOutput(normalizedKey, output);
7353
7385
  },
7354
7386
  execute: executeStep,
@@ -8500,6 +8532,8 @@ export class PlayContextImpl {
8500
8532
 
8501
8533
  while (true) {
8502
8534
  let response: Response;
8535
+ let providerCallStartedAt: number | null = null;
8536
+ let providerCallElapsedMs: number | null = null;
8503
8537
  const durableCallReceiptKey =
8504
8538
  options?.durableCallReceiptKey?.trim() || null;
8505
8539
  const executionAuthScopeDigest =
@@ -8533,7 +8567,6 @@ export class PlayContextImpl {
8533
8567
  }, timeoutMs)
8534
8568
  : null;
8535
8569
  try {
8536
- const providerCallStartedAt = Date.now();
8537
8570
  const ownershipStartedAt = Date.now();
8538
8571
  await options?.beforeProviderCall?.();
8539
8572
  if (runtimeReceiptReadTraceEnabled) {
@@ -8541,6 +8574,7 @@ export class PlayContextImpl {
8541
8574
  `[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`,
8542
8575
  );
8543
8576
  }
8577
+ providerCallStartedAt = Date.now();
8544
8578
  const heartbeatIntervalMs = hasReceiptHeartbeat
8545
8579
  ? runtimeLeaseHeartbeatIntervalFromExpiry({
8546
8580
  leaseExpiresAt: options?.receiptLeaseExpiresAt,
@@ -8640,10 +8674,11 @@ export class PlayContextImpl {
8640
8674
  `[perf] tool call id=${toolId} phase=integration_fetch_headers elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
8641
8675
  );
8642
8676
  }
8677
+ providerCallElapsedMs = Date.now() - providerCallStartedAt;
8643
8678
  this.resourceGovernor.observe({
8644
8679
  toolId,
8645
8680
  providerResourceKey: `tool:${toolId}`,
8646
- providerLatencyMs: Date.now() - providerCallStartedAt,
8681
+ providerLatencyMs: providerCallElapsedMs,
8647
8682
  providerSuccess: response.ok,
8648
8683
  provider429: response.status === 429,
8649
8684
  });
@@ -8654,16 +8689,18 @@ export class PlayContextImpl {
8654
8689
  if (heartbeatFailure) {
8655
8690
  throw heartbeatFailure;
8656
8691
  }
8657
- if (abortController?.signal.aborted) {
8658
- throw abortController.signal.reason instanceof Error
8692
+ const transportError = abortController?.signal.aborted
8693
+ ? abortController.signal.reason instanceof Error
8659
8694
  ? abortController.signal.reason
8660
8695
  : new Error(
8661
8696
  `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
8662
- );
8663
- }
8697
+ )
8698
+ : error;
8664
8699
  transportAttempt += 1;
8665
8700
  const message =
8666
- error instanceof Error ? error.message : String(error);
8701
+ transportError instanceof Error
8702
+ ? transportError.message
8703
+ : String(transportError);
8667
8704
  if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
8668
8705
  await this.governor.chargeBudget('retry');
8669
8706
  const retryAfterMs =
@@ -8679,8 +8716,11 @@ export class PlayContextImpl {
8679
8716
  await this.sleepWithCheckpointHeartbeat(retryAfterMs);
8680
8717
  continue;
8681
8718
  }
8682
- throw new Error(
8719
+ throw new ToolHttpError(
8683
8720
  `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
8721
+ null,
8722
+ 0,
8723
+ 'repairable',
8684
8724
  );
8685
8725
  } finally {
8686
8726
  if (timeoutHandle) {
@@ -8711,6 +8751,9 @@ export class PlayContextImpl {
8711
8751
  bodyText: text,
8712
8752
  retryAfterHeader: response.headers.get('retry-after'),
8713
8753
  transientHttpRetrySafe: retrySafeTransientHttp,
8754
+ ...(providerCallElapsedMs !== null
8755
+ ? { providerLatencyMs: providerCallElapsedMs }
8756
+ : {}),
8714
8757
  });
8715
8758
  if (failure.backpressureDelayMs !== null) {
8716
8759
  // Feed the server-observed Retry-After back into the shared
@@ -214,6 +214,7 @@ export type PlayStep =
214
214
  | {
215
215
  type: 'play_call';
216
216
  playId: string;
217
+ execution?: 'inline' | 'child-workflow';
217
218
  results?: PlayStepRowResult[];
218
219
  nestedSteps: PlayStep[];
219
220
  description?: string;
@@ -252,6 +253,7 @@ export type PlayDatasetSubstep =
252
253
  | {
253
254
  type: 'play_call';
254
255
  playId: string;
256
+ execution?: 'inline' | 'child-workflow';
255
257
  results?: PlayStepRowResult[];
256
258
  nestedSteps: PlayStep[];
257
259
  description?: string;