deepline 0.1.212 → 0.1.213

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -40,7 +40,6 @@ import {
   mapRowOutcomeRuntimeFields,
   resolveMapRowOutcomeKey,
 } from './map-row-outcome';
-import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
 import { buildChildRunId } from './child-run-id';
 import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
 import {
@@ -1521,12 +1520,14 @@ export class PlayContextImpl {
 
   private recordPlayCallStep(input: {
     playId: string;
+    execution?: 'inline' | 'child-workflow';
     description?: string | null;
     nestedSteps?: PlayStep[];
   }): void {
     const step = {
       type: 'play_call' as const,
       playId: input.playId,
+      ...(input.execution ? { execution: input.execution } : {}),
       nestedSteps: input.nestedSteps ?? [],
       description: normalizeStepDescription(input.description ?? undefined),
     };
@@ -1903,6 +1904,7 @@ export class PlayContextImpl {
     }
     const claimed = await this.#options.claimRuntimeStepReceipt({
       key,
+      leaseId: `receipt-lease:${crypto.randomUUID()}`,
       runId: this.currentReceiptOwnerRunId,
       runAttempt: this.currentRunAttempt,
       leaseAware: true,
@@ -1979,30 +1981,6 @@ export class PlayContextImpl {
     };
   }
 
-  private async skipRuntimeStepReceipt(
-    key: string,
-    runId: string,
-    leaseId?: string | null,
-  ): Promise<RuntimeStepReceipt | null> {
-    if (!this.#options.skipRuntimeStepReceipt) {
-      return null;
-    }
-    const skipped = await this.#options.skipRuntimeStepReceipt({
-      key,
-      runId,
-      runAttempt: this.currentRunAttempt,
-      ...(leaseId ? { leaseId } : {}),
-    });
-    if (!skipped || typeof skipped.key !== 'string' || !skipped.key.trim()) {
-      return null;
-    }
-    return {
-      ...skipped,
-      key: skipped.key.trim(),
-      runId: skipped.runId ?? null,
-    };
-  }
-
   private async heartbeatRuntimeStepReceipt(
     key: string,
     _runId: string,
@@ -2310,6 +2288,11 @@ export class PlayContextImpl {
       ? await this.dispatchChunkedRuntimeReceiptRequest(uniqueKeys, (chunk) =>
           claimReceipts({
             keys: chunk,
+            // A transport retry can replay this mutation after the store
+            // committed but before the response body was consumed. Stable
+            // caller-owned tokens distinguish that replay from a concurrent
+            // claimant without weakening the provider-call execution fence.
+            leaseIds: chunk.map(() => `receipt-lease:${crypto.randomUUID()}`),
             runId: this.currentReceiptOwnerRunId,
             runAttempt: this.currentRunAttempt,
             leaseAware: true,
@@ -2777,6 +2760,24 @@ export class PlayContextImpl {
         this.#options.completeRuntimeStepReceipt ||
         this.#options.completeRuntimeStepReceipts,
       ),
+      ...(this.#options.acquireRuntimeReceiptExecutionLock &&
+      this.#options.releaseRuntimeReceiptExecutionLock
+        ? {
+            acquireExecutionLock: ({ receiptKey, ownerExecutionId, ttlMs }) =>
+              this.#options.acquireRuntimeReceiptExecutionLock!({
+                key: receiptKey,
+                runId: this.currentReceiptOwnerRunId,
+                ownerExecutionId,
+                ttlMs,
+              }),
+            releaseExecutionLock: ({ receiptKey, ownerExecutionId }) =>
+              this.#options.releaseRuntimeReceiptExecutionLock!({
+                key: receiptKey,
+                runId: this.currentReceiptOwnerRunId,
+                ownerExecutionId,
+              }),
+          }
+        : {}),
     };
   }
 
@@ -2834,6 +2835,8 @@ export class PlayContextImpl {
       onClaimedResult?: (output: T, receiptKey: string) => T;
       shouldPersistFailure?: (error: unknown) => boolean;
       markRunningBeforeExecute?: boolean;
+      requiresExecutionLock?: boolean;
+      executionLockTtlMs?: number;
       execute: (context: { leaseId: string | null }) => Promise<T>;
     },
   ): Promise<T> {
@@ -2865,6 +2868,9 @@ export class PlayContextImpl {
       onClaimedResult: opts.onClaimedResult,
       shouldPersistFailure: opts.shouldPersistFailure,
       markRunningBeforeExecute: opts.markRunningBeforeExecute,
+      completedCacheOnly: operation === 'tool',
+      requiresExecutionLock: opts.requiresExecutionLock,
+      executionLockTtlMs: opts.executionLockTtlMs,
       formatError: (error) => this.formatRuntimeError(error),
       log: (message) => this.log(message),
       execute: opts.execute,
@@ -4016,12 +4022,6 @@ export class PlayContextImpl {
         resolvedTableNamespace = normalizeTableNamespace(
           mapStartResult.tableNamespace,
         );
-        if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
-          throw new RuntimeSheetRowsBlockedError({
-            tableNamespace: resolvedTableNamespace,
-            blockedRows: mapStartResult.blockedRows ?? [],
-          });
-        }
 
         const persistedRowIdentity = (
           row: Record<string, unknown>,
@@ -4456,12 +4456,6 @@ export class PlayContextImpl {
       resolvedTableNamespace = normalizeTableNamespace(
         mapStartResult.tableNamespace,
       );
-      if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
-        throw new RuntimeSheetRowsBlockedError({
-          tableNamespace: resolvedTableNamespace,
-          blockedRows: mapStartResult.blockedRows ?? [],
-        });
-      }
       const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
         resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
       const pendingRowsByKey = new Map<string, Record<string, unknown>>();
@@ -4574,6 +4568,7 @@ export class PlayContextImpl {
     );
 
     this.activeMapCellMeta = new Map();
+    const staleCompletionKeys = new Set<string>();
     const persistMapRows = async (rows: PersistableMapRow[]) => {
       if (!this.#options.onMapRowsCompleted || rows.length === 0) {
         return;
@@ -4597,7 +4592,7 @@ export class PlayContextImpl {
           );
           const flushStartedAt = Date.now();
           try {
-            await this.#options.onMapRowsCompleted!({
+            const writeResult = await this.#options.onMapRowsCompleted!({
               playName: this.#options.playName,
               playId: this.#options.playId,
               runId: this.#options.runId,
@@ -4609,6 +4604,11 @@ export class PlayContextImpl {
               ),
               staticPipeline: this.#options.staticPipeline ?? null,
             });
+            if (writeResult) {
+              for (const key of writeResult.staleDroppedKeys ?? []) {
+                staleCompletionKeys.add(key);
+              }
+            }
             this.resourceGovernor.observe({
               sheetFlushBytes: chunkBytes,
               sheetFlushLatencyMs: Date.now() - flushStartedAt,
@@ -4654,6 +4654,7 @@ export class PlayContextImpl {
           completedRows: duplicateReuseCount,
         },
         {
+          emitTerminalEvent: false,
           onRowError: options?.onRowError,
           executionRowKeys: rowsToExecuteEntries.map((entry) => entry.rowKey),
           executionRowIndexes: rowsToExecuteEntries.map(
@@ -4751,7 +4752,49 @@ export class PlayContextImpl {
     }
     this.activeMapCellMeta = null;
 
-    if (this.#options.onMapRowsCompleted) {
+    if (staleCompletionKeys.size > 0) {
+      results = results.filter((row, index) => {
+        const key = rowIdentity(row, itemOriginalIndexes[index] ?? index);
+        return !staleCompletionKeys.has(key);
+      });
+    }
+
+    const durableCompletedRows = Math.max(
+      0,
+      reusedCount + mapResult.completedRows.length - staleCompletionKeys.size,
+    );
+    const durableFailedRows = mapResult.failedRows.length;
+    const terminalFrame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
+    if (terminalFrame) {
+      this.setMapFrame({
+        ...terminalFrame,
+        status: 'completed',
+        // Incremental rows add keys after their commit. Pure maps use the
+        // durable aggregate count below instead of materializing up to millions
+        // of keys only to mark the terminal frame.
+        completedRowKeys: terminalFrame.completedRowKeys,
+        pendingRowKeys: [],
+        completedRowsCount: durableCompletedRows,
+        pendingRowsCount: 0,
+        failedRowsCount: durableFailedRows,
+        activeBoundaryId: null,
+        updatedAt: Date.now(),
+      });
+    }
+    this.emitExecutionEvent({
+      type: 'map.completed',
+      mapInvocationId: mapScope.mapInvocationId,
+      mapNodeId: mapScope.mapNodeId ?? null,
+      logicalNamespace: mapScope.logicalNamespace,
+      artifactTableNamespace: resolvedTableNamespace,
+      completedRows: durableCompletedRows,
+      failedRows: durableFailedRows,
+      totalRows: totalInputCount,
+      ...this.inlineChildAggregateEventFields(),
+      at: Date.now(),
+    });
+
+    if (this.#options.onMapRowsCompleted && staleCompletionKeys.size === 0) {
       results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
         mapName: normalizedMapNamespace,
         tableNamespace: resolvedTableNamespace,
@@ -4988,11 +5031,21 @@ export class PlayContextImpl {
       mapName: normalizedTableNamespace,
       budgetBytes: runtimeOptions?.retainedRowsMemoryBudgetBytes,
     });
-    const enqueueIncrementalPersist = (row: PersistableMapRow): void => {
-      void incrementalPersistence?.persistRows([row]).catch(() => {
-        // The final map-level flush awaits the same chain and surfaces the
-        // persistence failure loudly without holding a completed row slot open.
-      });
+    const enqueueIncrementalPersist = (
+      row: PersistableMapRow,
+      onCommitted: () => void,
+    ): void => {
+      if (!incrementalPersistence) {
+        onCommitted();
+        return;
+      }
+      void incrementalPersistence
+        .persistRows([row])
+        .then(onCommitted)
+        .catch(() => {
+          // The final map-level flush awaits the same chain and surfaces the
+          // persistence failure loudly without holding a completed row slot open.
+        });
     };
 
     if (completedRows > 0 || pendingRows !== totalRows) {
@@ -5158,14 +5211,16 @@ export class PlayContextImpl {
       // this replaces ran AFTER every row had already computed, emitting 150k
       // post-hoc map.progress events and re-copying the completed-keys array
       // per row — all theater, no live progress value.
-      updateMapFrameProgress({
-        completedRowKeys: results.map((_row, index) =>
-          executionRowKey(
-            this.toOutputRow(items[index] as Record<string, unknown>),
-            index,
+      if (!incrementalPersistence) {
+        updateMapFrameProgress({
+          completedRowKeys: results.map((_row, index) =>
+            executionRowKey(
+              this.toOutputRow(items[index] as Record<string, unknown>),
+              index,
+            ),
           ),
-        ),
-      });
+        });
+      }
       if (emitTerminalEvent) {
         updateMapFrameProgress({
           status: 'completed',
@@ -5346,9 +5401,6 @@ export class PlayContextImpl {
             });
             retainedRowsMemoryTracker.track(failedRow);
             failedRowsToPersist.push(failedRow);
-            updateMapFrameProgress({
-              failedRowKey: rowKey,
-            });
             this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
               rowId: idx,
               status: 'failed',
@@ -5357,7 +5409,9 @@ export class PlayContextImpl {
               error: formattedError,
               dataPatch: {},
             });
-            enqueueIncrementalPersist(failedRow);
+            enqueueIncrementalPersist(failedRow, () => {
+              updateMapFrameProgress({ failedRowKey: rowKey });
+            });
             return FAILED_ROW;
           }
           const cellValue = this.serializeCellValue(value);
@@ -5404,9 +5458,6 @@ export class PlayContextImpl {
 
         const merged = cloneCsvAliasedRow(baseRow, computedFields);
         activeFieldName = null;
-        updateMapFrameProgress({
-          completedRowKey: rowKey,
-        });
         this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
           rowId: idx,
           status: 'completed',
@@ -5426,7 +5477,9 @@ export class PlayContextImpl {
         });
         retainedRowsMemoryTracker.track(completedRow);
         completedRowsToPersist.push(completedRow);
-        enqueueIncrementalPersist(completedRow);
+        enqueueIncrementalPersist(completedRow, () => {
+          updateMapFrameProgress({ completedRowKey: rowKey });
+        });
         return publicRow;
       } catch (error) {
         if (isPlayRowExecutionSuspendedError(error)) {
@@ -5597,6 +5650,11 @@ export class PlayContextImpl {
       (result): result is Record<string, unknown> =>
         result !== WAITING_ROW && result !== FAILED_ROW,
     );
+    // A row is acknowledged only after its terminal sheet batch commits. The
+    // incremental promises publish their frame updates from the commit
+    // continuation, so this barrier also makes the terminal map event derive
+    // from durable row truth instead of resolver completion.
+    await incrementalPersistence?.flush();
     if (emitTerminalEvent) {
       updateMapFrameProgress({
         status: 'completed',
@@ -6081,7 +6139,10 @@ export class PlayContextImpl {
     });
     let executionAuthScopeDigest =
       (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
-    const cacheableToolResult = !isQueryResultDatasetReadRequest(toolId, input);
+    const eventWaitHandler =
+      (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
+    const cacheableToolResult =
+      !eventWaitHandler && !isQueryResultDatasetReadRequest(toolId, input);
     let durableCacheKey = await this.durableToolCallCacheKey({
       toolId,
       requestInput: input,
@@ -6089,8 +6150,6 @@ export class PlayContextImpl {
       staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
     });
     const checkpointCacheKeys = [durableCacheKey];
-    const eventWaitHandler =
-      (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
     const store = rowContext.getStore();
 
     const executeTool = async (context?: {
@@ -6306,12 +6365,10 @@ export class PlayContextImpl {
       authScopeAttempt < 2;
       authScopeAttempt += 1
     ) {
-      // Capture the lease the receipt store hands to `execute` for THIS
-      // attempt's key. On an auth-scope re-claim we mark that old-scope receipt
-      // `skipped` (below), and the skip is fenced on this run+attempt+lease so
-      // it can only supersede the receipt this attempt owns.
-      let claimedReceiptLeaseId: string | null = null;
       try {
+        const toolRetryPolicy = await this.#options
+          .getToolRetryPolicy?.(toolId)
+          .catch(() => null);
         return await this.executeWithRuntimeReceipt(
           'tool',
           normalizedKey,
@@ -6320,6 +6377,15 @@ export class PlayContextImpl {
             receiptKey: durableCacheKey,
             semanticKey: toolRequestIdentity,
             force: toolCachePolicy.force,
+            // Missing metadata is intentionally lock-free. Only explicitly
+            // dangerous non-idempotent side effects enter the fence path.
+            requiresExecutionLock:
+              toolRetryPolicy?.requiresExecutionFence === true,
+            executionLockTtlMs: Math.min(
+              600_000,
+              resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs) ??
+                300_000,
+            ),
             staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
             onClaimedResult: (output, receiptKey) =>
               markToolExecuteResultExecutionOutcome(output, {
@@ -6337,10 +6403,7 @@ export class PlayContextImpl {
             shouldPersistFailure: (error) =>
               !(error instanceof ToolExecuteAuthScopeChangedError),
             markRunningBeforeExecute: false,
-            execute: ({ leaseId }) => {
-              claimedReceiptLeaseId = leaseId;
-              return executeTool({ leaseId });
-            },
+            execute: ({ leaseId }) => executeTool({ leaseId }),
             runningReceiptWaitMaxAttempts:
               resolveRuntimeToolReceiptWaitMaxAttempts(
                 typeof options?.receiptWaitMs === 'number'
@@ -6356,36 +6419,9 @@ export class PlayContextImpl {
         ) {
           throw error;
         }
-        // Supersede the old-scope receipt so the run's node projection does not
-        // surface it as `failed`. shouldPersistFailure kept it out of the failed
-        // state, but the claim left it `running` with a now-orphaned lease under
-        // the OLD cache key; without an explicit terminal it projects as a failed
-        // node even though the work is about to be re-done under the new scope.
-        // `skipped` is the terminal for superseded work; the skip is fenced on
-        // this run+attempt+lease so it can only touch the receipt this attempt
-        // owns. Best-effort: a skip that cannot fence (store returns null) must
-        // not mask the reclaim itself.
-        const supersededReceiptKey = durableCacheKey;
-        try {
-          const skipped = await this.skipRuntimeStepReceipt(
-            supersededReceiptKey,
-            this.currentRunId,
-            claimedReceiptLeaseId,
-          );
-          if (skipped) {
-            this.log(
-              `ctx.tools.execute(${toolId}): auth_scope_changed_superseded ` +
-                `old-scope receipt ${toolRequestIdentity} (label: ${normalizedKey}) ` +
-                `marked skipped so the reclaim does not project as a failed node.`,
-            );
-          }
-        } catch (skipError) {
-          this.log(
-            `ctx.tools.execute(${toolId}): auth_scope_changed_supersede_skip_failed ` +
-              `(label: ${normalizedKey}); could not mark the old-scope receipt ` +
-              `skipped: ${this.formatRuntimeError(skipError)}`,
-          );
-        }
+        // Completed-receipt cache misses never publish a running receipt. The
+        // old auth-scope attempt therefore has no ownership metadata to clean
+        // up; re-key under the refreshed credential scope and retry once.
         // Make the bounded auth-scope re-claim observable in run logs instead
         // of a silent continuation: the credential identity changed after the
         // receipt key was prepared, so we evict the cached digest, re-resolve,
@@ -6552,6 +6588,8 @@ export class PlayContextImpl {
       }
       const childExecutionDecision = resolveChildExecutionStrategy({
         pipeline: resolvedPlay.staticPipeline,
+        execution: options?.execution,
+        childPlayName: resolvedName,
       });
       const launchThroughScheduler =
         this.shouldLaunchChildPlaysThroughScheduler() &&
@@ -6696,6 +6734,7 @@ export class PlayContextImpl {
               }
               this.recordPlayCallStep({
                 playId: resolvedName,
+                execution: options?.execution,
                 description: options?.description,
               });
               return checkpointResult.output;
@@ -6743,6 +6782,7 @@ export class PlayContextImpl {
               }
               this.recordPlayCallStep({
                 playId: resolvedName,
+                execution: options?.execution,
                 description: options?.description,
               });
               return terminal.output;
@@ -6843,6 +6883,7 @@ export class PlayContextImpl {
             }
             this.recordPlayCallStep({
               playId: resolvedName,
+              execution: options?.execution,
               description: options?.description,
               nestedSteps: childContext.getSteps(),
             });

package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts

@@ -214,6 +214,7 @@ export type PlayStep =
   | {
       type: 'play_call';
       playId: string;
+      execution?: 'inline' | 'child-workflow';
       results?: PlayStepRowResult[];
       nestedSteps: PlayStep[];
       description?: string;
@@ -252,6 +253,7 @@ export type PlayDatasetSubstep =
   | {
       type: 'play_call';
       playId: string;
+      execution?: 'inline' | 'child-workflow';
       results?: PlayStepRowResult[];
       nestedSteps: PlayStep[];
       description?: string;

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -24,6 +24,7 @@ import type { PreviousCell } from './cell-staleness';
 import type { MapRowOutcome } from './durability-store';
 import type { WorkReceiptFailureKind } from './work-receipts';
 import type { RunExecutionScope } from './run-execution-scope';
+import type { PlayCallExecution } from './play-call-execution';
 
 export interface RowState {
   results: Map<string, unknown>;
@@ -69,12 +70,26 @@ export interface RuntimeStepReceipt {
   claimState?: 'claimed' | 'existing';
 }
 
+export type AcquireRuntimeReceiptExecutionLockInput = {
+  key: string;
+  runId: string;
+  ownerExecutionId: string;
+  ttlMs: number;
+};
+
+export type ReleaseRuntimeReceiptExecutionLockInput = {
+  key: string;
+  runId: string;
+  ownerExecutionId: string;
+};
+
 export interface GetRuntimeStepReceiptInput {
   key: string;
 }
 
 export interface ClaimRuntimeStepReceiptInput {
   key: string;
+  leaseId?: string;
   runId: string;
   runAttempt?: number | null;
   leaseAware?: boolean;
@@ -128,6 +143,7 @@ export interface GetRuntimeStepReceiptsInput {
 
 export interface ClaimRuntimeStepReceiptsInput {
   keys: string[];
+  leaseIds?: string[];
   runId: string;
   runAttempt?: number | null;
   leaseAware?: boolean;
@@ -281,6 +297,7 @@ export type CustomerDbQueryHandler = (
 
 export interface PlayCallOptions {
   description?: string;
+  execution?: PlayCallExecution;
 }
 
 export interface StepOptions {
@@ -597,7 +614,10 @@ export interface ContextOptions {
     rows: MapRowOutcome[];
     outputFields: string[];
     staticPipeline?: PlayStaticPipeline | null;
-  }) => Promise<void>;
+  }) => Promise<void | {
+    updated: number;
+    staleDroppedKeys?: string[];
+  }>;
   playId?: string;
   runId?: string;
   /** Physical executor run that owns leases for an inline child context. */
@@ -634,6 +654,7 @@ export interface ContextOptions {
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolRetryPolicy?: (toolId: string) => Promise<{
     retrySafeTransientHttp?: boolean;
+    requiresExecutionFence?: boolean;
   } | null>;
   getToolActionCacheVersion?: (toolId: string) => Promise<string> | string;
   getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
@@ -672,6 +693,12 @@ export interface ContextOptions {
   getRuntimeStepReceipt?: (
     input: GetRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  acquireRuntimeReceiptExecutionLock?: (
+    input: AcquireRuntimeReceiptExecutionLockInput,
+  ) => Promise<{ ownerExecutionId: string; expiresAt: string } | null>;
+  releaseRuntimeReceiptExecutionLock?: (
+    input: ReleaseRuntimeReceiptExecutionLockInput,
+  ) => Promise<boolean>;
   getRuntimeStepReceipts?: (
     input: GetRuntimeStepReceiptsInput,
   ) =>
@@ -932,6 +959,7 @@ export type PlayStep =
   | {
       type: 'play_call';
       playId: string;
+      execution?: PlayCallExecution;
       results?: PlayStepRowResult[];
       nestedSteps: PlayStep[];
       description?: string;
@@ -971,6 +999,7 @@ export type PlayDatasetSubstep =
   | {
       type: 'play_call';
       playId: string;
+      execution?: PlayCallExecution;
       results?: PlayStepRowResult[];
       nestedSteps: PlayStep[];
       description?: string;