deepline 0.1.211 → 0.1.213

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -68,6 +68,7 @@ import {
 } from './child-play-await';
 import { submitChildPlayThroughCoordinator } from './child-play-submit';
 import { isRetryableWorkerRuntimeApiError } from './runtime-api-retry';
+import { formatCustomerConsoleValue } from './runtime/customer-console';
 import {
   attachToolResultListDataset,
   createToolExecuteResult,
@@ -116,6 +117,7 @@ import {
   isQueryResultDatasetTool,
 } from '../../../shared_libs/play-runtime/query-result-dataset';
 import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
+import { createRuntimeDatasetId } from '../../../shared_libs/play-runtime/dataset-id';
 import {
   getTopLevelPipelineSubsteps,
   getCompiledPipelineSubsteps,
@@ -196,12 +198,12 @@ import {
   PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
   runtimeLeaseHeartbeatIntervalMs,
 } from '../../../shared_libs/play-runtime/lease-policy';
-import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
 import {
   resolveRuntimeMapRowAdmission,
   runtimeMapJsonByteLength,
 } from '../../../shared_libs/play-runtime/map-memory-limits';
 import { createSerialTaskQueue } from '../../../shared_libs/play-runtime/serial-task-queue';
+import { resolveChildExecutionPlacement as resolveChildExecutionStrategy } from '../../../shared_libs/play-runtime/child-execution-placement';
 // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
 // the long-lived Play Harness Worker via env.HARNESS. We import the
 // `setHarnessBinding` setter eagerly so it's available the moment
@@ -214,12 +216,10 @@ import { createSerialTaskQueue } from '../../../shared_libs/play-runtime/serial-
 // re-bundle harness internals into per-play. Keep that in mind.
 import {
   harnessPersistCompletedSheetRows,
-  harnessHeartbeatSheetAttempt,
   harnessReadSheetDatasetRowKeys,
   harnessReadSheetDatasetRows,
   harnessReadStagedFileChunk,
   harnessReleaseRuntimeReceipts,
-  harnessReleaseSheetAttempt,
   harnessStartSheetDataset,
   setHarnessBinding,
 } from '../../../sdk/src/plays/harness-stub';
@@ -300,7 +300,7 @@ import type {
   LiveNodeProgressMap,
   LiveNodeProgressSnapshot,
 } from './runtime/live-progress';
-import { extractErrorBilling } from './runtime/tool-http-errors';
+import { extractErrorBilling, ToolHttpError } from './runtime/tool-http-errors';
 import {
   WorkflowAbortError,
   isAbortLikeError,
@@ -324,6 +324,55 @@ import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/sche
 // @ts-ignore - virtual module substituted by esbuild plugin at bundle time.
 import playFn from 'deepline-play-entry';
 
+const customerConsoleStorage = new AsyncLocalStorage<
+  (message: string) => void
+>();
+
+type CustomerConsoleGlobal = typeof globalThis & {
+  __deeplineCaptureCustomerConsole?: (
+    method: string,
+    args: unknown[],
+  ) => unknown;
+};
+
+(globalThis as CustomerConsoleGlobal).__deeplineCaptureCustomerConsole = (
+  method,
+  args,
+) => {
+  const log = customerConsoleStorage.getStore();
+  if (log) {
+    assertNoSecretTaint(args, `console.${method}`);
+    const message = args.map(formatCustomerConsoleValue).join(' ');
+    log(`[console.${method}] ${message}`.slice(0, 2_000));
+  }
+};
+
+function runCustomerPlay(
+  ctx: unknown,
+  input: PlayRunInputPayload,
+): Promise<unknown> {
+  return customerConsoleStorage.run(
+    (ctx as { log(message: string): void }).log,
+    () => (playFn as typeof runCustomerPlay)(ctx, input),
+  );
+}
+
+function runtimeDatasetIdentity(playName: string, tableNamespace: string) {
+  return {
+    datasetId: createRuntimeDatasetId(playName, tableNamespace),
+    path: `datasets.${tableNamespace}`,
+    tableNamespace,
+  };
+}
+
+function datasetAvailableLog(
+  runId: string,
+  dataset: ReturnType<typeof runtimeDatasetIdentity>,
+  count: number,
+): string {
+  return `Dataset ${dataset.path} available: ${count} persisted rows; export: deepline runs export ${runId} --dataset ${dataset.path} --out ${dataset.tableNamespace}.csv`;
+}
+
 type RunRequest = {
   runId: string;
   callbackUrl: string;
@@ -999,6 +1048,17 @@ type WorkerCtxCallbacks = {
   }) => void | Promise<void>;
   onMapStarted?: (nodeId: string, at?: number) => void;
   onMapCompleted?: (nodeId: string, at?: number) => void;
+  onDatasetLifecycle?: (event: {
+    datasetId: string;
+    path: string;
+    tableNamespace: string;
+    phase: 'registered' | 'available' | 'failed';
+    persistedRows?: number;
+    succeededRows?: number;
+    failedRows?: number;
+    complete?: boolean;
+    at: number;
+  }) => void | Promise<void>;
   onToolCalled?: (toolId: string, at?: number) => void;
   onToolFailed?: (toolId: string, at?: number) => void;
 };
@@ -1201,136 +1261,6 @@ function makeRuntimeSheetAttempt(input: {
   };
 }
 
-class RuntimeSheetAttemptLeaseLostError extends Error {
-  readonly rowKeys: string[];
-  readonly runId: string;
-  readonly attemptId: string;
-  readonly tableNamespace: string;
-
-  constructor(input: {
-    rowKeys: string[];
-    runId: string;
-    attemptId: string;
-    tableNamespace: string;
-    cause?: unknown;
-  }) {
-    const rowSummary =
-      input.rowKeys.length === 1
-        ? input.rowKeys[0]!
-        : `${input.rowKeys.length} rows`;
-    super(
-      `Runtime sheet attempt ${input.attemptId} lost ownership of ${rowSummary} in ${input.tableNamespace}.`,
-      { cause: input.cause },
-    );
-    this.name = 'RuntimeSheetAttemptLeaseLostError';
-    this.rowKeys = input.rowKeys;
-    this.runId = input.runId;
-    this.attemptId = input.attemptId;
-    this.tableNamespace = input.tableNamespace;
-  }
-}
-
-async function executeWithRuntimeSheetAttemptHeartbeat<T>(input: {
-  req: RunRequest;
-  tableNamespace: string;
-  sheetContract: PlaySheetContract;
-  attemptId?: string | null;
-  attemptOwnerRunId?: string | null;
-  attemptSeq?: number | null;
-  rowKeys: string[];
-  execute: () => Promise<T> | T;
-}): Promise<T> {
-  const attemptId = input.attemptId?.trim();
-  const rowKeys = [
-    ...new Set(input.rowKeys.map((key) => key.trim()).filter(Boolean)),
-  ];
-  if (!attemptId || rowKeys.length === 0) {
-    return await input.execute();
-  }
-  const heartbeatIntervalMs = runtimePolicyHeartbeatIntervalMs(
-    input.req,
-    runtimeSheetAttemptLeaseTtlMs(input.req) ??
-      PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
-  );
-
-  let stopped = false;
-  let timeout: ReturnType<typeof setTimeout> | null = null;
-  let rejectLeaseLost!: (error: RuntimeSheetAttemptLeaseLostError) => void;
-  const leaseLost = new Promise<never>((_, reject) => {
-    rejectLeaseLost = reject;
-  });
-
-  const stop = () => {
-    stopped = true;
-    if (timeout !== null) {
-      clearTimeout(timeout);
-      timeout = null;
-    }
-  };
-
-  const heartbeatOnce = async () => {
-    const result = await harnessHeartbeatSheetAttempt({
-      ...runtimeSheetSessionScope(input.req),
-      tableNamespace: input.tableNamespace,
-      sheetContract: input.sheetContract,
-      runId: input.req.runId,
-      attemptId,
-      attemptOwnerRunId: input.attemptOwnerRunId ?? input.req.runId,
-      attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
-      attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
-      keys: rowKeys,
-    });
-    const renewed = new Set(result.renewedKeys);
-    const lostKeys = rowKeys.filter((key) => !renewed.has(key));
-    if (lostKeys.length > 0) {
-      throw new RuntimeSheetAttemptLeaseLostError({
-        rowKeys: lostKeys,
-        runId: input.req.runId,
-        attemptId,
-        tableNamespace: input.tableNamespace,
-      });
-    }
-  };
-
-  const schedule = () => {
-    timeout = setTimeout(() => {
-      void (async () => {
-        try {
-          await heartbeatOnce();
-        } catch (error) {
-          stop();
-          if (error instanceof RuntimeSheetAttemptLeaseLostError) {
-            rejectLeaseLost(error);
-          } else {
-            rejectLeaseLost(
-              new RuntimeSheetAttemptLeaseLostError({
-                rowKeys,
-                runId: input.req.runId,
-                attemptId,
-                tableNamespace: input.tableNamespace,
-                cause: error,
-              }),
-            );
-          }
-          return;
-        }
-        if (!stopped) {
-          schedule();
-        }
-      })();
-    }, heartbeatIntervalMs);
-  };
-
-  schedule();
-  const execution = Promise.resolve().then(input.execute);
-  execution.catch(() => {});
-  try {
-    return await Promise.race([execution, leaseLost]);
-  } finally {
-    stop();
-  }
-}
-
 /**
  * Strip credentials and JWT-shaped tokens from any string before it lands in
  * a log buffer or upstream error message. The harness routinely echoes
@@ -2073,6 +2003,7 @@ async function callToolDirect(
   while (true) {
     requestAttempt += 1;
     let res: Response;
+    const providerCallStartedAt = nowMs();
     try {
       const runtimeApiTimeout = resolveToolRuntimeApiTimeout({
         requestInput: input,
@@ -2161,8 +2092,11 @@ async function callToolDirect(
     } catch (error) {
       transportAttempt += 1;
       const message = error instanceof Error ? error.message : String(error);
-      lastError = new Error(
+      lastError = new ToolHttpError(
         `Tool ${toolId} transport failed calling ${path} for run ${req.runId} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}: ${message}`,
+        null,
+        0,
+        'repairable',
       );
       if (
         transportAttempt >= TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS ||
@@ -2212,6 +2146,7 @@ async function callToolDirect(
       bodyText: text,
       retryAfterHeader: res.headers.get('retry-after'),
       transientHttpRetrySafe,
+      providerLatencyMs: nowMs() - providerCallStartedAt,
     });
     lastError = failure.error;
     if (failure.backpressureDelayMs !== null) {
@@ -2458,10 +2393,7 @@ function isRunFatalWorkerRowError(error: unknown): boolean {
     isRowIsolationExemptError(error) ||
     isRuntimeReceiptPersistenceError(error) ||
     isRuntimePersistenceCircuitOpenError(error) ||
-    error instanceof RuntimeSheetAttemptLeaseLostError ||
-    error instanceof RuntimeLeaseLostError ||
-    (error instanceof Error &&
-      error.name === 'RuntimeSheetAttemptLeaseLostError')
+    error instanceof RuntimeLeaseLostError
   );
 }
 
@@ -3541,53 +3473,18 @@ function resolveSheetContractFromReq(
   }
 }
 
-/**
- * Release attempt-scoped leases after terminal settlement. Completed and failed
- * rows are durable; retaining their live fences only delays the next repair run.
- */
+/** Release temporary work-receipt ownership after terminal settlement. */
 async function releaseRuntimeLeasesOnSettlement(input: {
   req: RunRequest;
   leasedSheetNamespaces: Set<string>;
   emit: (event: RunnerEvent) => void;
   reason: 'completed' | 'run-fatal';
 }): Promise<void> {
-  const { req, leasedSheetNamespaces, emit, reason } = input;
+  const { req, emit, reason } = input;
+  void input.leasedSheetNamespaces;
   const reasonLabel =
     reason === 'completed' ? 'completed settlement' : 'run-fatal teardown';
   const scope = runtimeSheetSessionScope(req);
-  for (const tableNamespace of leasedSheetNamespaces) {
-    const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
-    if (!sheetContract) continue;
-    try {
-      const released = await harnessReleaseSheetAttempt({
-        ...scope,
-        tableNamespace,
-        sheetContract,
-        runId: req.runId,
-        attemptOwnerRunId: req.runId,
-        attemptSeq: req.runAttempt ?? 0,
-      });
-      if (released.released > 0) {
-        emit({
-          type: 'log',
-          level: 'info',
-          message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on ${reasonLabel}.`,
-          ts: nowMs(),
-        });
-      }
-    } catch (releaseError) {
-      emit({
-        type: 'log',
-        level: 'warn',
-        message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on ${reasonLabel} (rerun will fall back to lease-TTL expiry): ${
-          releaseError instanceof Error
-            ? releaseError.message
-            : String(releaseError)
-        }`,
-        ts: nowMs(),
-      });
-    }
-  }
   try {
     const released = await harnessReleaseRuntimeReceipts({
       ...scope,
@@ -3662,6 +3559,7 @@ async function persistCompletedMapRows(input: {
   attemptOwnerRunId?: string | null;
   attemptExpiresAt?: string | null;
   attemptSeq?: number | null;
+  writeVersion?: number | null;
   budgetMeter?: WorkerWorkBudgetMeter;
   force?: boolean;
   forceFailedRows?: boolean;
@@ -3671,6 +3569,8 @@ async function persistCompletedMapRows(input: {
   visible: number;
   retryWritten: number | null;
   retryVisible: number | null;
+  staleDropped: number;
+  staleDroppedKeys: string[];
 }> {
   if (input.rows.length === 0) {
     return {
@@ -3679,6 +3579,8 @@ async function persistCompletedMapRows(input: {
       visible: 0,
       retryWritten: null,
       retryVisible: null,
+      staleDropped: 0,
+      staleDroppedKeys: [],
     };
   }
   const { req, tableNamespace } = input;
@@ -3714,6 +3616,7 @@ async function persistCompletedMapRows(input: {
     attemptOwnerRunId: input.attemptOwnerRunId ?? null,
     attemptExpiresAt: input.attemptExpiresAt ?? null,
     attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
+    writeVersion: input.writeVersion ?? null,
     forceFailedRows: input.forceFailedRows === true,
   };
   input.budgetMeter?.count('sheet');
@@ -3747,24 +3650,8 @@ async function persistCompletedMapRows(input: {
     return result.rows.length;
   };
   const result = await harnessPersistCompletedSheetRows(persistRequest);
-  const hasAttemptFence = Boolean(input.attemptId?.trim());
-  const assertAttemptStillOwnsTerminalWrites = (
-    written: number,
-    visible: number | null,
-  ) => {
-    // Recovery finalization may include rows completed by an earlier attempt.
-    // A complete terminal readback proves they were not lost.
-    if (
-      !hasAttemptFence ||
-      written === rows.length ||
-      (visible !== null && visible >= expectedVisibleRows)
-    ) {
-      return;
-    }
-    throw new Error(
-      `Runtime sheet attempt lost ownership for ${tableNamespace}: wrote ${written}/${rows.length}; run ${req.runId}; attempt ${input.attemptId}.`,
-    );
-  };
+  const staleDropped = result.staleDropped ?? 0;
+  const staleDroppedKeys = result.staleDroppedKeys ?? result.fencedKeys ?? [];
   console.warn('[play-runner.persist_completed_map_rows.result]', {
     runId: req.runId,
     playName: req.playName,
@@ -3793,6 +3680,7 @@ async function persistCompletedMapRows(input: {
         attemptOwnerRunId: input.attemptOwnerRunId ?? null,
         attemptExpiresAt: input.attemptExpiresAt ?? null,
         attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
+        writeVersion: input.writeVersion ?? null,
         attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
         inputOffset: 0,
       });
@@ -3801,7 +3689,6 @@ async function persistCompletedMapRows(input: {
       input.budgetMeter?.count('sheet');
       retryWritten = retry.rowsWritten;
       retryVisible = await readVisibleRowCount();
-      assertAttemptStillOwnsTerminalWrites(retryWritten, retryVisible);
       if (retryVisible >= expectedVisibleRows) {
         return {
           rows: rows.length,
@@ -3809,13 +3696,19 @@ async function persistCompletedMapRows(input: {
           visible: visibleRows,
           retryWritten,
           retryVisible,
+          staleDropped: staleDropped + (retry.staleDropped ?? 0),
+          staleDroppedKeys: [
+            ...new Set([
+              ...staleDroppedKeys,
+              ...(retry.staleDroppedKeys ?? retry.fencedKeys ?? []),
+            ]),
+          ],
         };
       }
       throw new Error(
         `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
       );
     }
-    assertAttemptStillOwnsTerminalWrites(result.rowsWritten, visibleRows);
   }
   if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
     throw new Error(
@@ -3828,6 +3721,8 @@ async function persistCompletedMapRows(input: {
     visible: visibleRows,
     retryWritten,
     retryVisible,
+    staleDropped,
+    staleDroppedKeys,
   };
 }
 
@@ -3841,6 +3736,7 @@ async function applyLiveMapRowUpdates(input: {
   attemptOwnerRunId?: string | null;
   attemptExpiresAt?: string | null;
   attemptSeq?: number | null;
+  writeVersion?: number | null;
   budgetMeter?: WorkerWorkBudgetMeter;
 }): Promise<void> {
   if (input.updates.length === 0) return;
@@ -3891,6 +3787,7 @@ async function prepareMapRows(input: {
   attemptOwnerRunId?: string | null;
   attemptExpiresAt?: string | null;
   attemptSeq?: number | null;
+  writeVersion?: number | null;
   budgetMeter?: WorkerWorkBudgetMeter;
   force?: boolean;
 }): Promise<{
@@ -3903,6 +3800,7 @@ async function prepareMapRows(input: {
   attemptOwnerRunId?: string | null;
   attemptExpiresAt?: string | null;
   attemptSeq?: number | null;
+  writeVersion?: number | null;
 }> {
   if (input.rows.length === 0) {
     return {
@@ -3915,6 +3813,7 @@ async function prepareMapRows(input: {
       attemptOwnerRunId: input.attemptOwnerRunId ?? null,
       attemptExpiresAt: input.attemptExpiresAt ?? null,
       attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
+      writeVersion: input.writeVersion ?? null,
     };
   }
   const sessionScope = runtimeSheetSessionScope(input.req);
@@ -3934,6 +3833,7 @@ async function prepareMapRows(input: {
     attemptOwnerRunId: input.attemptOwnerRunId ?? null,
     attemptExpiresAt: input.attemptExpiresAt ?? null,
     attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
+    writeVersion: input.writeVersion ?? null,
     attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
     ...(input.force ? { force: true } : {}),
   });
@@ -3971,6 +3871,7 @@ async function prepareMapRows(input: {
     attemptExpiresAt: result.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
     attemptSeq:
       result.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
+    writeVersion: result.writeVersion ?? input.writeVersion ?? null,
   };
 }
 
@@ -4024,18 +3925,6 @@ function childPipelineUsesCtxDataset(
   });
 }
 
-function childPipelineNeedsWorkflowScheduler(
-  pipeline: PlayStaticPipeline | null | undefined,
-): boolean {
-  if (!pipeline) return false;
-  return flattenStaticPipeline(pipeline).some(
-    (substep) =>
-      substep.type === 'tool' &&
-      (substep.isEventWait === true ||
-        substep.toolId === 'test_wait_for_event'),
-  );
-}
-
 /**
  * Build the per-(org,provider) rate port the distributed Rate State Backend
  * RPCs through. When the coordinator binding (or its rate RPCs) is absent we
@@ -4864,7 +4753,75 @@ function createMinimalWorkerCtx(
       return kept;
     };
 
+    // Reserve the whole logical map before dispatching any provider work. One
+    // schedule-time version represents the user's intent across every chunk;
+    // a later run can supersede it without waiting, and an older chunk can
+    // never allocate a newer version halfway through the map.
+    const inputCapabilities = datasetHandleCapabilities(inputRows);
+    if (!inputCapabilities.replayable) {
+      throw new Error(
+        `ctx.dataset("${name}") requires a replayable input so all row write intents can be reserved before execution.`,
+      );
+    }
+    const reservationRows: Record<string, unknown>[] = [];
+    const reservationExplicitKeys =
+      opts?.key === undefined ? null : new Set<string>();
+    let reservationOffset = 0;
+    let reservationChunkIndex = 0;
+    for await (const chunk of iterDatasetChunks(inputRows, rowsPerChunk)) {
+      assertNotAborted(abortSignal);
+      for (let localIndex = 0; localIndex < chunk.length; localIndex += 1) {
+        const row = chunk[localIndex]!;
+        const absoluteIndex = baseOffset + reservationOffset + localIndex;
+        const explicitKeyValue = resolveExplicitKeyValue(row, absoluteIndex);
+        if (explicitKeyValue !== null && reservationExplicitKeys) {
+          if (reservationExplicitKeys.has(explicitKeyValue)) continue;
+          reservationExplicitKeys.add(explicitKeyValue);
+        }
+        const rowKey = resolveRowKey(
+          row,
+          absoluteIndex,
+          reservationChunkIndex,
+          localIndex,
+        );
+        reservationRows.push({
+          ...runtimeCsvStorageRow(row),
+          ...mapRowOutcomeRuntimeFields({
+            key: rowKey,
+            inputIndex: absoluteIndex,
+          }),
+        });
+      }
+      reservationOffset += chunk.length;
+      reservationChunkIndex += 1;
+    }
+    const mapWriteAttempt = makeRuntimeSheetAttempt({
+      runId: req.runId,
+      runAttempt: req.runAttempt,
+      mapName: name,
+      chunkIndex: 'finalize',
+    });
+    const mapReservation =
+      reservationRows.length === 0
+        ? null
+        : await harnessStartSheetDataset({
+            ...runtimeSheetSessionScope(req),
+            tableNamespace: name,
+            sheetContract: augmentSheetContractWithDatasetFields({
+              contract: requireSheetContract(req, name),
+              rows: reservationRows,
+              outputFields,
+            }),
+            rows: reservationRows,
+            runId: req.runId,
+            inputOffset: 0,
+            attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
+            ...mapWriteAttempt,
+          });
+    const mapWriteVersion = mapReservation?.writeVersion ?? null;
+
     let totalRowsWritten = 0;
+    let datasetLifecycleRegistered = false;
 
     const volatileWorkflowChunkRows = new Map<
       number,
@@ -4912,9 +4869,26 @@ function createMinimalWorkerCtx(
         })),
         inputOffset: baseOffset + chunkStart,
         ...rowAttempt,
+        writeVersion: mapWriteVersion,
         budgetMeter: workBudgetMeter,
         force: !!req.force,
       });
+      if (!datasetLifecycleRegistered) {
+        datasetLifecycleRegistered = true;
+        const dataset = runtimeDatasetIdentity(req.playName, name);
+        await callbacks?.onDatasetLifecycle?.({
+          ...dataset,
+          phase: 'registered',
+          complete: false,
+          at: nowMs(),
+        });
+        emitEvent({
+          type: 'log',
+          level: 'info',
+          message: `Dataset ${dataset.path} registered`,
+          ts: nowMs(),
+        });
+      }
       const activeRowAttempt = {
         attemptId: prepared.attemptId ?? rowAttempt.attemptId,
         attemptOwnerRunId:
@@ -4922,10 +4896,8 @@ function createMinimalWorkerCtx(
         attemptExpiresAt:
           prepared.attemptExpiresAt ?? rowAttempt.attemptExpiresAt,
         attemptSeq: prepared.attemptSeq ?? rowAttempt.attemptSeq,
+        writeVersion: prepared.writeVersion ?? null,
       };
-      // Record that this run holds attempt leases in `name`'s sheet so a
-      // run-fatal teardown releases them (the table now exists).
-      leasedSheetNamespaces?.add(name);
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.prepare_rows',
@@ -4961,8 +4933,6 @@ function createMinimalWorkerCtx(
       const pendingRowsByKey = new Map<string, Record<string, unknown>>();
       const completedKeys = new Set<string>();
       const completedRowsByKey = new Map<string, Record<string, unknown>>();
-      const blockedKeys = new Set<string>();
-      const blockedRowsByKey = new Map<string, Record<string, unknown>>();
       const preparedKeys = new Set<string>();
       for (const row of prepared.pendingRows) {
         const key =
@@ -4989,66 +4959,12 @@ function createMinimalWorkerCtx(
           resolveMapRowOutcomeKey(row) ??
           deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
         if (key) {
-          blockedKeys.add(key);
-          blockedRowsByKey.set(key, row);
           preparedKeys.add(key);
         }
       }
-      if (blockedKeys.size > 0) {
-        const rowKeySamples = Array.from(blockedKeys).slice(
-          0,
-          MAP_ROW_FAILURE_SAMPLE_LIMIT,
-        );
-        const message =
-          `ctx.dataset("${name}") chunk ${chunkIndex} is blocked by ` +
-          `${blockedKeys.size} active runtime sheet row lease(s) owned by another attempt. ` +
-          `Retry after the owning attempt finishes or the row leases expire.` +
-          (rowKeySamples.length > 0
-            ? ` Blocked row keys: ${rowKeySamples.join(', ')}`
-            : '');
-        recordRunnerPerfTrace({
-          req,
-          phase: 'runner.map_chunk.blocked_rows',
-          ms: nowMs() - prepareStartedAt,
-          extra: {
-            mapName: name,
-            chunkIndex,
-            rowsBlocked: blockedKeys.size,
-            rowKeySamples,
-          },
-        });
-        enqueueMapProgressUpdate({
-          completed: preparedCompletedRows,
-          total: progressTotalRows,
-          waitingRows: blockedKeys.size,
-          startedAt: mapStartedAt,
-          message,
-        });
-        return {
-          status: 'blocked',
-          chunkIndex,
-          rangeStart: baseOffset + chunkStart,
-          rangeEnd: baseOffset + chunkStart + chunkRows.length,
-          rowsRead: chunkRows.length,
-          rowsBlocked: blockedKeys.size,
-          rowKeySamples,
-          blockedRowSamples: rowKeySamples.map((rowKey) => {
-            const blockedRow = blockedRowsByKey.get(rowKey);
-            const attemptId =
-              typeof blockedRow?.__deeplineAttemptId === 'string'
-                ? blockedRow.__deeplineAttemptId
-                : typeof blockedRow?._attempt_id === 'string'
-                  ? blockedRow._attempt_id
-                  : activeRowAttempt.attemptId;
-            return {
-              __deeplineRowKey: rowKey,
-              __deeplineAttemptId: attemptId,
-            };
-          }),
-          outputDatasetId: `map:${name}`,
-          message,
-        };
-      }
+      // Versioned scheduling never waits for row ownership. Compatibility
+      // adapters may still report an old `blockedRows` disposition; treat it
+      // as a stale/non-executable row and keep it out of this run's output.
       const missingPreparedRows = chunkEntries.filter(
         ({ rowKey }) => !preparedKeys.has(rowKey),
       );
@@ -5375,12 +5291,29 @@ function createMinimalWorkerCtx(
             ts: nowMs(),
           });
         }
+        const staleKeys = new Set(persistStats.staleDroppedKeys);
         for (const { executedIndex } of rowsToPersist) {
-          persistedExecutedIndexes.add(executedIndex);
+          const rowKey = uniqueRowsToExecuteEntries[executedIndex]!.rowKey;
+          if (staleKeys.has(rowKey)) continue;
+          if (!persistedExecutedIndexes.has(executedIndex)) {
+            persistedExecutedIndexes.add(executedIndex);
+            completedExecutedRows += 1;
+          }
         }
         for (const { executedIndex } of failedRowsToPersist) {
+          const rowKey = uniqueRowsToExecuteEntries[executedIndex]!.rowKey;
+          if (staleKeys.has(rowKey)) continue;
           persistedFailedIndexes.add(executedIndex);
         }
+        if (persistStats.staleDropped > 0) {
+          emitEvent({
+            type: 'log',
+            level: 'info',
+            message: `Dropped ${persistStats.staleDropped} stale runtime sheet write(s) for ctx.dataset("${name}"); a newer scheduled row version is visible.`,
+            ts: nowMs(),
+          });
+        }
+        reportChunkProgress(false);
       };
 
       const enqueuePersistExecutedRows = (): Promise<void> => {
@@ -5678,7 +5611,6 @@ function createMinimalWorkerCtx(
                       : undefined;
                   executedRows[myIndex] = enriched as T &
                     Record<string, unknown>;
-                  completedExecutedRows += 1;
                   await notePersistableRow(enriched);
                   reportChunkProgress(false);
                 } catch (rowError) {
@@ -5788,25 +5720,14 @@ function createMinimalWorkerCtx(
           return results;
         },
       );
-      const workerResults = await executeWithRuntimeSheetAttemptHeartbeat({
-        req,
-        tableNamespace: name,
-        sheetContract: requireSheetContract(req, name),
-        ...activeRowAttempt,
-        rowKeys: uniqueRowsToExecuteEntries.map((entry) => entry.rowKey),
-        execute: async () => {
-          while (!workerSettled) {
-            await Promise.race([
-              workerResultsPromise,
-              sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
-            ]);
-            if (!workerSettled) {
-              reportExecutionHeartbeat(false);
-            }
-          }
-          return await workerResultsPromise;
-        },
-      });
+      while (!workerSettled) {
+        await Promise.race([
+          workerResultsPromise,
+          sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
+        ]);
+        if (!workerSettled) reportExecutionHeartbeat(false);
+      }
+      const workerResults = await workerResultsPromise;
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.execute_workers',
@@ -6004,6 +5925,7 @@ function createMinimalWorkerCtx(
         executedIndex < executedRows.length;
         executedIndex += 1
       ) {
+        if (!persistedExecutedIndexes.has(executedIndex)) continue;
         const executedRow = executedRows[executedIndex];
         const key = uniqueRowsToExecuteEntries[executedIndex]!.rowKey;
         // Failed rows have no executed result; they stay out of the map output
@@ -6039,6 +5961,8 @@ function createMinimalWorkerCtx(
         ...mapRowOutcomeRuntimeFields({ key, inputIndex }),
       }));
       const completedRowsReused = completedChunkEntries.length;
+      const committedExecutedRows =
+        persistedExecutedIndexes.size + persistedFailedIndexes.size;
       const hashStartedAt = nowMs();
       const hash = await hashJson(publicOut);
       const includeCachedRowsInChunkResult = !workflowStep;
@@ -6068,8 +5992,8 @@ function createMinimalWorkerCtx(
           chunkIndex,
           rowsRead: chunkRows.length,
           rowsWritten: out.length,
-          rowsExecuted: executedRows.length,
-          rowsFailed: failedExecutedRows,
+          rowsExecuted: committedExecutedRows,
+          rowsFailed: persistedFailedIndexes.size,
           rowsCached: completedRowsReused,
         },
       });
@@ -6080,12 +6004,12 @@ function createMinimalWorkerCtx(
         rangeEnd: baseOffset + chunkStart + out.length,
         rowsRead: chunkRows.length,
         rowsWritten: out.length,
-        rowsExecuted: executedRows.length,
+        rowsExecuted: committedExecutedRows,
         rowsCached: completedRowsReused,
         rowsDuplicateReused: duplicateInputReuseCount,
         rowsInserted,
         rowsSkipped,
-        rowsFailed: failedExecutedRows,
+        rowsFailed: persistedFailedIndexes.size,
         rowFailureSamples,
         stepCellsCompleted,
         stepCellsSkipped,
@@ -6117,12 +6041,9 @@ function createMinimalWorkerCtx(
     let totalRowsInserted = 0;
     let totalRowsSkipped = 0;
     let totalRowsFailed = 0;
-    let totalRowsBlocked = 0;
     let totalStepCellsCompleted = 0;
     let totalStepCellsSkipped = 0;
     const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
-    const blockedRowKeySamples: string[] = [];
-    const blockedRowSamples: Record<string, unknown>[] = [];
 
     const runChunkStep = async (
       chunkRows: T[],
@@ -6261,6 +6182,23 @@ function createMinimalWorkerCtx(
                 finalizedRowsWritten,
               ),
       });
+      const dataset = runtimeDatasetIdentity(req.playName, name);
+      const persistedRows = finalizedRowsWritten + totalRowsFailed;
+      await callbacks?.onDatasetLifecycle?.({
+        ...dataset,
+        phase: 'available',
+        persistedRows,
+        succeededRows: finalizedRowsWritten,
+        failedRows: totalRowsFailed,
+        complete: true,
+        at: completedAt,
+      });
+      emitEvent({
+        type: 'log',
+        level: totalRowsFailed > 0 ? 'warn' : 'info',
+        message: datasetAvailableLog(req.runId, dataset, persistedRows),
+        ts: completedAt,
+      });
       callbacks?.onMapCompleted?.(mapNodeId, completedAt);
       emitEvent({
         type: 'log',
@@ -6299,6 +6237,7 @@ function createMinimalWorkerCtx(
             runId: req.runId,
             inputOffset: 0,
             attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
+            writeVersion: mapWriteVersion,
             ...repairAttempt,
           });
           const activeRepairAttempt = {
@@ -6308,6 +6247,7 @@ function createMinimalWorkerCtx(
             attemptExpiresAt:
               repairStart.attemptExpiresAt ?? repairAttempt.attemptExpiresAt,
             attemptSeq: repairStart.attemptSeq ?? repairAttempt.attemptSeq,
+            writeVersion: repairStart.writeVersion ?? null,
           };
           const repair = await persistCompletedMapRows({
             req,
@@ -6509,32 +6449,12 @@ function createMinimalWorkerCtx(
           );
         },
         onBatchComplete: async (chunkResult) => {
-          if (chunkResult.status === 'blocked') {
-            totalRowsBlocked += chunkResult.rowsBlocked;
-            for (const key of chunkResult.rowKeySamples) {
-              if (blockedRowKeySamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
-                break;
-              }
-              blockedRowKeySamples.push(key);
-            }
-            for (const row of chunkResult.blockedRowSamples) {
-              if (blockedRowSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
-                break;
-              }
-              blockedRowSamples.push(row);
-            }
-            emitEvent({
-              type: 'log',
-              level: 'warn',
-              message: chunkResult.message,
-              ts: nowMs(),
-            });
-            return true;
-          }
+          if (chunkResult.status === 'blocked') return true;
           if (chunkResult.fatalError) {
             throw new Error(
               `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
-                `outside the retryable chunk step. Provider calls already executed for ` +
+                `outside the retryable chunk step. Circuit breaker prevented queued row ` +
+                `dispatch. Provider calls already executed for ` +
                 `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
                 `Fix the runtime persistence cause and re-run to resume. ` +
                 `First error: ${chunkResult.fatalError}`,
@@ -6621,13 +6541,6 @@ function createMinimalWorkerCtx(
         ts: nowMs(),
       });
     }
-    if (totalRowsBlocked > 0) {
-      await flushQueuedMapProgressUpdates();
-      throw new RuntimeSheetRowsBlockedError({
-        tableNamespace: name,
-        blockedRows: blockedRowSamples,
-      });
-    }
     if (failFastRowErrors && totalRowsFailed > 0 && totalRowsWritten > 0) {
       // onRowError:'fail', PARTIAL failure (some rows committed): fail the run
       // without finalizing the dataset. The committed rows already persisted
@@ -6992,6 +6905,7 @@ function createMinimalWorkerCtx(
       options?: {
         description?: string;
         timeoutMs?: number;
+        execution?: 'inline' | 'child-workflow';
       },
     ): Promise<unknown> {
       const normalizedKey = normalizeContextKey(key, 'runPlay');
@@ -7066,8 +6980,35 @@ function createMinimalWorkerCtx(
           const childIsDatasetBacked = childPipelineUsesCtxDataset(
             childManifest.staticPipeline,
           );
+          const childWaitsForEvent = (
+            childManifest.staticPipeline
+              ? flattenStaticPipeline(childManifest.staticPipeline)
+              : []
+          ).some(
+            (step) =>
+              step.type === 'tool' &&
+              (step.isEventWait || step.toolId === 'test_wait_for_event'),
+          );
+          const authoredWorkflow =
+            options?.timeoutMs != null ||
+            childIsDatasetBacked ||
+            childWaitsForEvent;
+          if (options?.execution === 'inline' && authoredWorkflow) {
+            throw new Error(
+              `ctx.runPlay("${resolvedName}") requested inline execution for a child that requires a workflow.`,
+            );
+          }
+          const childExecutionDecision = resolveChildExecutionStrategy({
+            requiresWorkflow: authoredWorkflow,
+            execution: options?.execution,
+          });
           const childNeedsWorkflowScheduler =
-            childPipelineNeedsWorkflowScheduler(childManifest.staticPipeline);
+            childExecutionDecision.strategy === 'scheduled';
+          if (rowScope && childNeedsWorkflowScheduler) {
+            throw new Error(
+              `ctx.runPlay("${resolvedName}") cannot start a child workflow per dataset row.`,
+            );
+          }
           console.info('[play.runtime.span]', {
             event: 'play.runtime.span',
             phase: 'child_route',
@@ -7116,8 +7057,7 @@ function createMinimalWorkerCtx(
                 coordinatorBinding: cachedCoordinatorBinding,
                 makeRequestId,
                 coordinatorRequestHeaders,
-                allowInline:
-                  options?.timeoutMs == null && !childNeedsWorkflowScheduler,
+                allowInline: childExecutionDecision.strategy === 'inline',
                 body: {
                   name: resolvedName,
                   childIdempotencyKey: childInvocationId,
@@ -7607,9 +7547,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
       captureHarnessBinding(env);
       await probeHarnessOnce(env, runPrefix);
       const ctx = createMinimalWorkerCtx(req, emit, env);
-      const result = await (
-        playFn as (ctx: unknown, input: PlayRunInputPayload) => Promise<unknown>
-      )(ctx, req.runtimeInput);
+      const result = await runCustomerPlay(ctx, req.runtimeInput);
       emit({
         type: 'result',
         result,
@@ -8023,6 +7961,17 @@ async function executeRunRequest(
     },
     onMapStarted: (nodeId, at) => stepLifecycle?.onMapStarted(nodeId, at),
     onMapCompleted: (nodeId, at) => stepLifecycle?.onMapCompleted(nodeId, at),
+    onDatasetLifecycle: async (event) => {
+      const { at, ...dataset } = event;
+      pendingLedgerEvents.push({
+        ...dataset,
+        type: 'dataset.lifecycle',
+        runId: req.runId,
+        source: 'worker',
+        occurredAt: at,
+      });
+      await flushLedgerEvents(event.complete === true);
+    },
     onToolCalled: (toolId, at) => stepLifecycle?.onToolCalled(toolId, at),
     onToolFailed: (toolId, at) => stepLifecycle?.onToolFailed(toolId, at),
   };
@@ -8089,9 +8038,7 @@ async function executeRunRequest(
       : null;
   try {
     const playStartedAt = nowMs();
-    const result = await (
-      playFn as (ctx: unknown, input: PlayRunInputPayload) => Promise<unknown>
-    )(ctx, req.runtimeInput);
+    const result = await runCustomerPlay(ctx, req.runtimeInput);
     recordRunnerPerfTrace({
       req,
       phase: 'runner.play_function',
@@ -8135,7 +8082,33 @@ async function executeRunRequest(
         ms: nowMs() - ledgerFlushWaitStartedAt,
       });
       const resultDatasetStartedAt = nowMs();
-      await persistResultDatasets(req, promoted.handles, serializedResult);
+      const persistedResultCounts = await persistResultDatasets(
+        req,
+        promoted.handles,
+        serializedResult,
+      );
+      for (const dataset of promoted.handles) {
+        if (dataset.datasetKind === 'map') continue;
+        const identity = runtimeDatasetIdentity(
+          req.playName,
+          dataset.tableNamespace,
+        );
+        const count = persistedResultCounts.get(dataset.tableNamespace) ?? 0;
+        pendingLedgerEvents.push({
+          ...identity,
+          type: 'dataset.lifecycle',
+          runId: req.runId,
+          source: 'worker',
+          occurredAt: nowMs(),
+          phase: 'available',
+          persistedRows: count,
+          succeededRows: count,
+          failedRows: 0,
+          complete: true,
+        });
+        appendRunLogLine(datasetAvailableLog(req.runId, identity, count));
+      }
+      await flushLedgerEvents(true);
       resultDatasetsPersisted = true;
       recordRunnerPerfTrace({
         req,
@@ -8599,8 +8572,9 @@ async function persistResultDatasets(
   req: RunRequest,
   resultDatasets: ProjectedResultDatasetHandle[],
   serializedResult: unknown,
-): Promise<void> {
+): Promise<Map<string, number>> {
   const persistedNamespaces = new Set<string>();
+  const persistedCounts = new Map<string, number>();
   for (const dataset of resultDatasets) {
     if (dataset.datasetKind === 'map') continue;
     const handle = dataset.handle as WorkerDatasetHandle<
@@ -8639,6 +8613,7 @@ async function persistResultDatasets(
       inputOffset += chunk.length;
     }
     persistedNamespaces.add(dataset.tableNamespace);
+    persistedCounts.set(dataset.tableNamespace, inputOffset);
   }
 
   const datasets = collectDatasetEnvelopes(serializedResult);
@@ -8672,7 +8647,9 @@ async function persistResultDatasets(
       attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
       userEmail: req.userEmail,
     });
+    persistedCounts.set(dataset.tableNamespace, dataset.rows.length);
   }
+  return persistedCounts;
 }
 
 const RESULT_DATASET_PERSIST_CHUNK_ROWS = 5_000;