deepline 0.1.25 → 0.1.27

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -63,12 +63,37 @@ import {
   derivePlayRowIdentityFromKey,
 } from '../../../shared_libs/plays/row-identity';
 import {
+  getTopLevelPipelineSubsteps,
   getCompiledPipelineSubsteps,
   flattenStaticPipeline,
   resolveSheetContractForTableNamespace,
   sqlSafePlayColumnName,
+  type PlayStaticSubstep,
   type PlayStaticPipeline,
+  type PlaySheetContract,
 } from '../../../shared_libs/plays/static-pipeline';
+import {
+  PlayStepLifecycleTracker,
+  type PlayStepLifecycleEvent,
+} from '../../../shared_libs/play-runtime/step-lifecycle-tracker';
+import type {
+  PlayRunLedgerEvent,
+  PlayRunLedgerStepProgress,
+  PlayRunLedgerStepStatus,
+} from '../../../shared_libs/play-runtime/run-ledger';
+import {
+  createCsvDatasetHandle,
+  createInlineDatasetHandle,
+  createMaterializedDatasetHandle,
+  createPersistedDatasetHandle,
+  datasetRowCountHint,
+  isDatasetHandle,
+  iterDatasetChunks,
+  WORKER_DATASET_IN_MEMORY_ROWS,
+  WORKER_DATASET_PREVIEW_ROWS,
+  type WorkerDatasetHandle,
+  type WorkerDatasetInput,
+} from './runtime/dataset-handles';
 // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
 // the long-lived Play Harness Worker via env.HARNESS. We import the
 // `setHarnessBinding` setter eagerly so it's available the moment
@@ -80,9 +105,9 @@ import {
 // modules without going through this stub is how we'd accidentally
 // re-bundle harness internals into per-play. Keep that in mind.
 import {
-  harnessFetchStagedFile,
   harnessPersistCompletedSheetRows,
-  harnessPrewarmPostgresSessions,
+  harnessReadSheetDatasetRows,
+  harnessReadStagedFileChunk,
   harnessStartSheetDataset,
   setHarnessBinding,
 } from '../../../sdk/src/plays/harness-stub';
@@ -115,12 +140,14 @@ type RunRequest = {
   runtimeInput: Record<string, unknown>;
   /** Optional inline CSV rows (for plays where ctx.csv was passed inline data). */
   inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
-  /** R2 keys for input files keyed by logical filename (used by ctx.csv). */
-  inputR2Keys?: Record<string, string> | null;
+  /** Staged input files keyed by logical filename (used by ctx.csv). */
+  inputFiles?: Record<string, WorkerFileRef> | null;
   /** Files packaged with the play artifact (relative-path imports). */
   packagedFiles?: Array<{
     playPath: string;
     storageKey: string;
+    contentType?: string | null;
+    bytes?: number | null;
   }> | null;
   /** Partition fan-out: only process rows[start..end) of a sliced dataset. */
   partitionRange?: { start: number; end: number } | null;
@@ -148,6 +175,14 @@ type RunRequest = {
   totalRows?: number;
 };
 
+type WorkerFileRef = {
+  logicalPath: string;
+  fileName: string;
+  storageKey: string;
+  contentType?: string | null;
+  bytes?: number | null;
+};
+
 const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
 
 /** R2 binding injected by the Worker runtime (when present in deploy metadata). */
@@ -315,6 +350,7 @@ async function probeHarnessOnce(
  */
 const RUNTIME_API_TIMEOUT_MS = 30_000;
 const RUNTIME_API_PLAY_RUN_TIMEOUT_MS = 75_000;
+const RUNTIME_API_RETRY_DELAYS_MS = [250, 750, 1500] as const;
 let loggedMissingRuntimeApiBinding = false;
 
 async function fetchRuntimeApi(
@@ -383,132 +419,6 @@ const WORKER_PLAY_CALL_LIMITS = {
   maxConcurrentPlayCalls: 16,
 };
 
-/**
- * Produces a dataset-envelope-shaped object compatible with the legacy
- * SerializedPlayDataset shape (kind/datasetKind/count/columns/preview) so
- * tests + assertions that probe `result.rows.columns` etc. work without the
- * ctx changing semantics. Plays still iterate rows via array semantics.
- */
-function makeWorkerDataset<T extends Record<string, unknown>>(
-  name: string,
-  rows: T[],
-  options?: {
-    count?: number;
-    datasetKind?: 'csv' | 'map';
-    cacheSummary?: string | null;
-    workProgress?: {
-      total: number;
-      executed: number;
-      reused: number;
-      skipped: number;
-      pending: number;
-      failed: number;
-      degraded?: boolean;
-      duplicates?: {
-        exact?: number;
-        semantic?: number;
-        rejected?: number;
-      };
-    };
-  },
-): T[] & {
-  count(): Promise<number>;
-  peek(limit?: number): Promise<T[]>;
-  materialize(limit?: number): Promise<T[]>;
-  toJSON(): unknown;
-  datasetId: string;
-  tableNamespace: string;
-} {
-  const datasetId = `map:${name}`;
-  const count = Math.max(0, Math.floor(options?.count ?? rows.length));
-  const datasetKind = options?.datasetKind ?? 'map';
-  const cacheSummary = options?.cacheSummary ?? null;
-  const workProgress = options?.workProgress;
-  // Build the array result. JSON.stringify on arrays calls toJSON only if
-  // present on the array itself — we attach below. The dataset metadata is
-  // also exposed via own properties so plays can `enriched.count()` etc.
-  const arr = rows as T[] & {
-    count(): Promise<number>;
-    peek(limit?: number): Promise<T[]>;
-    materialize(limit?: number): Promise<T[]>;
-    toJSON(): unknown;
-    datasetId: string;
-    tableNamespace: string;
-  };
-  const previewLimit = 5;
-  const inferredColumns = (() => {
-    const cols = new Set<string>();
-    for (const r of rows) {
-      for (const k of Object.keys(r)) cols.add(k);
-    }
-    return [...cols];
-  })();
-  Object.defineProperty(arr, 'count', {
-    value: async () => count,
-    enumerable: false,
-  });
-  Object.defineProperty(arr, 'peek', {
-    value: async (limit = previewLimit) => rows.slice(0, Math.max(0, limit)),
-    enumerable: false,
-  });
-  Object.defineProperty(arr, 'materialize', {
-    value: async (limit?: number) =>
-      limit === undefined ? [...rows] : rows.slice(0, Math.max(0, limit)),
-    enumerable: false,
-  });
-  Object.defineProperty(arr, 'datasetId', {
-    value: datasetId,
-    enumerable: true,
-  });
-  Object.defineProperty(arr, 'tableNamespace', {
-    value: name,
-    enumerable: true,
-  });
-  Object.defineProperty(arr, '__deeplineDatasetCount', {
-    value: count,
-    enumerable: false,
-  });
-  Object.defineProperty(arr, '__deeplineDatasetKind', {
-    value: datasetKind,
-    enumerable: false,
-  });
-  Object.defineProperty(arr, '__deeplineCacheSummary', {
-    value: cacheSummary,
-    enumerable: false,
-  });
-  Object.defineProperty(arr, '__deeplineWorkProgress', {
-    value: workProgress,
-    enumerable: false,
-  });
-  // Plays often `return { rows: dataset, count: N }`. JSON.stringify on the
-  // array would normally produce `[row, row, ...]` — we want the dataset
-  // envelope shape instead so assertions seeing `result.rows.columns` pass.
-  // toJSON on an array is honored by JSON.stringify per ES spec.
-  // toJSON includes ALL rows so the workflow DO can persist the full
-  // dataset to the sheet table. We clone via plain-object copy to avoid
-  // re-entrant toJSON resolution (the dataset IS an array; passing it back
-  // via `preview: arr` would recurse forever through this same toJSON).
-  Object.defineProperty(arr, 'toJSON', {
-    value: () => {
-      const plainRows = rows.map((r) => ({ ...r }));
-      return {
-        kind: 'dataset' as const,
-        datasetKind,
-        datasetId,
-        count,
-        columns: inferredColumns,
-        preview: plainRows,
-        tableNamespace: name,
-        ...(cacheSummary ? { cacheSummary } : {}),
-        ...(workProgress ? { _metadata: { workProgress } } : {}),
-      };
-    },
-    enumerable: false,
-  });
-  void previewLimit;
-  return arr;
-}
-
 type RunnerEvent =
   | {
       type: 'log';
@@ -533,12 +443,53 @@ type WorkerCtxCallbacks = {
     nodeId: string;
     progress: LiveNodeProgressSnapshot;
   }) => void;
+  onMapStarted?: (nodeId: string, at?: number) => void;
+  onMapCompleted?: (nodeId: string, at?: number) => void;
+  onToolCalled?: (toolId: string, at?: number) => void;
+  onToolFailed?: (toolId: string, at?: number) => void;
 };
 
 function nowMs(): number {
   return Date.now();
 }
 
+function getStaticSubstepNodeId(
+  substep: PlayStaticSubstep,
+  index: number,
+): string {
+  switch (substep.type) {
+    case 'csv':
+      return `csv:${substep.field || index}`;
+    case 'map':
+      return `map:${substep.tableNamespace ?? substep.field}`;
+    case 'tool':
+      return `tool:${substep.field}:${substep.toolId}`;
+    case 'waterfall':
+      return `waterfall:${substep.id ?? substep.field}`;
+    case 'play_call':
+      return `play_call:${substep.field}:${substep.playId}`;
+    case 'run_javascript':
+      return `run_javascript:${substep.alias}`;
+    case 'code':
+      return `code:${substep.field || index}`;
+    default:
+      return `node:${index}`;
+  }
+}
+
+function buildOrderedNodeList(
+  contractSnapshot: unknown,
+): Array<{ nodeId: string; type: string }> {
+  const snapshot = isRecord(contractSnapshot) ? contractSnapshot : null;
+  const substeps = getTopLevelPipelineSubsteps(
+    (snapshot?.staticPipeline as PlayStaticPipeline | null | undefined) ?? null,
+  );
+  return substeps.map((substep, index) => ({
+    nodeId: getStaticSubstepNodeId(substep, index),
+    type: substep.type,
+  }));
+}
+
 function recordRunnerPerfTrace(input: {
   req: RunRequest;
   phase: string;
@@ -557,7 +508,7 @@ function recordRunnerPerfTrace(input: {
     source: 'dynamic_worker' as const,
     runId: input.req.runId,
     phase: `runner.${input.phase}`,
-    ...(input.ms !== undefined ? { ms: input.ms } : {}),
+    ms: input.ms ?? 0,
     ...(input.extra ?? {}),
   };
   console.log(
@@ -614,44 +565,83 @@ async function postRuntimeApi<T>(
   // Routes through the in-process RUNTIME_API binding when present; otherwise
   // falls back to a public fetch against `${baseUrl}${path}`. Either path
   // hits the same handler with the same auth — only the transport changes.
-  const res = await fetchRuntimeApi(baseUrl, '/api/v2/plays/internal/runtime', {
-    method: 'POST',
-    headers: {
-      'content-type': 'application/json',
-      authorization: `Bearer ${executorToken}`,
-      'x-deepline-request-id': makeRequestId(),
-    },
-    body: JSON.stringify(body),
-  });
-  if (!res.ok) {
+  const serializedBody = JSON.stringify(body);
+  let lastError: unknown = null;
+  for (
+    let attempt = 0;
+    attempt <= RUNTIME_API_RETRY_DELAYS_MS.length;
+    attempt += 1
+  ) {
+    let res: Response;
+    try {
+      res = await fetchRuntimeApi(baseUrl, '/api/v2/plays/internal/runtime', {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          authorization: `Bearer ${executorToken}`,
+          'x-deepline-request-id': makeRequestId(),
+        },
+        body: serializedBody,
+      });
+    } catch (error) {
+      lastError = error;
+      if (
+        attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
+        !isRetryableRuntimeApiError(error)
+      ) {
+        throw error;
+      }
+      await sleepRuntimeApiRetry(attempt);
+      continue;
+    }
+
+    if (res.ok) {
+      return (await res.json()) as T;
+    }
+
     const text = await res.text().catch(() => '');
-    throw new Error(
-      `runtime API ${res.status}: ${redactSecretsFromLogString(text.slice(0, 500))}`,
-    );
+    const redacted = redactSecretsFromLogString(text.slice(0, 500));
+    lastError = new Error(`runtime API ${res.status}: ${redacted}`);
+    if (
+      attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
+      !isRetryableRuntimeApiResponse(res.status, text)
+    ) {
+      throw lastError;
+    }
+    await sleepRuntimeApiRetry(attempt);
   }
-  return (await res.json()) as T;
+  throw lastError instanceof Error ? lastError : new Error(String(lastError));
 }
 
-async function postDeeplineApi(
-  req: RunRequest,
-  path: string,
-  body: unknown,
-): Promise<void> {
-  const res = await fetch(`${req.baseUrl.replace(/\/$/, '')}${path}`, {
-    method: 'POST',
-    headers: {
-      'content-type': 'application/json',
-      authorization: `Bearer ${req.executorToken}`,
-      'x-deepline-request-id': makeRequestId(),
-    },
-    body: JSON.stringify(body),
-  });
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(
-      `Deepline API ${path} ${res.status}: ${redactSecretsFromLogString(text.slice(0, 500))}`,
-    );
+function isRetryableRuntimeApiError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT/i.test(
+    message,
+  );
+}
+
+function isRetryableRuntimeApiResponse(status: number, body: string): boolean {
+  if (
+    status === 408 ||
+    status === 429 ||
+    status === 502 ||
+    status === 503 ||
+    status === 504
+  ) {
+    return true;
   }
+  return (
+    status === 500 &&
+    /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|UND_ERR_CONNECT_TIMEOUT/i.test(
+      body,
+    )
+  );
+}
+
+async function sleepRuntimeApiRetry(attempt: number): Promise<void> {
+  await new Promise((resolve) =>
+    setTimeout(resolve, RUNTIME_API_RETRY_DELAYS_MS[attempt] ?? 0),
+  );
 }
 
 function describeRuntimeApiBody(body: unknown): string {
@@ -973,6 +963,21 @@ async function executeTool(
   return callToolDirect(req, args);
 }
 
+async function executeToolWithLifecycle(
+  req: RunRequest,
+  args: { id: string; toolId: string; input: Record<string, unknown> },
+  workflowStep: WorkflowStep | undefined,
+  callbacks: WorkerCtxCallbacks | undefined,
+): Promise<ToolExecuteResult> {
+  callbacks?.onToolCalled?.(args.toolId, nowMs());
+  try {
+    return await executeTool(req, args, workflowStep);
+  } catch (error) {
+    callbacks?.onToolFailed?.(args.toolId, nowMs());
+    throw error;
+  }
+}
+
 function isToolExecuteRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
@@ -1032,14 +1037,19 @@ async function waitForSyntheticIntegrationEvent(
       ? Math.max(1, Math.round(input.timeout_ms))
       : 30_000;
   await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
-    action: 'update_run_status',
+    action: 'append_run_events',
     playId: req.runId,
-    status: 'running',
-    runtimeBackend: 'cf_workflows_dynamic_worker',
-    waitKind: 'integration_event_batch',
-    waitUntil: nowMs() + timeoutMs,
-    activeBoundaryId: `integration_event:${eventKey}`,
-    lastCheckpointAt: nowMs(),
+    events: [
+      {
+        type: 'log.appended',
+        runId: req.runId,
+        source: 'worker',
+        occurredAt: nowMs(),
+        lines: [
+          `Waiting for integration_event:${eventKey} for up to ${timeoutMs}ms.`,
+        ],
+      } satisfies PlayRunLedgerEvent,
+    ],
   });
   try {
     const event = (await (
@@ -1751,6 +1761,7 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
   outputDatasetId: string;
   hash: string;
   preview: T[];
+  cachedRows?: T[];
 };
 
 function toWorkflowSerializableValue<T>(value: T): T {
@@ -1932,6 +1943,8 @@ async function executeWorkerWaterfall(
   toolNameOrSpec: string | WorkerInlineWaterfallSpec,
   input: Record<string, unknown>,
   opts?: WorkerWaterfallOptions,
+  callbacks?: WorkerCtxCallbacks,
+  workflowStep?: WorkflowStep,
 ): Promise<unknown | null> {
   // Inline-spec form
   if (typeof toolNameOrSpec === 'object' && toolNameOrSpec) {
@@ -1948,20 +1961,32 @@ async function executeWorkerWaterfall(
                 toolId?: unknown,
                 toolInput?: unknown,
               ) =>
-                await executeTool(
+                await executeToolWithLifecycle(
                   req,
                   normalizeToolExecuteArgs(requestOrKey, toolId, toolInput),
+                  workflowStep,
+                  callbacks,
                 ),
             },
             tool: async (key, toolId, toolInput) =>
-              await executeTool(req, { id: key, toolId, input: toolInput }),
+              await executeToolWithLifecycle(
+                req,
+                { id: key, toolId, input: toolInput },
+                workflowStep,
+                callbacks,
+              ),
           });
         } else {
-          result = await executeTool(req, {
-            id: step.id,
-            toolId: step.toolId,
-            input: step.mapInput(input),
-          });
+          result = await executeToolWithLifecycle(
+            req,
+            {
+              id: step.id,
+              toolId: step.toolId,
+              input: step.mapInput(input),
+            },
+            workflowStep,
+            callbacks,
+          );
         }
       } catch {
         continue;
@@ -2047,7 +2072,12 @@ async function executeWorkerWaterfall(
   const providers = opts?.providers ?? [];
   if (providers.length === 0) {
     try {
-      return await executeTool(req, { id: toolName, toolId: toolName, input });
+      return await executeToolWithLifecycle(
+        req,
+        { id: toolName, toolId: toolName, input },
+        workflowStep,
+        callbacks,
+      );
     } catch {
       return null;
     }
@@ -2055,11 +2085,16 @@ async function executeWorkerWaterfall(
   let lastError: Error | null = null;
   for (const provider of providers) {
     try {
-      const result = await executeTool(req, {
-        id: `${toolName}:${provider}`,
-        toolId: toolName,
-        input: { ...input, provider },
-      });
+      const result = await executeToolWithLifecycle(
+        req,
+        {
+          id: `${toolName}:${provider}`,
+          toolId: toolName,
+          input: { ...input, provider },
+        },
+        workflowStep,
+        callbacks,
+      );
       if (resultHasContent(result)) {
         recorder.push({
           waterfallId: toolName,
@@ -2114,6 +2149,118 @@ function makeCsvParserState(): CsvParserState {
   return { field: '', row: [], inQuotes: false, pendingCr: false };
 }
 
+function normalizeExpectedBytes(value: unknown): number | null {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
+    ? value
+    : null;
+}
+
+function hasByteLengthMismatch(
+  expectedBytes: number | null | undefined,
+  actualBytes: number | null | undefined,
+): boolean {
+  return (
+    typeof expectedBytes === 'number' &&
+    typeof actualBytes === 'number' &&
+    actualBytes !== expectedBytes
+  );
+}
+
+async function* iterReadableStreamChunks(
+  body: ReadableStream<Uint8Array>,
+): AsyncGenerator<Uint8Array, void, void> {
+  const reader = body.getReader();
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) return;
+      if (value && value.byteLength > 0) yield value;
+    }
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+function singleByteChunk(bytes: Uint8Array): AsyncIterable<Uint8Array> {
+  return {
+    async *[Symbol.asyncIterator]() {
+      if (bytes.byteLength > 0) yield bytes;
+    },
+  };
+}
+
+async function* guardExpectedByteChunks(input: {
+  req: RunRequest;
+  logicalPath: string;
+  storageKey: string;
+  source: string;
+  chunks: AsyncIterable<Uint8Array>;
+  expectedBytes?: number | null;
+  reportedBytes?: number | null;
+  fallback?: () => AsyncIterable<Uint8Array>;
+}): AsyncGenerator<Uint8Array, void, void> {
+  const expectedBytes =
+    normalizeExpectedBytes(input.expectedBytes) ??
+    normalizeExpectedBytes(input.reportedBytes);
+  let bytesRead = 0;
+  let sawChunk = false;
+  let skippedEmptyChunks = 0;
+
+  for await (const value of input.chunks) {
+    if (!value || value.byteLength === 0) {
+      skippedEmptyChunks += 1;
+      continue;
+    }
+    sawChunk = true;
+    bytesRead += value.byteLength;
+    yield value;
+  }
+
+  if (!sawChunk) {
+    if (typeof expectedBytes === 'number' && expectedBytes > 0) {
+      recordRunnerPerfTrace({
+        req: input.req,
+        phase: 'csv.open_empty_body',
+        extra: {
+          source: input.source,
+          logicalPath: input.logicalPath,
+          expectedBytes,
+          reportedBytes: normalizeExpectedBytes(input.reportedBytes),
+          skippedEmptyChunks,
+          storageKey: input.storageKey,
+        },
+      });
+      if (input.fallback) {
+        yield* input.fallback();
+        return;
+      }
+      throw new Error(
+        `ctx.csv("${input.logicalPath}"): ${input.source} returned an empty body for ` +
+          `${expectedBytes} byte staged file ${input.storageKey}.`,
+      );
+    }
+    return;
+  }
+
+  if (expectedBytes !== null && bytesRead !== expectedBytes) {
+    recordRunnerPerfTrace({
+      req: input.req,
+      phase: 'csv.read_mismatch',
+      extra: {
+        source: input.source,
+        logicalPath: input.logicalPath,
+        expectedBytes,
+        actualBytes: bytesRead,
+        storageKey: input.storageKey,
+      },
+    });
+    throw new Error(
+      `ctx.csv("${input.logicalPath}"): ${input.source} streamed ${bytesRead} bytes ` +
+        `for ${expectedBytes} byte staged file ${input.storageKey}.`,
+    );
+  }
+}
+
 /**
  * Push one buffered text chunk through the CSV state machine. Accumulates
  * fully-terminated rows into `out`; partial trailing field/row stays in
@@ -2182,11 +2329,10 @@ function flushCsvParser(state: CsvParserState, out: string[][]): void {
  * to every subsequent row. Stops cleanly on stream end and flushes any
  * trailing row.
  */
-async function* streamCsvRowsFromBody<T extends Record<string, unknown>>(
-  body: ReadableStream<Uint8Array>,
+async function* streamCsvRowsFromByteChunks<T extends Record<string, unknown>>(
+  byteChunks: AsyncIterable<Uint8Array>,
   chunkSize: number,
 ): AsyncGenerator<T[], void, void> {
-  const reader = body.getReader();
   const decoder = new TextDecoder('utf-8');
   const state = makeCsvParserState();
   const physicalRowBuffer: string[][] = [];
@@ -2195,7 +2341,13 @@ async function* streamCsvRowsFromBody<T extends Record<string, unknown>>(
 
   const flushPhysicalRowsAsObjects = (terminal: boolean): T[][] => {
     const yielded: T[][] = [];
-    if (physicalRowBuffer.length === 0) return yielded;
+    if (physicalRowBuffer.length === 0) {
+      if (terminal && pendingChunk.length > 0) {
+        yielded.push(pendingChunk);
+        pendingChunk = [];
+      }
+      return yielded;
+    }
     if (!headers) {
       headers = physicalRowBuffer.shift() ?? null;
       if (!headers) return yielded;
@@ -2219,224 +2371,240 @@ async function* streamCsvRowsFromBody<T extends Record<string, unknown>>(
     return yielded;
   };
 
-  try {
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) {
-        flushCsvParser(state, physicalRowBuffer);
-        for (const chunk of flushPhysicalRowsAsObjects(true)) yield chunk;
-        return;
-      }
-      const text = decoder.decode(value, { stream: true });
-      pushCsvTextIntoParser(state, text, physicalRowBuffer);
-      for (const chunk of flushPhysicalRowsAsObjects(false)) yield chunk;
-    }
-  } finally {
-    reader.releaseLock();
+  for await (const value of byteChunks) {
+    if (value.byteLength === 0) continue;
+    const text = decoder.decode(value, { stream: true });
+    pushCsvTextIntoParser(state, text, physicalRowBuffer);
+    for (const chunk of flushPhysicalRowsAsObjects(false)) yield chunk;
+  }
+  const trailingText = decoder.decode();
+  if (trailingText) {
+    pushCsvTextIntoParser(state, trailingText, physicalRowBuffer);
   }
+  flushCsvParser(state, physicalRowBuffer);
+  for (const chunk of flushPhysicalRowsAsObjects(true)) yield chunk;
   void TARGET_CSV_DECODE_CHUNK_BYTES; // referenced for future tuning
 }
 
+function readHarnessStagedFileChunks(input: {
+  req: RunRequest;
+  logicalPath: string;
+  storageKey: string;
+  expectedBytes?: number | null;
+}): AsyncIterable<Uint8Array> {
+  const expectedBytes = normalizeExpectedBytes(input.expectedBytes);
+  return {
+    async *[Symbol.asyncIterator]() {
+      let offset = 0;
+      let observedBytes = 0;
+      let objectSize: number | null = null;
+      let loggedOpen = false;
+      let done = false;
+
+      while (!done) {
+        const result = await harnessReadStagedFileChunk({
+          storageKey: input.storageKey,
+          offset,
+          length: TARGET_CSV_DECODE_CHUNK_BYTES,
+        });
+        const actualObjectSize = normalizeExpectedBytes(result.objectSize);
+        if (actualObjectSize === null) {
+          throw new Error(
+            `ctx.csv("${input.logicalPath}"): harness returned an invalid object size for ${input.storageKey}.`,
+          );
+        }
+        if (objectSize !== null && objectSize !== actualObjectSize) {
+          throw new Error(
+            `ctx.csv("${input.logicalPath}"): staged file size changed while streaming ` +
+              `${input.storageKey}; started at ${objectSize} bytes, now ${actualObjectSize}.`,
+          );
+        }
+        objectSize = actualObjectSize;
+        if (hasByteLengthMismatch(expectedBytes, actualObjectSize)) {
+          throw new Error(
+            `ctx.csv("${input.logicalPath}"): harness staged file size mismatch for ` +
+              `storageKey=${input.storageKey}; expected ${expectedBytes} bytes, got ${actualObjectSize}.`,
+          );
+        }
+        if (result.offset !== offset) {
+          throw new Error(
+            `ctx.csv("${input.logicalPath}"): harness returned offset ${result.offset} while ` +
+              `reading offset ${offset} from ${input.storageKey}.`,
+          );
+        }
+
+        const chunk = result.chunk;
+        if (!(chunk instanceof Uint8Array)) {
+          throw new Error(
+            `ctx.csv("${input.logicalPath}"): harness returned a non-byte chunk for ${input.storageKey}.`,
+          );
+        }
+        if (chunk.byteLength !== result.bytesRead) {
+          throw new Error(
+            `ctx.csv("${input.logicalPath}"): harness chunk metadata mismatch for ` +
+              `${input.storageKey}; bytesRead=${result.bytesRead}, chunk=${chunk.byteLength}.`,
+          );
+        }
+        if (chunk.byteLength === 0 && !result.done) {
+          throw new Error(
+            `ctx.csv("${input.logicalPath}"): harness returned an empty non-terminal chunk for ${input.storageKey}.`,
+          );
+        }
+
+        if (!loggedOpen) {
+          loggedOpen = true;
+          recordRunnerPerfTrace({
+            req: input.req,
+            phase: 'csv.open',
+            extra: {
+              source: 'harness_rpc_range',
+              logicalPath: input.logicalPath,
+              expectedBytes,
+              actualBytes: actualObjectSize,
+              chunkBytes: TARGET_CSV_DECODE_CHUNK_BYTES,
+              storageKey: input.storageKey,
+            },
+          });
+        }
+
+        offset += chunk.byteLength;
+        observedBytes += chunk.byteLength;
+        done = result.done;
+        if (chunk.byteLength > 0) yield chunk;
+      }
+
+      const requiredBytes = expectedBytes ?? objectSize;
+      if (
+        typeof requiredBytes === 'number' &&
+        observedBytes !== requiredBytes
+      ) {
+        recordRunnerPerfTrace({
+          req: input.req,
+          phase: 'csv.read_mismatch',
+          extra: {
+            source: 'harness_rpc_range',
+            logicalPath: input.logicalPath,
+            expectedBytes: requiredBytes,
+            actualBytes: observedBytes,
+            storageKey: input.storageKey,
+          },
+        });
+        throw new Error(
+          `ctx.csv("${input.logicalPath}"): harness streamed ${observedBytes} bytes ` +
+            `for ${requiredBytes} byte staged file ${input.storageKey}.`,
+        );
+      }
+    },
+  };
+}
+
 /**
- * R2 reader that returns a body stream. Per-play Workers loaded via
+ * Dataset source adapter that returns byte chunks. Per-play Workers loaded via
  * WorkerLoader cannot accept a raw R2Bucket binding (CF Workflows refuses to
  * serialize R2Bucket through its workflow-state path), so per-play Workers
  * stream staged files through the long-lived harness Worker service binding.
  * Returns null only if the asset is genuinely missing (404).
  */
-async function openR2BodyStream(input: {
+async function openFileByteChunks(input: {
   req: RunRequest;
   env: WorkerEnv;
   logicalPath: string;
-  storageKey: string;
-}): Promise<ReadableStream<Uint8Array> | null> {
+  file: WorkerFileRef;
+}): Promise<AsyncIterable<Uint8Array> | null> {
+  const storageKey = input.file.storageKey;
+  const expectedBytes = normalizeExpectedBytes(input.file.bytes);
+  if (expectedBytes === null) {
+    throw new Error(
+      `ctx.csv("${input.logicalPath}"): staged dataset handle is missing a byte length for ${storageKey}.`,
+    );
+  }
   if (input.env.PLAYS_BUCKET) {
-    const object = await input.env.PLAYS_BUCKET.get(input.storageKey);
+    const object = await input.env.PLAYS_BUCKET.get(storageKey);
     if (object) {
-      return object.body;
+      if (hasByteLengthMismatch(expectedBytes, object.size)) {
+        recordRunnerPerfTrace({
+          req: input.req,
+          phase: 'csv.open_mismatch',
+          extra: {
+            source: 'direct_r2',
+            logicalPath: input.logicalPath,
+            expectedBytes,
+            actualBytes: object.size,
+            storageKey,
+          },
+        });
+        await object.body.cancel().catch(() => undefined);
+      } else {
+        recordRunnerPerfTrace({
+          req: input.req,
+          phase: 'csv.open',
+          extra: {
+            source: 'direct_r2',
+            logicalPath: input.logicalPath,
+            expectedBytes,
+            actualBytes: object.size,
+            storageKey,
+          },
+        });
+        return guardExpectedByteChunks({
+          req: input.req,
+          logicalPath: input.logicalPath,
+          storageKey,
+          source: 'direct_r2',
+          chunks: iterReadableStreamChunks(object.body),
+          expectedBytes,
+          reportedBytes: object.size,
+          fallback: () =>
+            readHarnessStagedFileChunks({
+              req: input.req,
+              logicalPath: input.logicalPath,
+              storageKey,
+              expectedBytes,
+            }),
+        });
+      }
     }
   }
   if (input.env.PLAY_ASSETS) {
     try {
       const text = await input.env.PLAY_ASSETS.readText(input.logicalPath);
       const bytes = new TextEncoder().encode(text);
-      return new ReadableStream<Uint8Array>({
-        start(controller) {
-          controller.enqueue(bytes);
-          controller.close();
+      if (hasByteLengthMismatch(expectedBytes, bytes.byteLength)) {
+        throw new Error(
+          `ctx.csv("${input.logicalPath}"): packaged asset size mismatch for ` +
+            `storageKey=${storageKey}; expected ${expectedBytes} bytes, got ${bytes.byteLength}.`,
+        );
+      }
+      recordRunnerPerfTrace({
+        req: input.req,
+        phase: 'csv.open',
+        extra: {
+          source: 'play_assets',
+          logicalPath: input.logicalPath,
+          expectedBytes,
+          actualBytes: bytes.byteLength,
+          storageKey,
         },
       });
+      return singleByteChunk(bytes);
     } catch (error) {
       if (!/missing from R2|not found|No such object/i.test(String(error))) {
-        throw error;
-      }
-    }
-  }
-
-  // The harness fetch path returns a real Response body backed by R2.
-  // Errors are loud: we want CI / regression failures to surface the real
-  // cause (auth, missing object, network) rather than getting squashed into a
-  // generic "R2 asset is not reachable".
-  const response = await harnessFetchStagedFile({
-    executorToken: input.req.executorToken,
-    storageKey: input.storageKey,
-  });
-  if (response.status === 404) {
-    throw new Error(
-      `ctx.csv("${input.logicalPath}"): harness R2 fetch returned 404 for storageKey=${input.storageKey}. ` +
-        `The staged file is missing from R2; the upload either failed silently before the run started, ` +
-        `or the storageKey threaded through the workflow params no longer matches what the harness resolves.`,
-    );
-  }
-  if (!response.ok || !response.body) {
-    const body = await response.text().catch(() => '');
-    throw new Error(
-      `ctx.csv("${input.logicalPath}"): harness R2 fetch failed ${response.status}: ${body.slice(0, 400)}`,
-    );
-  }
-  return response.body;
-}
-
-/**
- * Streaming CSV dataset. Backed by R2 (or a signed URL when PLAYS_BUCKET
- * isn't bound). Looks like a length-0 array to plays that pass it straight
- * to `ctx.map`; ctx.map detects the streaming surface via `iterChunks` and
- * uses it instead of `slice()`. Plays that try to access rows synchronously
- * (`csv[0]`, `csv.length`) are intentionally given an empty array — they
- * must use ctx.map (the supported surface), call `materialize()` (bounded),
- * or iterate via `for await (const row of csv)`.
- */
-type StreamingCsvDataset<T extends Record<string, unknown>> = T[] & {
-  count(): Promise<number>;
-  peek(limit?: number): Promise<T[]>;
-  materialize(limit?: number): Promise<T[]>;
-  iterChunks(chunkSize: number): AsyncIterable<T[]>;
-  toJSON(): unknown;
-  datasetId: string;
-  tableNamespace: string;
-  __deeplineDatasetKind: 'csv';
-  /** Marker so `ctx.map` can detect this is streaming-only and switch path. */
-  __deeplineStreamingDataset: true;
-};
-
-const MAX_MATERIALIZE_ROWS_DEFAULT = 50_000;
-
-function makeStreamingCsvDataset<T extends Record<string, unknown>>(input: {
-  name: string;
-  logicalPath: string;
-  renameOptions?: CsvRenameOptions;
-  open: () => Promise<ReadableStream<Uint8Array> | null>;
-}): StreamingCsvDataset<T> {
-  const datasetId = `csv:${input.name}`;
-  const arr = [] as T[] as StreamingCsvDataset<T>;
-  let cachedCount: number | null = null;
-
-  async function* doStream(chunkSize: number): AsyncGenerator<T[], void, void> {
-    const body = await input.open();
-    if (!body) {
-      throw new Error(
-        `ctx.csv("${input.logicalPath}"): R2 asset is not reachable (no PLAYS_BUCKET binding and signed URL unavailable).`,
-      );
-    }
-    for await (const chunk of streamCsvRowsFromBody<T>(
-      body,
-      Math.max(1, Math.floor(chunkSize)),
-    )) {
-      yield applyCsvRenameProjection(chunk, input.renameOptions) as T[];
+        throw error;
+      }
     }
   }
 
-  Object.defineProperty(arr, 'iterChunks', {
-    value: (chunkSize: number) => ({
-      [Symbol.asyncIterator]: () => doStream(chunkSize),
-    }),
-    enumerable: false,
-  });
-  Object.defineProperty(arr, Symbol.asyncIterator, {
-    value: async function* () {
-      for await (const chunk of doStream(1_000)) {
-        for (const row of chunk) yield row;
-      }
-    },
-    enumerable: false,
-  });
-  Object.defineProperty(arr, 'count', {
-    value: async () => {
-      if (cachedCount !== null) return cachedCount;
-      let total = 0;
-      for await (const chunk of doStream(5_000)) total += chunk.length;
-      cachedCount = total;
-      return total;
-    },
-    enumerable: false,
-  });
-  Object.defineProperty(arr, 'peek', {
-    value: async (limit = 10) => {
-      const out: T[] = [];
-      for await (const chunk of doStream(Math.max(1, limit))) {
-        for (const row of chunk) {
-          out.push(row);
-          if (out.length >= limit) return out;
-        }
-      }
-      return out;
-    },
-    enumerable: false,
-  });
-  Object.defineProperty(arr, 'materialize', {
-    value: async (limit?: number) => {
-      const cap = limit ?? MAX_MATERIALIZE_ROWS_DEFAULT;
-      const out: T[] = [];
-      for await (const chunk of doStream(5_000)) {
-        for (const row of chunk) {
-          if (out.length >= cap) {
-            return out;
-          }
-          out.push(row);
-        }
-      }
-      return out;
-    },
-    enumerable: false,
-  });
-  Object.defineProperty(arr, 'datasetId', {
-    value: datasetId,
-    enumerable: true,
-  });
-  Object.defineProperty(arr, 'tableNamespace', {
-    value: input.name,
-    enumerable: true,
-  });
-  Object.defineProperty(arr, '__deeplineStreamingDataset', {
-    value: true,
-    enumerable: false,
+  // Dynamic Workers cannot receive a raw R2Bucket binding, and both previous
+  // fallbacks were different data planes: service-binding fetch bodies could
+  // arrive empty across WorkerLoader isolates, while app-signed URLs pointed at
+  // the app namespace instead of the preview harness namespace. The harness owns
+  // staged R2 now, so the only fallback is typed bounded range RPC.
+  return readHarnessStagedFileChunks({
+    req: input.req,
+    logicalPath: input.logicalPath,
+    storageKey,
+    expectedBytes,
   });
-  Object.defineProperty(arr, '__deeplineDatasetKind', {
-    value: 'csv',
-    enumerable: false,
-  });
-  Object.defineProperty(arr, 'toJSON', {
-    value: () => ({
-      kind: 'dataset' as const,
-      datasetKind: 'csv',
-      datasetId,
-      count: cachedCount,
-      streaming: true,
-      tableNamespace: input.name,
-    }),
-    enumerable: false,
-  });
-  return arr;
-}
-
-function isStreamingDataset<T extends Record<string, unknown>>(
-  value: unknown,
-): value is StreamingCsvDataset<T> {
-  return (
-    Array.isArray(value) &&
-    (value as { __deeplineStreamingDataset?: unknown })
-      .__deeplineStreamingDataset === true
-  );
 }
 
 /**
@@ -2492,6 +2660,60 @@ function requireSheetContract(
   return contract;
 }
 
+function isDatasetPayloadField(field: string): boolean {
+  return (
+    field.length > 0 &&
+    !field.startsWith('__deepline') &&
+    field !== '_key' &&
+    field !== '_status' &&
+    field !== '_run_id' &&
+    field !== '_error' &&
+    field !== '_stage' &&
+    field !== '_provider' &&
+    field !== '_input_index' &&
+    field !== '_created_at' &&
+    field !== '_updated_at' &&
+    field !== '_cell_meta'
+  );
+}
+
+function augmentSheetContractWithDatasetFields(input: {
+  contract: PlaySheetContract;
+  rows: readonly Record<string, unknown>[];
+  outputFields?: readonly string[];
+}): PlaySheetContract {
+  const outputFields = new Set(input.outputFields ?? []);
+  const existingFields = new Set(
+    input.contract.columns.flatMap((column) =>
+      typeof column.field === 'string' ? [column.field] : [],
+    ),
+  );
+  const existingSqlNames = new Set(
+    input.contract.columns.map((column) => column.sqlName),
+  );
+  const columns = [...input.contract.columns];
+  for (const row of input.rows) {
+    for (const field of Object.keys(row)) {
+      if (!isDatasetPayloadField(field) || existingFields.has(field)) {
+        continue;
+      }
+      const sqlName = sqlSafePlayColumnName(field);
+      if (existingSqlNames.has(sqlName)) {
+        continue;
+      }
+      existingFields.add(field);
+      existingSqlNames.add(sqlName);
+      columns.push({
+        id: `runtime:${input.contract.tableNamespace}:${field}`,
+        sqlName,
+        source: outputFields.has(field) ? 'mapField' : 'input',
+        field,
+      });
+    }
+  }
+  return { ...input.contract, columns };
+}
+
 async function persistCompletedMapRows(input: {
   req: RunRequest;
   tableNamespace: string;
@@ -2500,19 +2722,24 @@ async function persistCompletedMapRows(input: {
   extraOutputFields?: string[];
 }): Promise<void> {
   if (input.rows.length === 0) return;
+  const outputFields = [
+    ...input.outputFields,
+    ...(input.extraOutputFields ?? []).filter(
+      (field) => !input.outputFields.includes(field),
+    ),
+  ];
   await harnessPersistCompletedSheetRows({
     baseUrl: input.req.baseUrl,
     executorToken: input.req.executorToken,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
-    sheetContract: requireSheetContract(input.req, input.tableNamespace),
+    sheetContract: augmentSheetContractWithDatasetFields({
+      contract: requireSheetContract(input.req, input.tableNamespace),
+      rows: input.rows,
+      outputFields,
+    }),
     rows: input.rows,
-    outputFields: [
-      ...input.outputFields,
-      ...(input.extraOutputFields ?? []).filter(
-        (field) => !input.outputFields.includes(field),
-      ),
-    ],
+    outputFields,
     runId: input.req.runId,
     userEmail: input.req.userEmail,
     preloadedDbSessions: input.req.preloadedDbSessions ?? null,
@@ -2537,12 +2764,37 @@ async function prepareMapRows(input: {
     executorToken: input.req.executorToken,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
-    sheetContract: requireSheetContract(input.req, input.tableNamespace),
+    sheetContract: augmentSheetContractWithDatasetFields({
+      contract: requireSheetContract(input.req, input.tableNamespace),
+      rows: input.rows,
+    }),
     rows: input.rows.map((row) => ({ ...row })),
     runId: input.req.runId,
     userEmail: input.req.userEmail,
     preloadedDbSessions: input.req.preloadedDbSessions ?? null,
   });
+  for (const timing of result.timings ?? []) {
+    const phase =
+      typeof timing.phase === 'string' && timing.phase.trim()
+        ? timing.phase.trim()
+        : 'unknown';
+    const ms =
+      typeof timing.ms === 'number' && Number.isFinite(timing.ms)
+        ? timing.ms
+        : 0;
+    const { phase: _phase, ms: _ms, ...extra } = timing;
+    void _phase;
+    void _ms;
+    recordRunnerPerfTrace({
+      req: input.req,
+      phase: `sheet_start.${phase}`,
+      ms,
+      extra: {
+        tableNamespace: input.tableNamespace,
+        ...extra,
+      },
+    });
+  }
   return {
     inserted: result.inserted,
     skipped: result.skipped,
@@ -2700,7 +2952,7 @@ function createMinimalWorkerCtx(
   const callDepth = rootGovernance?.callDepth ?? 0;
   const runMap = async <T extends Record<string, unknown>>(
     name: string,
-    rows: T[],
+    rows: WorkerDatasetInput<T>,
     fieldsDef: Record<
       string,
       | unknown
@@ -2715,7 +2967,8 @@ function createMinimalWorkerCtx(
   ): Promise<unknown> => {
     const mapStartedAt = nowMs();
     const mapNodeId = `map:${name}`;
-    const sliced = rows;
+    const inputRows = rows;
+    const rowCountHint = datasetRowCountHint(inputRows);
     const baseOffset = 0;
     const fieldEntries = Object.entries(fieldsDef);
     const plan = req.executionPlan;
@@ -2723,12 +2976,8 @@ function createMinimalWorkerCtx(
       (candidate) =>
         candidate.mapName === name || candidate.tableNamespace === name,
     );
-    const streaming = isStreamingDataset<T>(sliced);
-    // For streaming inputs we don't know the row count upfront — pass
-    // `totalRows: 0` so chooseMapChunkSize falls back to the preferred /
-    // default chunk size rather than trying to budget against an unknown.
     const rowsPerChunk = chooseMapChunkSize({
-      totalRows: streaming ? 0 : sliced.length,
+      totalRows: rowCountHint,
       mapCount: Math.max(1, plan?.maps.length ?? 1),
       stepsPerChunk: planMap?.stepsPerChunk ?? 1,
       preferredChunkSize: planMap?.defaultChunkSize,
@@ -2750,14 +2999,12 @@ function createMinimalWorkerCtx(
       typeof total === 'number' && Number.isFinite(total) && total > 0
         ? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
         : `${completed.toLocaleString()} rows processed`;
+    callbacks?.onMapStarted?.(mapNodeId, mapStartedAt);
     updateMapProgress({
       completed: 0,
-      total: streaming ? undefined : sliced.length,
+      total: rowCountHint ?? undefined,
       startedAt: mapStartedAt,
-      message: formatMapProgressMessage(
-        0,
-        streaming ? undefined : sliced.length,
-      ),
+      message: formatMapProgressMessage(0, rowCountHint ?? undefined),
     });
     const explicitRowKeysSeen =
       opts?.key === undefined ? null : new Map<string, number>();
@@ -2983,6 +3230,7 @@ function createMinimalWorkerCtx(
                     input?: unknown,
                     _opts?: { description?: string },
                   ): Promise<unknown> => {
+                    void _opts;
                     assertNotAborted(abortSignal);
                     const request = normalizeToolExecuteArgs(
                       requestOrKey,
@@ -3008,6 +3256,8 @@ function createMinimalWorkerCtx(
                     toolNameOrSpec,
                     waterfallInput,
                     waterfallOpts,
+                    callbacks,
+                    workflowStep,
                   ),
               };
               for (const [key, value] of fieldEntries) {
@@ -3219,10 +3469,16 @@ function createMinimalWorkerCtx(
         outputDatasetId: `map:${name}`,
         hash,
         preview: toWorkflowSerializableValue(out.slice(0, 5)),
+        cachedRows:
+          out.length <= WORKER_DATASET_IN_MEMORY_ROWS
+            ? toWorkflowSerializableValue(out)
+            : undefined,
       };
     };
 
-    const out: Array<T & Record<string, unknown>> = [];
+    const previewRows: Array<T & Record<string, unknown>> = [];
+    const cachedRows: Array<T & Record<string, unknown>> = [];
+    let canCacheRows = true;
     let totalRowsExecuted = 0;
     let totalRowsCached = 0;
     let totalRowsDuplicateReused = 0;
@@ -3260,6 +3516,7 @@ function createMinimalWorkerCtx(
         `(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
         `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
       const completedAt = nowMs();
+      callbacks?.onMapCompleted?.(mapNodeId, completedAt);
       updateMapProgress({
         completed: totalRowsWritten,
         total: totalRowsWritten,
@@ -3273,9 +3530,29 @@ function createMinimalWorkerCtx(
         message: cacheSummary,
         ts: nowMs(),
       });
-      return makeWorkerDataset(name, out, {
+      return createPersistedDatasetHandle({
+        playName: req.playName,
+        name,
         count: totalRowsWritten,
-        cacheSummary,
+        previewRows,
+        cachedRows: canCacheRows ? cachedRows : null,
+        readRows: async ({ limit, offset }) => {
+          const result = await harnessReadSheetDatasetRows({
+            baseUrl: req.baseUrl,
+            executorToken: req.executorToken,
+            playName: req.playName,
+            tableNamespace: name,
+            runId: req.runId,
+            limit,
+            offset,
+            userEmail: req.userEmail,
+            preloadedDbSessions: req.preloadedDbSessions ?? null,
+          });
+          return result.rows as Array<T & Record<string, unknown>>;
+        },
+        trace: (phase, ms, extra) =>
+          recordRunnerPerfTrace({ req, phase, ms, extra }),
+        nowMs,
         workProgress: {
           total: totalRowsWritten,
           executed: totalRowsExecuted,
@@ -3290,110 +3567,61 @@ function createMinimalWorkerCtx(
       });
     };
 
-    if (streaming) {
-      let totalRowsWritten = 0;
-      let chunkIndex = 0;
-      let chunkStart = 0;
-      const streamingDataset = sliced as unknown as StreamingCsvDataset<T>;
-      for await (const chunkRows of streamingDataset.iterChunks(rowsPerChunk)) {
-        assertNotAborted(abortSignal);
-        if (chunkRows.length === 0) continue;
-        assertUniqueExplicitRowKeys(chunkRows, chunkStart);
-        const chunkResult = await runChunkStep(
-          chunkRows,
-          chunkStart,
-          chunkIndex,
+    let totalRowsWritten = 0;
+    let chunkIndex = 0;
+    let chunkStart = 0;
+    for await (const chunkRows of iterDatasetChunks(inputRows, rowsPerChunk)) {
+      assertNotAborted(abortSignal);
+      if (chunkRows.length === 0) continue;
+      assertUniqueExplicitRowKeys(chunkRows, chunkStart);
+      const chunkResult = await runChunkStep(chunkRows, chunkStart, chunkIndex);
+      totalRowsWritten += chunkResult.rowsWritten;
+      totalRowsExecuted += chunkResult.rowsExecuted;
+      totalRowsCached += chunkResult.rowsCached;
+      totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
+      totalRowsInserted += chunkResult.rowsInserted;
+      totalRowsSkipped += chunkResult.rowsSkipped;
+      updateMapProgress({
+        completed: totalRowsWritten,
+        total: rowCountHint ?? undefined,
+        message: formatMapProgressMessage(
+          totalRowsWritten,
+          rowCountHint ?? undefined,
+        ),
+      });
+      if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
+        previewRows.push(
+          ...chunkResult.preview.slice(
+            0,
+            WORKER_DATASET_PREVIEW_ROWS - previewRows.length,
+          ),
         );
-        totalRowsWritten += chunkResult.rowsWritten;
-        totalRowsExecuted += chunkResult.rowsExecuted;
-        totalRowsCached += chunkResult.rowsCached;
-        totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
-        totalRowsInserted += chunkResult.rowsInserted;
-        totalRowsSkipped += chunkResult.rowsSkipped;
-        updateMapProgress({
-          completed: totalRowsWritten,
-          message: formatMapProgressMessage(totalRowsWritten),
-        });
-        if (out.length < 10) {
-          out.push(...chunkResult.preview.slice(0, 10 - out.length));
-        }
-        chunkStart += chunkRows.length;
-        chunkIndex += 1;
       }
-      const dataset = finalize(totalRowsWritten);
-      recordRunnerPerfTrace({
-        req,
-        phase: 'runner.map.total',
-        ms: nowMs() - mapStartedAt,
-        extra: {
-          mapName: name,
-          rowsWritten: totalRowsWritten,
-          streaming: true,
-        },
-      });
-      return dataset;
-    }
-
-    if (workflowStep && sliced.length > rowsPerChunk) {
-      let totalRowsWritten = 0;
-      for (let start = 0; start < sliced.length; start += rowsPerChunk) {
-        assertNotAborted(abortSignal);
-        const end = Math.min(sliced.length, start + rowsPerChunk);
-        const chunkRows = sliced.slice(start, end);
-        const chunkIndex = Math.floor(start / rowsPerChunk);
-        assertUniqueExplicitRowKeys(chunkRows, start);
-        const chunkResult = await runChunkStep(chunkRows, start, chunkIndex);
-        totalRowsWritten += chunkResult.rowsWritten;
-        totalRowsExecuted += chunkResult.rowsExecuted;
-        totalRowsCached += chunkResult.rowsCached;
-        totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
-        totalRowsInserted += chunkResult.rowsInserted;
-        totalRowsSkipped += chunkResult.rowsSkipped;
-        updateMapProgress({
-          completed: totalRowsWritten,
-          total: sliced.length,
-          message: formatMapProgressMessage(totalRowsWritten, sliced.length),
-        });
-        if (out.length < 10) {
-          out.push(...chunkResult.preview.slice(0, 10 - out.length));
+      if (canCacheRows) {
+        const nextRows = chunkResult.cachedRows ?? [];
+        if (
+          nextRows.length === chunkResult.rowsWritten &&
+          cachedRows.length + nextRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
+        ) {
+          cachedRows.push(...nextRows);
+        } else {
+          cachedRows.length = 0;
+          canCacheRows = false;
         }
       }
-      const dataset = finalize(totalRowsWritten);
-      recordRunnerPerfTrace({
-        req,
-        phase: 'runner.map.total',
-        ms: nowMs() - mapStartedAt,
-        extra: {
-          mapName: name,
-          rowsWritten: totalRowsWritten,
-          streaming: false,
-        },
-      });
-      return dataset;
+      chunkStart += chunkRows.length;
+      chunkIndex += 1;
     }
-
-    assertUniqueExplicitRowKeys(sliced, 0);
-    const chunkResult = await runChunkStep(sliced, 0, 0);
-    totalRowsExecuted = chunkResult.rowsExecuted;
-    totalRowsCached = chunkResult.rowsCached;
-    totalRowsDuplicateReused = chunkResult.rowsDuplicateReused;
-    totalRowsInserted = chunkResult.rowsInserted;
-    totalRowsSkipped = chunkResult.rowsSkipped;
-    out.push(...chunkResult.preview);
-    updateMapProgress({
-      completed: chunkResult.rowsWritten,
-      total: sliced.length,
-      message: formatMapProgressMessage(chunkResult.rowsWritten, sliced.length),
-    });
-    const dataset = finalize(chunkResult.rowsWritten);
+    const dataset = finalize(totalRowsWritten);
     recordRunnerPerfTrace({
       req,
       phase: 'runner.map.total',
       ms: nowMs() - mapStartedAt,
       extra: {
         mapName: name,
-        rowsWritten: chunkResult.rowsWritten,
-        streaming: false,
+        rowsWritten: totalRowsWritten,
+        inputKind: rowCountHint === null ? 'streaming' : 'known_count',
+        chunks: chunkIndex,
       },
     });
     return dataset;
@@ -3407,7 +3635,7 @@ function createMinimalWorkerCtx(
 
     constructor(
       private readonly name: string,
-      private readonly rows: T[],
+      private readonly rows: WorkerDatasetInput<T>,
     ) {}
 
     step(name: string, resolver: WorkerStepProgramStep['resolver']): this {
@@ -3482,18 +3710,13 @@ function createMinimalWorkerCtx(
     async csv<T extends Record<string, unknown> = Record<string, unknown>>(
       arg: unknown,
       options?: CsvRenameOptions,
-    ): Promise<T[]> {
+    ): Promise<WorkerDatasetHandle<T>> {
       const csvStartedAt = nowMs();
       if (Array.isArray(arg)) {
-        // Inline rows passed at call site — already in memory, keep the
-        // legacy array-backed dataset shape.
-        const dataset = makeWorkerDataset(
-          'csv',
-          applyCsvRenameProjection(arg as T[], options),
-          {
-            datasetKind: 'csv',
-          },
-        ) as unknown as T[];
+        const dataset = createInlineDatasetHandle(
+          applyCsvRenameProjection(arg as T[], options) as T[],
+          { name: 'csv', kind: 'csv' },
+        );
         recordRunnerPerfTrace({
           req,
           phase: 'runner.csv',
@@ -3504,15 +3727,10 @@ function createMinimalWorkerCtx(
       }
       const filename = String(arg ?? '');
       if (req.inlineCsv && filename === req.inlineCsv.name) {
-        // Inline CSV pre-staged by the dispatcher (small files <1 MiB). Already
-        // in memory; no streaming needed.
-        const dataset = makeWorkerDataset(
-          'csv',
-          applyCsvRenameProjection(req.inlineCsv.rows as T[], options),
-          {
-            datasetKind: 'csv',
-          },
-        ) as unknown as T[];
+        const dataset = createInlineDatasetHandle(
+          applyCsvRenameProjection(req.inlineCsv.rows as T[], options) as T[],
+          { name: filename, kind: 'csv' },
+        );
         recordRunnerPerfTrace({
           req,
           phase: 'runner.csv',
@@ -3521,52 +3739,73 @@ function createMinimalWorkerCtx(
         });
         return dataset;
       }
-      // Resolution order: explicit inputR2Keys (runtime input) → packaged
+      // Resolution order: explicit inputFiles (runtime input) → packaged
       // files (relative-path imports bundled with the play artifact).
-      let r2Key = req.inputR2Keys?.[filename];
-      if (!r2Key && req.packagedFiles) {
+      let file = req.inputFiles?.[filename] ?? null;
+      if (!file && req.packagedFiles) {
         const matchByPath = req.packagedFiles.find(
           (f) =>
             f.playPath === filename ||
             f.playPath === filename.replace(/^\.\//, ''),
         );
-        if (matchByPath) r2Key = matchByPath.storageKey;
+        if (matchByPath) {
+          file = {
+            logicalPath: matchByPath.playPath,
+            fileName:
+              matchByPath.playPath.split('/').pop() ?? matchByPath.playPath,
+            storageKey: matchByPath.storageKey,
+            contentType: matchByPath.contentType,
+            bytes: matchByPath.bytes,
+          };
+        }
       }
-      if (!r2Key) {
+      if (!file?.storageKey) {
         throw new Error(
           `ctx.csv("${filename}"): no inline rows or R2 asset binding registered. ` +
-            'Pass inline rows, or upload to R2 and register packagedFiles/inputR2Keys in the run config.',
+            'Pass inline rows, or upload to R2 and register packagedFiles/inputFiles in the run config.',
+        );
+      }
+      const selectedFile = file;
+      const expectedBytes = normalizeExpectedBytes(selectedFile.bytes);
+      if (expectedBytes === null) {
+        throw new Error(
+          `ctx.csv("${filename}"): staged dataset handle is missing a byte length for ` +
+            `${selectedFile.storageKey}. Re-stage the file with bytes metadata.`,
         );
       }
-      // Streaming path: returns a length-0 dataset shell whose iterChunks()
-      // pulls 1 MiB-ish text chunks from R2 and yields parsed row chunks.
-      // ctx.map detects the streaming surface via __deeplineStreamingDataset
-      // and switches its chunked execution loop to consume iterChunks
-      // directly, so 2M-row CSVs never get fully materialized in memory.
-      const storageKey = r2Key;
-      const dataset = makeStreamingCsvDataset<T>({
+      const dataset = createCsvDatasetHandle<T>({
         name: filename,
         logicalPath: filename,
+        expectedBytes,
         renameOptions: options,
+        nowMs,
+        streamRows: streamCsvRowsFromByteChunks,
+        trace: (phase, ms, extra) =>
+          recordRunnerPerfTrace({ req, phase, ms, extra }),
         open: () =>
-          openR2BodyStream({
+          openFileByteChunks({
             req,
             env,
             logicalPath: filename,
-            storageKey,
+            file: selectedFile,
           }),
-      }) as unknown as T[];
+      });
       recordRunnerPerfTrace({
         req,
         phase: 'runner.csv',
         ms: nowMs() - csvStartedAt,
-        extra: { mode: 'streaming_r2', filename },
+        extra: {
+          mode: 'streaming_file',
+          filename,
+          expectedBytes,
+          storageKey: selectedFile.storageKey,
+        },
       });
       return dataset;
     },
     map<T extends Record<string, unknown>>(
       name: string,
-      rows: T[],
+      rows: WorkerDatasetInput<T>,
       fieldsDef?:
         | Record<
             string,
@@ -3600,7 +3839,12 @@ function createMinimalWorkerCtx(
       input: Record<string, unknown>,
     ): Promise<unknown> => {
       assertNotAborted(abortSignal);
-      return executeTool(req, { id: key, toolId, input }, workflowStep);
+      return executeToolWithLifecycle(
+        req,
+        { id: key, toolId, input },
+        workflowStep,
+        callbacks,
+      );
     },
     tools: {
       async execute(
@@ -3609,11 +3853,13 @@ function createMinimalWorkerCtx(
         input?: unknown,
         _opts?: { description?: string },
       ): Promise<unknown> {
+        void _opts;
         assertNotAborted(abortSignal);
-        return executeTool(
+        return executeToolWithLifecycle(
           req,
           normalizeToolExecuteArgs(requestOrKey, toolId, input),
           workflowStep,
+          callbacks,
         );
       },
     },
@@ -3640,7 +3886,15 @@ function createMinimalWorkerCtx(
       input: Record<string, unknown>,
       opts?: WorkerWaterfallOptions,
     ): Promise<unknown | null> {
-      return executeWorkerWaterfall(req, [], toolNameOrSpec, input, opts);
+      return executeWorkerWaterfall(
+        req,
+        [],
+        toolNameOrSpec,
+        input,
+        opts,
+        callbacks,
+        workflowStep,
+      );
     },
     async sleep(ms: number): Promise<void> {
       assertNotAborted(abortSignal);
@@ -3993,17 +4247,10 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
   });
 }
 
-/** Cap on `liveLogs` retained both in-memory and persisted to Convex. */
-const LIVE_LOG_BUFFER_LIMIT = 500;
-/** Min wall-clock interval between live-log flushes during a run. */
-const LIVE_LOG_FLUSH_INTERVAL_MS = 500;
-/**
- * Initial flush delay for live logs. Short plays should not pay an extra
- * non-terminal Convex write just to show a transient "running" log state; the
- * terminal status carries the full log buffer. Longer plays still flush early
- * enough for the dashboard to feel alive.
- */
-const LIVE_LOG_FIRST_FLUSH_DELAY_MS = 30_000;
+/** Cap on run log lines retained in the terminal output compatibility shape. */
+const RUN_LOG_BUFFER_LIMIT = 500;
+/** Min wall-clock interval between live run-ledger flushes during a run. */
+const RUN_LEDGER_FLUSH_INTERVAL_MS = 500;
 
 async function executeRunRequest(
   req: RunRequest,
@@ -4033,99 +4280,240 @@ async function executeRunRequest(
   });
   const abortController = options?.abortController ?? new AbortController();
   const abortSignal = abortController.signal;
-  const postgresPrewarmStartedAt = nowMs();
-  await harnessPrewarmPostgresSessions({
-    executorToken: req.executorToken,
-    sessions: req.preloadedDbSessions ?? [],
-  });
-  recordRunnerPerfTrace({
-    req,
-    phase: 'runner.prewarm_postgres',
-    ms: nowMs() - postgresPrewarmStartedAt,
-    extra: {
-      sessions: req.preloadedDbSessions?.length ?? 0,
+  let runLogBuffer: string[] = [];
+  let pendingRunLogLines: string[] = [];
+  let stepProgressByNodeId: LiveNodeProgressMap = {};
+  let dirtyProgressNodeIds = new Set<string>();
+  let pendingLedgerEvents: PlayRunLedgerEvent[] = [
+    {
+      type: 'run.started',
+      runId: req.runId,
+      playName: req.playName,
+      source: 'worker',
+      occurredAt: startedAt,
+      runtimeBackend: 'cf_workflows_dynamic_worker',
     },
-  });
-  // Maintain a rolling buffer of log lines emitted during the run. This is
-  // what the play-page UI consumes via Convex polling + diffPlayRunStreamEvents
-  // → play.run.log SSE events. Without periodic flushing, the play page only
-  // sees the final terminal status with no intermediate logs/progress.
-  let liveLogs: string[] = [];
-  let liveLogsDirty = false;
-  let liveNodeProgress: LiveNodeProgressMap = {};
-  let lastLiveLogFlushAt =
-    nowMs() - LIVE_LOG_FLUSH_INTERVAL_MS + LIVE_LOG_FIRST_FLUSH_DELAY_MS;
-  let liveLogFlushInFlight: Promise<void> = Promise.resolve();
-  const appendLiveLog = (line: string) => {
+  ];
+  let lastLedgerFlushAt = 0;
+  let ledgerFlushInFlight: Promise<void> = Promise.resolve();
+
+  const appendRunLogLine = (line: string) => {
     const trimmed = redactSecretsFromLogString(line.trim());
     if (!trimmed) return;
-    liveLogs = [...liveLogs, trimmed].slice(-LIVE_LOG_BUFFER_LIMIT);
-    liveLogsDirty = true;
+    runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
+    pendingRunLogLines = [...pendingRunLogLines, trimmed].slice(
+      -RUN_LOG_BUFFER_LIMIT,
+    );
   };
-  const updateLiveNodeProgress = (input: {
+
+  const updateStepProgress = (input: {
     nodeId: string;
     progress: LiveNodeProgressSnapshot;
   }) => {
     const nodeId = input.nodeId.trim();
     if (!nodeId) return;
-    liveNodeProgress = {
-      ...liveNodeProgress,
+    stepProgressByNodeId = {
+      ...stepProgressByNodeId,
       [nodeId]: {
-        ...(liveNodeProgress[nodeId] ?? {}),
+        ...(stepProgressByNodeId[nodeId] ?? {}),
         ...input.progress,
       },
     };
+    dirtyProgressNodeIds.add(nodeId);
+  };
+
+  const stepProgressSnapshot = () => ({ ...stepProgressByNodeId });
+
+  const appendStepLifecycleEvent = (event: PlayStepLifecycleEvent) => {
+    updateStepProgress({
+      nodeId: event.nodeId,
+      progress: {
+        ...(event.transition === 'started'
+          ? { startedAt: event.at }
+          : { completedAt: event.at }),
+        updatedAt: event.at,
+      },
+    });
+    pendingLedgerEvents = [
+      ...pendingLedgerEvents,
+      {
+        type:
+          event.transition === 'started'
+            ? 'step.started'
+            : event.transition === 'failed'
+              ? 'step.failed'
+              : 'step.completed',
+        runId: req.runId,
+        source: 'worker',
+        occurredAt: event.at,
+        stepId: event.nodeId,
+        kind: event.type,
+      },
+    ];
+    flushLedgerEvents(false);
+  };
+
+  const drainPendingLedgerEvents = (
+    occurredAt: number,
+  ): PlayRunLedgerEvent[] => {
+    const events = pendingLedgerEvents;
+    pendingLedgerEvents = [];
+
+    if (pendingRunLogLines.length > 0) {
+      events.push({
+        type: 'log.appended',
+        runId: req.runId,
+        source: 'worker',
+        occurredAt,
+        lines: pendingRunLogLines,
+      });
+      pendingRunLogLines = [];
+    }
+
+    if (dirtyProgressNodeIds.size > 0) {
+      for (const nodeId of dirtyProgressNodeIds) {
+        const progress = stepProgressByNodeId[nodeId];
+        if (!progress) continue;
+        const normalizedProgress: PlayRunLedgerStepProgress = {
+          ...(typeof progress.completed === 'number'
+            ? { completed: progress.completed }
+            : {}),
+          ...(typeof progress.total === 'number'
+            ? { total: progress.total }
+            : {}),
+          ...(typeof progress.failed === 'number'
+            ? { failed: progress.failed }
+            : {}),
+          ...(typeof progress.message === 'string' && progress.message
+            ? { message: progress.message }
+            : {}),
+          ...(typeof progress.artifactTableNamespace === 'string' ||
+          progress.artifactTableNamespace === null
+            ? { artifactTableNamespace: progress.artifactTableNamespace }
+            : {}),
+          updatedAt:
+            typeof progress.updatedAt === 'number'
+              ? progress.updatedAt
+              : occurredAt,
+        };
+        const status: PlayRunLedgerStepStatus =
+          typeof progress.completedAt === 'number' ? 'completed' : 'running';
+        events.push({
+          type: 'step.progress',
+          runId: req.runId,
+          source: 'worker',
+          occurredAt:
+            typeof progress.updatedAt === 'number'
+              ? progress.updatedAt
+              : occurredAt,
+          stepId: nodeId,
+          status,
+          progress: normalizedProgress,
+        });
+      }
+      dirtyProgressNodeIds = new Set<string>();
+    }
+
+    return events;
   };
-  const liveNodeProgressSnapshot = () => ({ ...liveNodeProgress });
-  const flushLiveLogs = (force: boolean): void => {
+
+  const flushLedgerEvents = (force: boolean): void => {
     if (!options?.persistResultDatasets) return;
-    if (!liveLogsDirty && !force) return;
     const now = nowMs();
-    if (!force && now - lastLiveLogFlushAt < LIVE_LOG_FLUSH_INTERVAL_MS) return;
-    lastLiveLogFlushAt = now;
-    liveLogsDirty = false;
-    const snapshot = [...liveLogs];
-    liveLogFlushInFlight = liveLogFlushInFlight
+    if (!force && now - lastLedgerFlushAt < RUN_LEDGER_FLUSH_INTERVAL_MS) {
+      return;
+    }
+    const events = drainPendingLedgerEvents(now);
+    if (events.length === 0) return;
+    lastLedgerFlushAt = now;
+    ledgerFlushInFlight = ledgerFlushInFlight
       .catch(() => undefined)
       .then(async () => {
         try {
           await postRuntimeApi(req.baseUrl, req.executorToken, {
-            action: 'update_run_status',
+            action: 'append_run_events',
             playId: req.runId,
-            status: 'running',
-            runtimeBackend: 'cf_workflows_dynamic_worker',
-            liveLogs: snapshot,
-            liveNodeProgress: liveNodeProgressSnapshot(),
-            lastCheckpointAt: now,
+            events,
           });
         } catch {
-          // Best-effort; the terminal update still carries the final logs.
+          pendingLedgerEvents = [...events, ...pendingLedgerEvents];
+          throw new Error('runtime run-ledger append failed');
         }
+      })
+      .catch(() => undefined);
+  };
+
+  const flushTerminalLedgerEvents = async (
+    terminalEvent: PlayRunLedgerEvent,
+  ): Promise<void> => {
+    if (!options?.persistResultDatasets) return;
+    await ledgerFlushInFlight.catch(() => undefined);
+    const now = nowMs();
+    pendingRunLogLines = runLogBuffer;
+    dirtyProgressNodeIds = new Set([
+      ...dirtyProgressNodeIds,
+      ...Object.keys(stepProgressByNodeId),
+    ]);
+    pendingLedgerEvents = [...pendingLedgerEvents, terminalEvent];
+    const events = drainPendingLedgerEvents(now);
+    if (events.length === 0) return;
+    try {
+      await postRuntimeApi(req.baseUrl, req.executorToken, {
+        action: 'append_run_events',
+        playId: req.runId,
+        events,
       });
+    } catch (error) {
+      pendingLedgerEvents = [...events, ...pendingLedgerEvents];
+      throw error;
+    }
+  };
+
+  const orderedNodes = buildOrderedNodeList(req.contractSnapshot);
+  const stepLifecycle =
+    orderedNodes.length > 0
+      ? new PlayStepLifecycleTracker(
+          orderedNodes,
+          () => stepProgressByNodeId,
+          appendStepLifecycleEvent,
+          nowMs,
+        )
+      : null;
+  const workerCallbacks: WorkerCtxCallbacks = {
+    onNodeProgress: (input) => {
+      updateStepProgress(input);
+      flushLedgerEvents(false);
+    },
+    onMapStarted: (nodeId, at) => stepLifecycle?.onMapStarted(nodeId, at),
+    onMapCompleted: (nodeId, at) => stepLifecycle?.onMapCompleted(nodeId, at),
+    onToolCalled: (toolId, at) => stepLifecycle?.onToolCalled(toolId, at),
+    onToolFailed: (toolId, at) => stepLifecycle?.onToolFailed(toolId, at),
   };
 
   const wrappedEmit = (event: RunnerEvent) => {
     if (event.type === 'log') {
-      appendLiveLog(event.message);
-      flushLiveLogs(false);
+      appendRunLogLine(event.message);
+      flushLedgerEvents(false);
     } else if (event.type === 'error') {
       // Sanitize the inbound message before it enters the live-log buffer.
       // The downstream `emit` still receives the raw event so the console /
       // NDJSON stream can keep its full debugging fidelity.
       const sanitizedMessage = redactSecretsFromLogString(event.message);
-      appendLiveLog(`[error] ${sanitizedMessage}`);
-      flushLiveLogs(true);
+      appendRunLogLine(`[error] ${sanitizedMessage}`);
+      flushLedgerEvents(true);
     }
     emit(event);
   };
 
+  stepLifecycle?.markPreMapStepsStarted(startedAt);
+  flushLedgerEvents(false);
   const ctx = createMinimalWorkerCtx(
     req,
     wrappedEmit,
     env,
     workflowStep,
     abortSignal,
-    { onNodeProgress: updateLiveNodeProgress },
+    workerCallbacks,
   );
   try {
     const playStartedAt = nowMs();
@@ -4140,6 +4528,7 @@ async function executeRunRequest(
       phase: 'runner.play_function',
       ms: nowMs() - playStartedAt,
     });
+    stepLifecycle?.markAllTerminal(nowMs());
     const serializeStartedAt = nowMs();
     const serializedResult = serializePlayReturnValue(result);
     recordRunnerPerfTrace({
@@ -4148,53 +4537,74 @@ async function executeRunRequest(
       ms: nowMs() - serializeStartedAt,
     });
     if (options?.persistResultDatasets) {
-      const persistStartedAt = nowMs();
-      await liveLogFlushInFlight.catch(() => undefined);
+      const ledgerFlushWaitStartedAt = nowMs();
+      await ledgerFlushInFlight.catch(() => undefined);
       recordRunnerPerfTrace({
         req,
-        phase: 'runner.live_log_flush_wait',
-        ms: nowMs() - persistStartedAt,
+        phase: 'runner.run_ledger_flush_wait',
+        ms: nowMs() - ledgerFlushWaitStartedAt,
       });
       const resultDatasetStartedAt = nowMs();
-      await persistResultDatasets(req, serializedResult);
+      await persistResultDatasets(req, result, serializedResult);
       recordRunnerPerfTrace({
         req,
         phase: 'runner.persist_result_datasets',
         ms: nowMs() - resultDatasetStartedAt,
       });
       const terminalResult = trimResultForStatus(serializedResult);
-      const terminalUpdateStartedAt = nowMs();
-      await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
-        action: 'update_run_status',
-        playId: req.runId,
-        status: 'completed',
-        error: null,
-        result: terminalResult,
-        runtimeBackend: 'cf_workflows_dynamic_worker',
-        waitKind: null,
-        waitUntil: null,
-        activeBoundaryId: null,
-        liveLogs,
-        liveNodeProgress: liveNodeProgressSnapshot(),
-        lastCheckpointAt: nowMs(),
-      });
-      recordRunnerPerfTrace({
-        req,
-        phase: 'runner.terminal_status_update',
-        ms: nowMs() - terminalUpdateStartedAt,
+      const terminalOccurredAt = nowMs();
+      const terminalLedgerPromise = (async () => {
+        const terminalUpdateStartedAt = nowMs();
+        await flushTerminalLedgerEvents({
+          type: 'run.completed',
+          runId: req.runId,
+          source: 'worker',
+          occurredAt: terminalOccurredAt,
+          result: terminalResult,
+        });
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.terminal_ledger_append',
+          ms: nowMs() - terminalUpdateStartedAt,
+        });
+      })().catch((error) => {
+        console.error(
+          `[play-harness] non-fatal terminal ledger append failed runId=${req.runId}: ${
+            error instanceof Error ? error.message : String(error)
+          }`,
+        );
       });
 
+      await terminalLedgerPromise;
+
       const billingStartedAt = nowMs();
-      await finalizeWorkerComputeBilling({
+      const billingPromise = finalizeWorkerComputeBilling({
         req,
         success: true,
         actionEstimate: 4,
+      }).then(() => {
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.compute_billing_finalize',
+          ms: nowMs() - billingStartedAt,
+        });
       });
-      recordRunnerPerfTrace({
-        req,
-        phase: 'runner.compute_billing_finalize',
-        ms: nowMs() - billingStartedAt,
-      });
+      if (extractMaxCreditsPerRun(req.contractSnapshot) !== null) {
+        await billingPromise;
+      } else {
+        const nonBlockingBillingPromise = billingPromise.catch((error) => {
+          console.error(
+            `[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
+              error instanceof Error ? error.message : String(error)
+            }`,
+          );
+        });
+        if (options?.waitUntil) {
+          options.waitUntil(nonBlockingBillingPromise);
+        } else {
+          await nonBlockingBillingPromise;
+        }
+      }
     }
     const parentSignalStartedAt = nowMs();
     await signalParentPlayTerminal({
@@ -4222,11 +4632,12 @@ async function executeRunRequest(
       playName: req.playName,
       result: serializedResult,
       outputRows: inferOutputRows(serializedResult),
-      liveLogs,
-      liveNodeProgress: liveNodeProgressSnapshot(),
+      liveLogs: runLogBuffer,
+      liveNodeProgress: stepProgressSnapshot(),
       durationMs: nowMs() - startedAt,
     };
   } catch (error) {
+    stepLifecycle?.markStartedFailed(nowMs());
     const aborted = isAbortLikeError(error);
     if (aborted) {
       // Flip the controller so any concurrent user code observes the abort
@@ -4237,19 +4648,15 @@ async function executeRunRequest(
     }
     const message = error instanceof Error ? error.message : String(error);
     if (options?.persistResultDatasets) {
-      await liveLogFlushInFlight.catch(() => undefined);
-      await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
-        action: 'update_run_status',
-        playId: req.runId,
-        status: aborted ? 'cancelled' : 'failed',
+      appendRunLogLine(
+        `${aborted ? '[cancelled]' : '[error]'} ${redactSecretsFromLogString(message)}`,
+      );
+      await flushTerminalLedgerEvents({
+        type: aborted ? 'run.cancelled' : 'run.failed',
+        runId: req.runId,
+        source: 'worker',
+        occurredAt: nowMs(),
         error: message,
-        runtimeBackend: 'cf_workflows_dynamic_worker',
-        waitKind: null,
-        waitUntil: null,
-        activeBoundaryId: null,
-        liveLogs,
-        liveNodeProgress: liveNodeProgressSnapshot(),
-        lastCheckpointAt: nowMs(),
       });
       await finalizeWorkerComputeBilling({
         req,
@@ -4338,6 +4745,12 @@ function runRequestFromWorkflowParams(
 ): RunRequest {
   const inputFile = isRecord(params.inputFile) ? params.inputFile : null;
   const fileName = String(inputFile?.name ?? inputFile?.path ?? 'input.csv');
+  const inputStorageKey =
+    typeof inputFile?.r2Key === 'string'
+      ? inputFile.r2Key
+      : typeof inputFile?.storageKey === 'string'
+        ? inputFile.storageKey
+        : null;
   return {
     runId: String(params.runId ?? ''),
     callbackUrl: String(params.baseUrl ?? ''),
@@ -4350,14 +4763,30 @@ function runRequestFromWorkflowParams(
       ? (params.input as Record<string, unknown>)
       : {},
     inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
-    inputR2Keys:
-      inputFile && typeof inputFile.r2Key === 'string'
-        ? { [fileName]: inputFile.r2Key }
+    inputFiles:
+      inputFile && inputStorageKey
+        ? {
+            [fileName]: {
+              logicalPath: String(
+                inputFile.logicalPath ?? inputFile.path ?? fileName,
+              ),
+              fileName,
+              storageKey: inputStorageKey,
+              contentType:
+                typeof inputFile.contentType === 'string'
+                  ? inputFile.contentType
+                  : null,
+              bytes: normalizeExpectedBytes(inputFile.bytes),
+            },
+          }
         : null,
     packagedFiles: Array.isArray(params.packagedFiles)
       ? params.packagedFiles.filter(isRecord).map((file) => ({
           playPath: String(file.playPath ?? ''),
           storageKey: String(file.storageKey ?? ''),
+          contentType:
+            typeof file.contentType === 'string' ? file.contentType : null,
+          bytes: normalizeExpectedBytes(file.bytes),
         }))
       : null,
     partitionRange: null,
@@ -4425,11 +4854,39 @@ function isPlayCallGovernanceSnapshot(
 async function persistResultDatasets(
   req: RunRequest,
   result: unknown,
+  serializedResult: unknown,
 ): Promise<void> {
-  const datasets = collectDatasetEnvelopes(result);
+  const persistedNamespaces = new Set<string>();
+  for (const dataset of collectDatasetHandles(result)) {
+    if (dataset.datasetKind === 'map') continue;
+    let inputOffset = 0;
+    for await (const chunk of iterDatasetChunks(
+      dataset.handle,
+      RESULT_DATASET_PERSIST_CHUNK_ROWS,
+    )) {
+      if (chunk.length === 0) continue;
+      await harnessStartSheetDataset({
+        baseUrl: req.baseUrl,
+        executorToken: req.executorToken,
+        playName: req.playName,
+        tableNamespace: dataset.tableNamespace,
+        sheetContract: requireSheetContract(req, dataset.tableNamespace),
+        rows: chunk.map((row) => ({ ...row })),
+        runId: req.runId,
+        inputOffset,
+        userEmail: req.userEmail,
+        preloadedDbSessions: req.preloadedDbSessions ?? null,
+      });
+      inputOffset += chunk.length;
+    }
+    persistedNamespaces.add(dataset.tableNamespace);
+  }
+
+  const datasets = collectDatasetEnvelopes(serializedResult);
   for (const dataset of datasets) {
     if (dataset.datasetKind === 'map') continue;
     if (dataset.rows.length === 0) continue;
+    if (persistedNamespaces.has(dataset.tableNamespace)) continue;
     await harnessStartSheetDataset({
       baseUrl: req.baseUrl,
       executorToken: req.executorToken,
@@ -4438,12 +4895,63 @@ async function persistResultDatasets(
       sheetContract: requireSheetContract(req, dataset.tableNamespace),
       rows: dataset.rows,
       runId: req.runId,
+      inputOffset: 0,
       userEmail: req.userEmail,
       preloadedDbSessions: req.preloadedDbSessions ?? null,
     });
   }
 }
 
+const RESULT_DATASET_PERSIST_CHUNK_ROWS = 5_000;
+
+function collectDatasetHandles(value: unknown): Array<{
+  tableNamespace: string;
+  datasetKind: 'csv' | 'map' | null;
+  handle: WorkerDatasetHandle<Record<string, unknown>>;
+}> {
+  const datasets: Array<{
+    tableNamespace: string;
+    datasetKind: 'csv' | 'map' | null;
+    handle: WorkerDatasetHandle<Record<string, unknown>>;
+  }> = [];
+  const seen = new WeakSet<object>();
+  const walk = (candidate: unknown, depth: number) => {
+    if (depth > 12 || candidate == null) return;
+    if (isDatasetHandle(candidate)) {
+      const metadata = candidate.toJSON() as Record<string, unknown>;
+      const tableNamespace =
+        typeof metadata.tableNamespace === 'string'
+          ? metadata.tableNamespace
+          : null;
+      const datasetKind =
+        metadata.datasetKind === 'csv' || metadata.datasetKind === 'map'
+          ? metadata.datasetKind
+          : null;
+      if (tableNamespace) {
+        datasets.push({
+          tableNamespace,
+          datasetKind,
+          handle: candidate as WorkerDatasetHandle<Record<string, unknown>>,
+        });
+      }
+      return;
+    }
+    if (Array.isArray(candidate)) {
+      for (const item of candidate) walk(item, depth + 1);
+      return;
+    }
+    if (typeof candidate !== 'object') return;
+    const object = candidate as Record<string, unknown>;
+    if (seen.has(object)) return;
+    seen.add(object);
+    for (const child of Object.values(object)) {
+      walk(child, depth + 1);
+    }
+  };
+  walk(value, 0);
+  return datasets;
+}
+
 function serializePlayReturnValue(value: unknown): unknown {
   return serializeValue(value, 0);
 }
@@ -4498,64 +5006,10 @@ function trimResultShape(value: unknown): unknown {
 
 function serializeValue(value: unknown, depth: number): unknown {
   if (depth > 20 || value == null) return value;
+  if (isDatasetHandle(value)) {
+    return serializeValue(value.toJSON(), depth + 1);
+  }
   if (Array.isArray(value)) {
-    const tableNamespace =
-      typeof (value as unknown as { tableNamespace?: unknown })
-        .tableNamespace === 'string'
-        ? (value as unknown as { tableNamespace: string }).tableNamespace
-        : null;
-    const datasetId =
-      typeof (value as unknown as { datasetId?: unknown }).datasetId ===
-      'string'
-        ? (value as unknown as { datasetId: string }).datasetId
-        : null;
-    const datasetCount =
-      typeof (value as unknown as { __deeplineDatasetCount?: unknown })
-        .__deeplineDatasetCount === 'number'
-        ? (value as unknown as { __deeplineDatasetCount: number })
-            .__deeplineDatasetCount
-        : value.length;
-    const datasetKind =
-      (value as unknown as { __deeplineDatasetKind?: unknown })
-        .__deeplineDatasetKind === 'csv'
-        ? 'csv'
-        : 'map';
-    const cacheSummary =
-      typeof (value as unknown as { __deeplineCacheSummary?: unknown })
-        .__deeplineCacheSummary === 'string'
-        ? (value as unknown as { __deeplineCacheSummary: string })
-            .__deeplineCacheSummary
-        : null;
-    const workProgress = isRecord(
-      (value as unknown as { __deeplineWorkProgress?: unknown })
-        .__deeplineWorkProgress,
-    )
-      ? (
-          value as unknown as {
-            __deeplineWorkProgress: Record<string, unknown>;
-          }
-        ).__deeplineWorkProgress
-      : null;
-    const previewRows = value
-      .slice(0, 5)
-      .map((row) => serializeValue(row, depth + 1))
-      .filter(isRecord);
-    if (tableNamespace && datasetId) {
-      const columns = inferColumns(
-        value.map((row) => serializeValue(row, depth + 1)).filter(isRecord),
-      );
-      return {
-        kind: 'dataset' as const,
-        datasetKind,
-        datasetId,
-        count: datasetCount,
-        columns,
-        preview: previewRows,
-        tableNamespace,
-        ...(cacheSummary ? { cacheSummary } : {}),
-        ...(workProgress ? { _metadata: { workProgress } } : {}),
-      };
-    }
     return value.map((entry) => serializeValue(entry, depth + 1));
   }
   if (typeof value !== 'object') return value;
@@ -4566,16 +5020,6 @@ function serializeValue(value: unknown, depth: number): unknown {
   return out;
 }
 
-function inferColumns(rows: ReadonlyArray<Record<string, unknown>>): string[] {
-  const columns = new Set<string>();
-  for (const row of rows) {
-    for (const key of Object.keys(row)) {
-      columns.add(key);
-    }
-  }
-  return [...columns];
-}
-
 function collectDatasetEnvelopes(value: unknown): Array<{
   tableNamespace: string;
   datasetKind: 'csv' | 'map' | null;
@@ -4714,10 +5158,17 @@ export class TenantWorkflow extends WorkflowEntrypoint<
       // user via tail/SSE. Retry with backoff before giving up; if we drop
       // it, the user is stuck staring at the opaque CF reference id.
       const errorPayload = JSON.stringify({
-        action: 'update_run_status',
+        action: 'append_run_events',
         playId: req.runId,
-        status: 'failed',
-        error: `TenantWorkflow.run threw: ${detail.name ?? 'Error'}: ${detail.message}\n${detail.stack ?? ''}`,
+        events: [
+          {
+            type: 'run.failed',
+            runId: req.runId,
+            source: 'worker',
+            occurredAt: nowMs(),
+            error: `TenantWorkflow.run threw: ${detail.name ?? 'Error'}: ${detail.message}\n${detail.stack ?? ''}`,
+          } satisfies PlayRunLedgerEvent,
+        ],
       });
       const backoffMs = [200, 500, 1500];
       let lastCallbackError: unknown = null;
@@ -4850,6 +5301,10 @@ function inferOutputRows(result: unknown): number {
   const datasets: number[] = [];
   const walk = (value: unknown, depth: number) => {
     if (depth > 6 || value == null) return;
+    if (isDatasetHandle(value)) {
+      datasets.push(value.toJSON().count);
+      return;
+    }
     if (Array.isArray(value)) {
       for (const item of value) walk(item, depth + 1);
       return;
@@ -4858,14 +5313,9 @@ function inferOutputRows(result: unknown): number {
     const record = value as Record<string, unknown>;
     if (
       typeof record.tableNamespace === 'string' &&
-      (typeof record.count === 'number' ||
-        typeof record.__deeplineDatasetCount === 'number')
+      typeof record.count === 'number'
     ) {
-      datasets.push(
-        typeof record.count === 'number'
-          ? record.count
-          : Number(record.__deeplineDatasetCount),
-      );
+      datasets.push(record.count);
     }
     for (const [key, child] of Object.entries(record)) {
       if (key === 'preview') continue;