deepline 0.1.20 → 0.1.22

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -56,6 +56,7 @@ import {
   type ToolResultMetadataInput,
 } from '../../../shared_libs/play-runtime/tool-result';
 import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
+import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
 import type { PlayRuntimeManifestMap } from '../../../shared_libs/plays/compiler-manifest';
 import {
   derivePlayRowIdentity,
@@ -80,6 +81,8 @@ import {
 // re-bundle harness internals into per-play. Keep that in mind.
 import {
   harnessFetchStagedFile,
+  harnessPersistCompletedSheetRows,
+  harnessPrewarmPostgresSessions,
   harnessStartSheetDataset,
   setHarnessBinding,
 } from '../../../sdk/src/plays/harness-stub';
@@ -129,6 +132,7 @@ type RunRequest = {
   childPlayManifests?: PlayRuntimeManifestMap | null;
   /** Internal ctx.runPlay lineage. Public SDK/users never see this. */
   playCallGovernance?: PlayCallGovernanceSnapshot | null;
+  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
   /** Cloudflare coordinator URL for direct Workflow control-plane signals. */
   coordinatorUrl?: string | null;
   /** Request-scoped coordinator auth token for preview/dev direct control calls. */
@@ -191,6 +195,10 @@ type WorkerEnv = {
       runId: string,
       payload: Record<string, unknown>,
     ): Promise<void>;
+    recordRunEvent(
+      runId: string,
+      event: Record<string, unknown>,
+    ): Promise<void>;
   };
   /**
    * Service binding to the long-lived Play Harness Worker
@@ -516,10 +524,58 @@ type WorkflowRunOutput = {
   durationMs: number;
 };
 
+type LiveNodeProgressSnapshot = {
+  completed?: number;
+  total?: number;
+  failed?: number;
+  message?: string;
+  updatedAt?: number;
+  startedAt?: number;
+  completedAt?: number;
+  artifactTableNamespace?: string | null;
+};
+
+type LiveNodeProgressMap = Record<string, LiveNodeProgressSnapshot>;
+
+type WorkerCtxCallbacks = {
+  onNodeProgress?: (input: {
+    nodeId: string;
+    progress: LiveNodeProgressSnapshot;
+  }) => void;
+};
+
 function nowMs(): number {
   return Date.now();
 }
 
+function recordRunnerPerfTrace(input: {
+  req: RunRequest;
+  phase: string;
+  ms?: number;
+  extra?: Record<string, unknown>;
+}): void {
+  if (!input.req.runId || !input.phase) return;
+  const payload = {
+    ts: Date.now(),
+    source: 'dynamic_worker' as const,
+    runId: input.req.runId,
+    phase: `runner.${input.phase}`,
+    ...(input.ms !== undefined ? { ms: input.ms } : {}),
+    ...(input.extra ?? {}),
+  };
+  console.log(
+    `[deepline-run:${input.req.runId}] [perf-trace] ${JSON.stringify(payload)}`,
+  );
+  cachedCoordinatorBinding
+    ?.recordPerfTrace(input.req.runId, payload)
+    .catch((error: unknown) => {
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(
+        `[deepline-run:${input.req.runId}] failed to forward runner perf trace: ${message}`,
+      );
+    });
+}
+
 function makeRequestId(): string {
   // Workers crypto.randomUUID is available without nodejs_compat.
   return crypto.randomUUID();
@@ -660,7 +716,7 @@ async function submitChildPlayThroughCoordinator(input: {
   const coordinatorUrl = input.req.coordinatorUrl?.trim();
   if (coordinatorUrl) {
     // Keep child plays on the same coordinator/Workflow submit path as
-    // top-level runs; the RPC binding remains only as a no-URL fallback.
+    // top-level runs when the coordinator URL is present.
     const res = await fetch(
       `${coordinatorUrl.replace(/\/$/, '')}/workflow/${encodeURIComponent(
         input.req.runId,
@@ -1056,6 +1112,29 @@ async function callToolDirect(
   let lastError: Error | null = null;
 
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    if (toolId === 'test_transient_500' || toolId === 'test_transient_429') {
+      const syntheticResult = executeSyntheticTransientRetry(
+        toolId,
+        input,
+        attempt,
+      );
+      if (syntheticResult.ok) {
+        return wrapWorkerToolResult(
+          toolId,
+          syntheticResult.result,
+          syntheticToolMetadata(toolId),
+        );
+      }
+      lastError = new Error(
+        `tool ${toolId} ${syntheticResult.status} attempt ${attempt}/${maxAttempts}: ${syntheticResult.message}`,
+      );
+      if (attempt >= maxAttempts) {
+        throw lastError;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1_000));
+      continue;
+    }
+
     const res = await fetchRuntimeApi(req.baseUrl, path, {
       method: 'POST',
       headers: {
@@ -1241,6 +1320,41 @@ async function executeSyntheticTestRateLimitBatch(
   };
 }
 
+type SyntheticTransientRetryResult =
+  | { ok: true; result: Record<string, unknown> }
+  | { ok: false; status: number; message: string };
+
+function executeSyntheticTransientRetry(
+  toolId: string,
+  input: Record<string, unknown>,
+  attempt: number,
+): SyntheticTransientRetryResult {
+  const failuresBeforeSuccess =
+    typeof input.failures_before_success === 'number' &&
+    Number.isInteger(input.failures_before_success) &&
+    input.failures_before_success >= 0
+      ? input.failures_before_success
+      : 1;
+  if (attempt <= failuresBeforeSuccess) {
+    const status = toolId === 'test_transient_429' ? 429 : 502;
+    return {
+      ok: false,
+      status,
+      message: `Synthetic transient ${status} for attempt ${attempt}`,
+    };
+  }
+  return {
+    ok: true,
+    result: {
+      status: 'completed',
+      provider: 'test',
+      key: String(input.key ?? 'transient'),
+      attempts: attempt,
+      recovered: attempt > 1,
+    },
+  };
+}
+
 function executeSyntheticTestRateLimit(
   input: Record<string, unknown>,
 ): Record<string, unknown> {
@@ -1417,7 +1531,23 @@ class WorkerToolBatchScheduler {
     workflowStep?: WorkflowStep,
   ): Promise<unknown> {
     return new Promise((resolve, reject) => {
-      this.queue.push({ id, toolId, input, workflowStep, resolve, reject });
+      const queuedAt = nowMs();
+      this.queue.push({
+        id,
+        toolId,
+        input,
+        workflowStep,
+        resolve: (value) => {
+          recordRunnerPerfTrace({
+            req: this.req,
+            phase: 'runner.tool.request',
+            ms: nowMs() - queuedAt,
+            extra: { id, toolId },
+          });
+          resolve(value);
+        },
+        reject,
+      });
       this.scheduleDrain();
     });
   }
@@ -1447,6 +1577,7 @@ class WorkerToolBatchScheduler {
     const requests = this.queue;
     this.queue = [];
     this.scheduled = false;
+    const drainStartedAt = nowMs();
     await Promise.all(
       [...groupWorkerToolRequestsByTool(requests).entries()].map(
         async ([toolId, groupedRequests]) => {
@@ -1454,6 +1585,12 @@ class WorkerToolBatchScheduler {
         },
       ),
     );
+    recordRunnerPerfTrace({
+      req: this.req,
+      phase: 'runner.tool.drain',
+      ms: nowMs() - drainStartedAt,
+      extra: { requests: requests.length },
+    });
     if (this.queue.length > 0) {
       this.scheduleDrain();
     }
@@ -1465,6 +1602,7 @@ class WorkerToolBatchScheduler {
   ): Promise<void> {
     const strategy = getDefaultPlayRuntimeBatchStrategy(toolId);
     if (!strategy || toolId === 'test_wait_for_event') {
+      const groupStartedAt = nowMs();
       await Promise.all(
         requests.map(async (request) => {
           try {
@@ -1480,14 +1618,27 @@ class WorkerToolBatchScheduler {
           }
         }),
       );
+      recordRunnerPerfTrace({
+        req: this.req,
+        phase: 'runner.tool.group',
+        ms: nowMs() - groupStartedAt,
+        extra: { toolId, requests: requests.length, batched: false },
+      });
       return;
     }
 
+    const batchStartedAt = nowMs();
     await executeBatchedWorkerToolGroup({
       req: this.req,
       requests,
       strategy,
     });
+    recordRunnerPerfTrace({
+      req: this.req,
+      phase: 'runner.tool.group',
+      ms: nowMs() - batchStartedAt,
+      extra: { toolId, requests: requests.length, batched: true },
+    });
   }
 }
 
@@ -2292,7 +2443,7 @@ function isStreamingDataset<T extends Record<string, unknown>>(
 
 /**
  * Resolve the sheet contract for a (play, namespace) from the request's
- * contractSnapshot. Required by both direct-Neon writes and the HTTP fallback.
+ * contractSnapshot. Required by direct runtime sheet persistence.
  */
 function resolveSheetContractFromReq(
   req: RunRequest,
@@ -2320,13 +2471,13 @@ function resolveSheetContractFromReq(
 }
 
 /**
- * Direct-Neon writes from the Workers harness. Resolves the org runtime
- * Postgres URL via `create_db_session` (cached by runtime-api per
- * runId+namespace), then runs the same mega-CTE SQL Daytona uses — but over
- * `@neondatabase/serverless` so it runs inside the V8 isolate.
+ * Direct-Neon writes from the shared harness Worker. Resolves the org runtime
+ * Postgres URL via `create_db_session` inside that long-lived Worker, then
+ * runs the same row persistence path Daytona uses without bundling Neon into
+ * every per-play dynamic Worker.
  *
- * No HTTP fallback: if the contract/session can't be obtained, we want a
- * loud failure rather than silently regressing to per-chunk Vercel hops.
+ * If the contract/session can't be obtained, fail loudly rather than silently
+ * regressing to per-chunk Vercel hops.
  */
 function requireSheetContract(
   req: RunRequest,
@@ -2351,12 +2502,9 @@ async function persistCompletedMapRows(input: {
   extraOutputFields?: string[];
 }): Promise<void> {
   if (input.rows.length === 0) return;
-  await postRuntimeApi<{
-    ok: true;
-    rowsWritten: number;
-    tableNamespace: string;
-  }>(input.req.baseUrl, input.req.executorToken, {
-    action: 'persist_completed_sheet_rows',
+  await harnessPersistCompletedSheetRows({
+    baseUrl: input.req.baseUrl,
+    executorToken: input.req.executorToken,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
     sheetContract: requireSheetContract(input.req, input.tableNamespace),
@@ -2368,6 +2516,7 @@ async function persistCompletedMapRows(input: {
       ),
     ],
     runId: input.req.runId,
+    userEmail: input.req.userEmail,
   });
 }
 
@@ -2384,19 +2533,15 @@ async function prepareMapRows(input: {
   if (input.rows.length === 0) {
     return { inserted: 0, skipped: 0, pendingRows: [], completedRows: [] };
   }
-  const result = await postRuntimeApi<{
-    inserted: number;
-    skipped: number;
-    pendingRows: Record<string, unknown>[];
-    completedRows: Record<string, unknown>[];
-    tableNamespace: string;
-  }>(input.req.baseUrl, input.req.executorToken, {
-    action: 'start_sheet_dataset',
+  const result = await harnessStartSheetDataset({
+    baseUrl: input.req.baseUrl,
+    executorToken: input.req.executorToken,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
     sheetContract: requireSheetContract(input.req, input.tableNamespace),
     rows: input.rows.map((row) => ({ ...row })),
     runId: input.req.runId,
+    userEmail: input.req.userEmail,
   });
   return {
     inserted: result.inserted,
@@ -2494,6 +2639,7 @@ function createMinimalWorkerCtx(
   env: WorkerEnv,
   workflowStep?: WorkflowStep,
   abortSignal?: AbortSignal,
+  callbacks?: WorkerCtxCallbacks,
 ): unknown {
   let playCallCount = 0;
   const parentChildCalls: Record<string, number> = {};
@@ -2567,6 +2713,8 @@ function createMinimalWorkerCtx(
     >,
     opts?: WorkerMapOptions,
   ): Promise<unknown> => {
+    const mapStartedAt = nowMs();
+    const mapNodeId = `map:${name}`;
     const sliced = rows;
     const baseOffset = 0;
     const fieldEntries = Object.entries(fieldsDef);
@@ -2587,6 +2735,30 @@ function createMinimalWorkerCtx(
       softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
     });
     const outputFields = fieldEntries.map(([field]) => field);
+    const updateMapProgress = (progress: LiveNodeProgressSnapshot) => {
+      callbacks?.onNodeProgress?.({
+        nodeId: mapNodeId,
+        progress: {
+          artifactTableNamespace: name,
+          failed: 0,
+          ...progress,
+          updatedAt: progress.updatedAt ?? nowMs(),
+        },
+      });
+    };
+    const formatMapProgressMessage = (completed: number, total?: number) =>
+      typeof total === 'number' && Number.isFinite(total) && total > 0
+        ? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
+        : `${completed.toLocaleString()} rows processed`;
+    updateMapProgress({
+      completed: 0,
+      total: streaming ? undefined : sliced.length,
+      startedAt: mapStartedAt,
+      message: formatMapProgressMessage(
+        0,
+        streaming ? undefined : sliced.length,
+      ),
+    });
     const explicitRowKeysSeen =
       opts?.key === undefined ? null : new Map<string, number>();
     const resolveExplicitKeyValue = (
@@ -2669,12 +2841,21 @@ function createMinimalWorkerCtx(
       chunkStart: number,
       chunkIndex: number,
     ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
+      const chunkStartedAt = nowMs();
       assertNotAborted(abortSignal);
+      const keyStartedAt = nowMs();
       const chunkEntries = chunkRows.map((row, localIndex) => {
         const absoluteIndex = baseOffset + chunkStart + localIndex;
         const rowKey = resolveRowKey(row, absoluteIndex);
         return { row, absoluteIndex, rowKey };
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.keys',
+        ms: nowMs() - keyStartedAt,
+        extra: { mapName: name, chunkIndex, rows: chunkRows.length },
+      });
+      const prepareStartedAt = nowMs();
       const prepared = await prepareMapRows({
         req,
         tableNamespace: name,
@@ -2683,6 +2864,20 @@ function createMinimalWorkerCtx(
           __deeplineRowKey: rowKey,
         })),
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.prepare_rows',
+        ms: nowMs() - prepareStartedAt,
+        extra: {
+          mapName: name,
+          chunkIndex,
+          rows: chunkRows.length,
+          inserted: prepared.inserted,
+          skipped: prepared.skipped,
+          pendingRows: prepared.pendingRows.length,
+          completedRows: prepared.completedRows.length,
+        },
+      });
       const pendingKeys = new Set<string>();
       const completedKeys = new Set<string>();
       const preparedKeys = new Set<string>();
@@ -2901,8 +3096,56 @@ function createMinimalWorkerCtx(
           })),
         });
       };
+      const workersStartedAt = nowMs();
       const workerResults = await Promise.allSettled(workers);
-      await persistExecutedRows();
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.execute_workers',
+        ms: nowMs() - workersStartedAt,
+        extra: {
+          mapName: name,
+          chunkIndex,
+          rowsToExecute: rowsToExecute.length,
+          concurrency,
+        },
+      });
+      const persistRowsStartedAt = nowMs();
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.persist_rows_start',
+        ms: 0,
+        extra: {
+          mapName: name,
+          chunkIndex,
+          rowsToExecute: rowsToExecute.length,
+        },
+      });
+      try {
+        await persistExecutedRows();
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.map_chunk.persist_rows',
+          ms: nowMs() - persistRowsStartedAt,
+          extra: {
+            mapName: name,
+            chunkIndex,
+            rowsToExecute: rowsToExecute.length,
+          },
+        });
+      } catch (error) {
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.map_chunk.persist_rows_error',
+          ms: nowMs() - persistRowsStartedAt,
+          extra: {
+            mapName: name,
+            chunkIndex,
+            rowsToExecute: rowsToExecute.length,
+            error: error instanceof Error ? error.message : String(error),
+          },
+        });
+        throw error;
+      }
       const rejectedWorker = workerResults.find(
         (result): result is PromiseRejectedResult =>
           result.status === 'rejected',
@@ -2942,6 +3185,27 @@ function createMinimalWorkerCtx(
           return resultByKey.get(key);
         })
         .filter((row): row is T & Record<string, unknown> => Boolean(row));
+      const hashStartedAt = nowMs();
+      const hash = await hashJson(out);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.hash',
+        ms: nowMs() - hashStartedAt,
+        extra: { mapName: name, chunkIndex, rows: out.length },
+      });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.total',
+        ms: nowMs() - chunkStartedAt,
+        extra: {
+          mapName: name,
+          chunkIndex,
+          rowsRead: chunkRows.length,
+          rowsWritten: out.length,
+          rowsExecuted: executedRows.length,
+          rowsCached: Math.max(0, out.length - executedRows.length),
+        },
+      });
       return {
         chunkIndex,
         rangeStart: baseOffset + chunkStart,
@@ -2954,7 +3218,7 @@ function createMinimalWorkerCtx(
         rowsInserted,
         rowsSkipped,
         outputDatasetId: `map:${name}`,
-        hash: await hashJson(out),
+        hash,
         preview: toWorkflowSerializableValue(out.slice(0, 5)),
       };
     };
@@ -2978,7 +3242,7 @@ function createMinimalWorkerCtx(
         workflowStep.do as unknown as (
           name: string,
           config: {
-            retries: { limit: number; delay: string; backoff: 'exponential' };
+            retries: { limit: number; delay: number; backoff: 'exponential' };
           },
           callback: () => Promise<
             WorkerMapChunkSummary<T & Record<string, unknown>>
@@ -2986,7 +3250,7 @@ function createMinimalWorkerCtx(
         ) => Promise<WorkerMapChunkSummary<T & Record<string, unknown>>>
       )(
         deterministicMapChunkStepName({ mapName: name, chunkIndex }),
-        { retries: { limit: 5, delay: '5 seconds', backoff: 'exponential' } },
+        { retries: { limit: 5, delay: 100, backoff: 'exponential' } },
         async () => await processChunk(chunkRows, chunkStart, chunkIndex),
       )) as WorkerMapChunkSummary<T & Record<string, unknown>>;
     };
@@ -2996,6 +3260,14 @@ function createMinimalWorkerCtx(
         `Map completed: ${totalRowsWritten} results ` +
         `(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
         `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
+      const completedAt = nowMs();
+      updateMapProgress({
+        completed: totalRowsWritten,
+        total: totalRowsWritten,
+        completedAt,
+        updatedAt: completedAt,
+        message: formatMapProgressMessage(totalRowsWritten, totalRowsWritten),
+      });
       emitEvent({
         type: 'log',
         level: 'info',
@@ -3039,13 +3311,24 @@ function createMinimalWorkerCtx(
         totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
         totalRowsInserted += chunkResult.rowsInserted;
         totalRowsSkipped += chunkResult.rowsSkipped;
+        updateMapProgress({
+          completed: totalRowsWritten,
+          message: formatMapProgressMessage(totalRowsWritten),
+        });
         if (out.length < 10) {
           out.push(...chunkResult.preview.slice(0, 10 - out.length));
         }
         chunkStart += chunkRows.length;
         chunkIndex += 1;
       }
-      return finalize(totalRowsWritten);
+      const dataset = finalize(totalRowsWritten);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map.total',
+        ms: nowMs() - mapStartedAt,
+        extra: { mapName: name, rowsWritten: totalRowsWritten, streaming: true },
+      });
+      return dataset;
     }
 
     if (workflowStep && sliced.length > rowsPerChunk) {
@@ -3063,11 +3346,23 @@ function createMinimalWorkerCtx(
         totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
         totalRowsInserted += chunkResult.rowsInserted;
         totalRowsSkipped += chunkResult.rowsSkipped;
+        updateMapProgress({
+          completed: totalRowsWritten,
+          total: sliced.length,
+          message: formatMapProgressMessage(totalRowsWritten, sliced.length),
+        });
         if (out.length < 10) {
           out.push(...chunkResult.preview.slice(0, 10 - out.length));
         }
       }
-      return finalize(totalRowsWritten);
+      const dataset = finalize(totalRowsWritten);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map.total',
+        ms: nowMs() - mapStartedAt,
+        extra: { mapName: name, rowsWritten: totalRowsWritten, streaming: false },
+      });
+      return dataset;
     }
 
     assertUniqueExplicitRowKeys(sliced, 0);
@@ -3078,7 +3373,23 @@ function createMinimalWorkerCtx(
     totalRowsInserted = chunkResult.rowsInserted;
     totalRowsSkipped = chunkResult.rowsSkipped;
     out.push(...chunkResult.preview);
-    return finalize(chunkResult.rowsWritten);
+    updateMapProgress({
+      completed: chunkResult.rowsWritten,
+      total: sliced.length,
+      message: formatMapProgressMessage(chunkResult.rowsWritten, sliced.length),
+    });
+    const dataset = finalize(chunkResult.rowsWritten);
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.map.total',
+      ms: nowMs() - mapStartedAt,
+      extra: {
+        mapName: name,
+        rowsWritten: chunkResult.rowsWritten,
+        streaming: false,
+      },
+    });
+    return dataset;
   };
 
   class WorkerMapBuilder<T extends Record<string, unknown>> {
@@ -3165,28 +3476,43 @@ function createMinimalWorkerCtx(
       arg: unknown,
       options?: CsvRenameOptions,
     ): Promise<T[]> {
+      const csvStartedAt = nowMs();
       if (Array.isArray(arg)) {
         // Inline rows passed at call site — already in memory, keep the
         // legacy array-backed dataset shape.
-        return makeWorkerDataset(
+        const dataset = makeWorkerDataset(
           'csv',
           applyCsvRenameProjection(arg as T[], options),
           {
             datasetKind: 'csv',
           },
         ) as unknown as T[];
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.csv',
+          ms: nowMs() - csvStartedAt,
+          extra: { mode: 'inline_array', rows: arg.length },
+        });
+        return dataset;
       }
       const filename = String(arg ?? '');
       if (req.inlineCsv && filename === req.inlineCsv.name) {
         // Inline CSV pre-staged by the dispatcher (small files <1 MiB). Already
         // in memory; no streaming needed.
-        return makeWorkerDataset(
+        const dataset = makeWorkerDataset(
           'csv',
           applyCsvRenameProjection(req.inlineCsv.rows as T[], options),
           {
             datasetKind: 'csv',
           },
         ) as unknown as T[];
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.csv',
+          ms: nowMs() - csvStartedAt,
+          extra: { mode: 'inline_csv', rows: req.inlineCsv.rows.length },
+        });
+        return dataset;
       }
       // Resolution order: explicit inputR2Keys (runtime input) → packaged
       // files (relative-path imports bundled with the play artifact).
@@ -3211,7 +3537,7 @@ function createMinimalWorkerCtx(
       // and switches its chunked execution loop to consume iterChunks
       // directly, so 2M-row CSVs never get fully materialized in memory.
       const storageKey = r2Key;
-      return makeStreamingCsvDataset<T>({
+      const dataset = makeStreamingCsvDataset<T>({
         name: filename,
         logicalPath: filename,
         renameOptions: options,
@@ -3223,6 +3549,13 @@ function createMinimalWorkerCtx(
             storageKey,
           }),
       }) as unknown as T[];
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.csv',
+        ms: nowMs() - csvStartedAt,
+        extra: { mode: 'streaming_r2', filename },
+      });
+      return dataset;
     },
     map<T extends Record<string, unknown>>(
       name: string,
@@ -3253,346 +3586,6 @@ function createMinimalWorkerCtx(
       throw new Error(
         'ctx.map(key, rows, fields, options) was removed. Use ctx.map(key, rows).step(...).run(options).',
       );
-      /*
-      const sliced = rows;
-      const baseOffset = 0;
-      const fieldEntries = Object.entries(fieldsDef);
-      const plan = req.executionPlan;
-      const planMap = plan?.maps.find(
-        (candidate) =>
-          candidate.mapName === name || candidate.tableNamespace === name,
-      );
-      const streaming = isStreamingDataset<T>(sliced);
-      // For streaming inputs we don't know the row count upfront — pass
-      // `totalRows: 0` so chooseMapChunkSize falls back to the preferred /
-      // default chunk size rather than trying to budget against an unknown.
-      const rowsPerChunk = chooseMapChunkSize({
-        totalRows: streaming ? 0 : sliced.length,
-        mapCount: Math.max(1, plan?.maps.length ?? 1),
-        stepsPerChunk: planMap?.stepsPerChunk ?? 1,
-        preferredChunkSize: planMap?.defaultChunkSize,
-        softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
-      });
-      const outputFields = fieldEntries.map(([field]) => field);
-
-      // Workflow steps have bounded CPU but unbounded wall time, so long
-      // network calls are OK here as long as we checkpoint by chunk and do not
-      // collapse a giant map into one monolithic step result.
-
-      const processChunk = async (
-        chunkRows: T[],
-        chunkStart: number,
-        chunkIndex: number,
-      ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
-        assertNotAborted(abortSignal);
-        const prepared = await prepareMapRows({
-          req,
-          tableNamespace: name,
-          rows: chunkRows,
-        });
-        const pendingKeys = new Set<string>();
-        const completedKeys = new Set<string>();
-        const preparedKeys = new Set<string>();
-        for (const row of prepared.pendingRows) {
-          const key = derivePlayRowIdentity(
-            publicCsvInputRow(row),
-            name,
-            mapLogicFingerprint,
-          );
-          if (key) {
-            pendingKeys.add(key);
-            preparedKeys.add(key);
-          }
-        }
-        for (const row of prepared.completedRows) {
-          const key =
-            typeof row.__deeplineRowKey === 'string'
-              ? row.__deeplineRowKey
-              : derivePlayRowIdentity(
-                  publicCsvInputRow(row),
-                  name,
-                  mapLogicFingerprint,
-                );
-          if (key) {
-            completedKeys.add(key);
-            preparedKeys.add(key);
-          }
-        }
-        const missingPreparedRows = chunkRows.filter((row) => {
-          const key = derivePlayRowIdentity(
-            publicCsvInputRow(row),
-            name,
-            mapLogicFingerprint,
-          );
-          return !key || !preparedKeys.has(key);
-        });
-        const rowsToExecute = chunkRows.filter((row) => {
-          const key = derivePlayRowIdentity(
-            publicCsvInputRow(row),
-            name,
-            mapLogicFingerprint,
-          );
-          return !key || pendingKeys.has(key) || !completedKeys.has(key);
-        });
-        const rowsInserted = prepared.inserted + missingPreparedRows.length;
-        const rowsSkipped = Math.max(
-          0,
-          prepared.skipped - missingPreparedRows.length,
-        );
-        const concurrency = Math.max(1, Math.min(opts?.concurrency ?? 10, 100));
-        const executedRows: Array<T & Record<string, unknown>> = new Array(
-          rowsToExecute.length,
-        );
-        const generatedOutputFields = new Set<string>();
-        let idx = 0;
-        const workers: Array<Promise<void>> = [];
-        for (let w = 0; w < concurrency; w += 1) {
-          workers.push(
-            (async () => {
-              while (true) {
-                if (abortSignal?.aborted) return;
-                const myIndex = idx++;
-                if (myIndex >= rowsToExecute.length) return;
-                const row = rowsToExecute[myIndex]!;
-                const absoluteIndex = baseOffset + chunkStart + myIndex;
-                const enriched: Record<string, unknown> = cloneCsvAliasedRow(row);
-                const fieldOutputs: Record<string, unknown> = {};
-                const waterfallOutputs: RecordedWaterfallOutput[] = [];
-                const rowCtx = {
-                  ...(ctx as Record<string, unknown>),
-                  waterfall: (
-                    toolNameOrSpec: string | WorkerInlineWaterfallSpec,
-                    waterfallInput: Record<string, unknown>,
-                    waterfallOpts?: WorkerWaterfallOptions,
-                  ) =>
-                    executeWorkerWaterfall(
-                      req,
-                      waterfallOutputs,
-                      toolNameOrSpec,
-                      waterfallInput,
-                      waterfallOpts,
-                    ),
-                };
-                for (const [key, value] of fieldEntries) {
-                  if (typeof value === 'function') {
-                    const resolved = await (
-                      value as (
-                        r: T,
-                        c: unknown,
-                        f: Record<string, unknown>,
-                        i: number,
-                      ) => Promise<unknown> | unknown
-                    )(row, rowCtx, fieldOutputs, absoluteIndex);
-                    enriched[key] = resolved;
-                    fieldOutputs[key] = resolved;
-                  } else {
-                    enriched[key] = value;
-                    fieldOutputs[key] = value;
-                  }
-                }
-                for (const waterfallOutput of waterfallOutputs) {
-                  const columnName =
-                    `${sqlishIdentifierPart(waterfallOutput.waterfallId)}__` +
-                    sqlishIdentifierPart(waterfallOutput.stepId);
-                  enriched[columnName] = waterfallOutput.value;
-                  generatedOutputFields.add(columnName);
-                }
-                executedRows[myIndex] = enriched as T & Record<string, unknown>;
-              }
-            })(),
-          );
-        }
-        await Promise.all(workers);
-        if (executedRows.length > 0) {
-          await persistCompletedMapRows({
-            req,
-            tableNamespace: name,
-            outputFields,
-            extraOutputFields: Array.from(generatedOutputFields),
-            rows: executedRows.map((row, executedIndex) => ({
-              ...row,
-              __deeplineRowKey: derivePlayRowIdentity(
-                publicCsvInputRow(rowsToExecute[executedIndex]!),
-                name,
-                mapLogicFingerprint,
-              ),
-            })),
-          });
-        }
-        const resultByKey = new Map<string, T & Record<string, unknown>>();
-        for (const completedRow of prepared.completedRows) {
-          const key =
-            typeof completedRow.__deeplineRowKey === 'string'
-              ? completedRow.__deeplineRowKey
-              : derivePlayRowIdentity(
-                  publicCsvInputRow(completedRow),
-                  name,
-                  mapLogicFingerprint,
-                );
-          if (key) {
-            const { __deeplineRowKey: _rowKey, ...cleanedRow } =
-              publicCsvInputRow(completedRow);
-            void _rowKey;
-            resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
-          }
-        }
-        for (
-          let executedIndex = 0;
-          executedIndex < executedRows.length;
-          executedIndex += 1
-        ) {
-          const executedRow = executedRows[executedIndex]!;
-          const key = derivePlayRowIdentity(
-            publicCsvInputRow(rowsToExecute[executedIndex]!),
-            name,
-            mapLogicFingerprint,
-          );
-          if (key) resultByKey.set(key, executedRow);
-        }
-        const out = chunkRows
-          .map((row) => {
-            const key = derivePlayRowIdentity(
-              publicCsvInputRow(row),
-              name,
-              mapLogicFingerprint,
-            );
-            return key ? resultByKey.get(key) : undefined;
-          })
-          .filter((row): row is T & Record<string, unknown> => Boolean(row));
-        return {
-          chunkIndex,
-          rangeStart: baseOffset + chunkStart,
-          rangeEnd: baseOffset + chunkStart + out.length,
-          rowsRead: chunkRows.length,
-          rowsWritten: out.length,
-          rowsExecuted: executedRows.length,
-          rowsCached: prepared.completedRows.length,
-          rowsInserted,
-          rowsSkipped,
-          outputDatasetId: `map:${name}`,
-          hash: await hashJson(out),
-          preview: toWorkflowSerializableValue(out.slice(0, 5)),
-        };
-      };
-
-      const out: Array<T & Record<string, unknown>> = [];
-      let totalRowsExecuted = 0;
-      let totalRowsCached = 0;
-      let totalRowsInserted = 0;
-      let totalRowsSkipped = 0;
-
-      const runChunkStep = async (
-        chunkRows: T[],
-        chunkStart: number,
-        chunkIndex: number,
-      ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
-        if (!workflowStep) {
-          return await processChunk(chunkRows, chunkStart, chunkIndex);
-        }
-        return (await (
-          workflowStep.do as unknown as (
-            name: string,
-            config: {
-              retries: { limit: number; delay: string; backoff: 'exponential' };
-            },
-            callback: () => Promise<
-              WorkerMapChunkSummary<T & Record<string, unknown>>
-            >,
-          ) => Promise<WorkerMapChunkSummary<T & Record<string, unknown>>>
-        )(
-          deterministicMapChunkStepName({ mapName: name, chunkIndex }),
-          { retries: { limit: 5, delay: '5 seconds', backoff: 'exponential' } },
-          async () => await processChunk(chunkRows, chunkStart, chunkIndex),
-        )) as WorkerMapChunkSummary<T & Record<string, unknown>>;
-      };
-
-      const finalize = (totalRowsWritten: number) => {
-        const cacheSummary =
-          `Map completed: ${totalRowsWritten} results ` +
-          `(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
-          `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
-        emitEvent({
-          type: 'log',
-          level: 'info',
-          message: cacheSummary,
-          ts: nowMs(),
-        });
-        return makeWorkerDataset(name, out, {
-          count: totalRowsWritten,
-          cacheSummary,
-          workProgress: {
-            total: totalRowsWritten,
-            executed: totalRowsExecuted,
-            reused: totalRowsCached,
-            skipped: totalRowsCached,
-            pending: 0,
-            failed: 0,
-          },
-        });
-      };
-
-      // Streaming path: input came from `ctx.csv` over R2 and we never
-      // materialized the full row array. Pull row chunks from the dataset's
-      // iterChunks() and run each through processChunk inside its own
-      // workflow step. Memory stays bounded by `rowsPerChunk`.
-      if (streaming) {
-        let totalRowsWritten = 0;
-        let chunkIndex = 0;
-        let chunkStart = 0;
-        const streamingDataset = sliced as unknown as StreamingCsvDataset<T>;
-        for await (const chunkRows of streamingDataset.iterChunks(
-          rowsPerChunk,
-        )) {
-          assertNotAborted(abortSignal);
-          if (chunkRows.length === 0) continue;
-          const chunkResult = await runChunkStep(
-            chunkRows,
-            chunkStart,
-            chunkIndex,
-          );
-          totalRowsWritten += chunkResult.rowsWritten;
-          totalRowsExecuted += chunkResult.rowsExecuted;
-          totalRowsCached += chunkResult.rowsCached;
-          totalRowsInserted += chunkResult.rowsInserted;
-          totalRowsSkipped += chunkResult.rowsSkipped;
-          if (out.length < 10) {
-            out.push(...chunkResult.preview.slice(0, 10 - out.length));
-          }
-          chunkStart += chunkRows.length;
-          chunkIndex += 1;
-        }
-        return finalize(totalRowsWritten);
-      }
-
-      // Materialized array path (inline rows or small CSV).
-      if (workflowStep && sliced.length > rowsPerChunk) {
-        let totalRowsWritten = 0;
-        for (let start = 0; start < sliced.length; start += rowsPerChunk) {
-          assertNotAborted(abortSignal);
-          const end = Math.min(sliced.length, start + rowsPerChunk);
-          const chunkRows = sliced.slice(start, end);
-          const chunkIndex = Math.floor(start / rowsPerChunk);
-          const chunkResult = await runChunkStep(chunkRows, start, chunkIndex);
-          totalRowsWritten += chunkResult.rowsWritten;
-          totalRowsExecuted += chunkResult.rowsExecuted;
-          totalRowsCached += chunkResult.rowsCached;
-          totalRowsInserted += chunkResult.rowsInserted;
-          totalRowsSkipped += chunkResult.rowsSkipped;
-          if (out.length < 10) {
-            out.push(...chunkResult.preview.slice(0, 10 - out.length));
-          }
-        }
-        return finalize(totalRowsWritten);
-      }
-
-      const chunkResult = await processChunk(sliced, 0, 0);
-      totalRowsExecuted = chunkResult.rowsExecuted;
-      totalRowsCached = chunkResult.rowsCached;
-      totalRowsInserted = chunkResult.rowsInserted;
-      totalRowsSkipped = chunkResult.rowsSkipped;
-      out.push(...chunkResult.preview);
-      return finalize(chunkResult.rowsWritten);
-      */
     },
     tool: async (
       key: string,
@@ -4012,6 +4005,7 @@ async function executeRunRequest(
   workflowStep?: WorkflowStep,
   options?: {
     persistResultDatasets?: boolean;
+    waitUntil?: (promise: Promise<unknown>) => void;
     /**
      * Cooperative cancellation token. CF Workflows surfaces termination as a
      * thrown error from any in-progress step; the harness catches that, flips
@@ -4022,14 +4016,36 @@ async function executeRunRequest(
 ): Promise<WorkflowRunOutput> {
   installProcessExitTrap();
   const startedAt = nowMs();
+  recordRunnerPerfTrace({
+    req,
+    phase: 'runner.execute_start',
+    extra: {
+      persistResultDatasets: Boolean(options?.persistResultDatasets),
+      hasWorkflowStep: Boolean(workflowStep),
+    },
+  });
   const abortController = options?.abortController ?? new AbortController();
   const abortSignal = abortController.signal;
+  const postgresPrewarmStartedAt = nowMs();
+  await harnessPrewarmPostgresSessions({
+    executorToken: req.executorToken,
+    sessions: req.preloadedDbSessions ?? [],
+  });
+  recordRunnerPerfTrace({
+    req,
+    phase: 'runner.prewarm_postgres',
+    ms: nowMs() - postgresPrewarmStartedAt,
+    extra: {
+      sessions: req.preloadedDbSessions?.length ?? 0,
+    },
+  });
   // Maintain a rolling buffer of log lines emitted during the run. This is
   // what the play-page UI consumes via Convex polling + diffPlayRunStreamEvents
   // → play.run.log SSE events. Without periodic flushing, the play page only
   // sees the final terminal status with no intermediate logs/progress.
   let liveLogs: string[] = [];
   let liveLogsDirty = false;
+  let liveNodeProgress: LiveNodeProgressMap = {};
   let lastLiveLogFlushAt =
     nowMs() - LIVE_LOG_FLUSH_INTERVAL_MS + LIVE_LOG_FIRST_FLUSH_DELAY_MS;
   let liveLogFlushInFlight: Promise<void> = Promise.resolve();
@@ -4039,6 +4055,21 @@ async function executeRunRequest(
     liveLogs = [...liveLogs, trimmed].slice(-LIVE_LOG_BUFFER_LIMIT);
     liveLogsDirty = true;
   };
+  const updateLiveNodeProgress = (input: {
+    nodeId: string;
+    progress: LiveNodeProgressSnapshot;
+  }) => {
+    const nodeId = input.nodeId.trim();
+    if (!nodeId) return;
+    liveNodeProgress = {
+      ...liveNodeProgress,
+      [nodeId]: {
+        ...(liveNodeProgress[nodeId] ?? {}),
+        ...input.progress,
+      },
+    };
+  };
+  const liveNodeProgressSnapshot = () => ({ ...liveNodeProgress });
   const flushLiveLogs = (force: boolean): void => {
     if (!options?.persistResultDatasets) return;
     if (!liveLogsDirty && !force) return;
@@ -4057,6 +4088,7 @@ async function executeRunRequest(
             status: 'running',
             runtimeBackend: 'cf_workflows_dynamic_worker',
             liveLogs: snapshot,
+            liveNodeProgress: liveNodeProgressSnapshot(),
             lastCheckpointAt: now,
           });
         } catch {
@@ -4086,37 +4118,77 @@ async function executeRunRequest(
     env,
     workflowStep,
     abortSignal,
+    { onNodeProgress: updateLiveNodeProgress },
   );
   try {
+    const playStartedAt = nowMs();
     const result = await (
       playFn as (
         ctx: unknown,
         input: Record<string, unknown>,
       ) => Promise<unknown>
     )(ctx, req.runtimeInput);
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.play_function',
+      ms: nowMs() - playStartedAt,
+    });
+    const serializeStartedAt = nowMs();
     const serializedResult = serializePlayReturnValue(result);
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.serialize_result',
+      ms: nowMs() - serializeStartedAt,
+    });
     if (options?.persistResultDatasets) {
+      const persistStartedAt = nowMs();
       await liveLogFlushInFlight.catch(() => undefined);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.live_log_flush_wait',
+        ms: nowMs() - persistStartedAt,
+      });
+      const resultDatasetStartedAt = nowMs();
       await persistResultDatasets(req, serializedResult);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.persist_result_datasets',
+        ms: nowMs() - resultDatasetStartedAt,
+      });
       const terminalResult = trimResultForStatus(serializedResult);
+      const terminalUpdateStartedAt = nowMs();
       await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
         action: 'update_run_status',
         playId: req.runId,
         status: 'completed',
+        error: null,
         result: terminalResult,
         runtimeBackend: 'cf_workflows_dynamic_worker',
         waitKind: null,
         waitUntil: null,
         activeBoundaryId: null,
         liveLogs,
+        liveNodeProgress: liveNodeProgressSnapshot(),
         lastCheckpointAt: nowMs(),
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.terminal_status_update',
+        ms: nowMs() - terminalUpdateStartedAt,
+      });
+      const billingStartedAt = nowMs();
       await finalizeWorkerComputeBilling({
         req,
         success: true,
         actionEstimate: 4,
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.compute_billing_finalize',
+        ms: nowMs() - billingStartedAt,
+      });
     }
+    const parentSignalStartedAt = nowMs();
     await signalParentPlayTerminal({
       req,
       status: 'completed',
@@ -4128,6 +4200,16 @@ async function executeRunRequest(
         }`,
       );
     });
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.parent_terminal_signal',
+      ms: nowMs() - parentSignalStartedAt,
+    });
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.execute_total',
+      ms: nowMs() - startedAt,
+    });
     return {
       playName: req.playName,
       result: serializedResult,
@@ -4156,6 +4238,7 @@ async function executeRunRequest(
         waitUntil: null,
         activeBoundaryId: null,
         liveLogs,
+        liveNodeProgress: liveNodeProgressSnapshot(),
         lastCheckpointAt: nowMs(),
       });
       await finalizeWorkerComputeBilling({
@@ -4285,6 +4368,9 @@ function runRequestFromWorkflowParams(
     playCallGovernance: isPlayCallGovernanceSnapshot(params.playCallGovernance)
       ? params.playCallGovernance
       : null,
+    preloadedDbSessions: Array.isArray(params.preloadedDbSessions)
+      ? (params.preloadedDbSessions as PreloadedRuntimeDbSession[])
+      : null,
     coordinatorUrl:
       typeof params.coordinatorUrl === 'string' && params.coordinatorUrl.trim()
         ? params.coordinatorUrl.trim()
@@ -4525,6 +4611,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
   Record<string, unknown>
 > {
   declare readonly env: WorkerEnv;
+  declare readonly ctx: ExecutionContext;
 
   /**
    * Cancellation model:
@@ -4568,14 +4655,30 @@ export class TenantWorkflow extends WorkflowEntrypoint<
         (runnerEvent) => {
           if (runnerEvent.type === 'log') {
             console.log(`${runPrefix} ${runnerEvent.message}`);
+            void this.env.COORDINATOR?.recordRunEvent(req.runId, {
+              runId: req.runId,
+              type: 'log',
+              line: runnerEvent.message,
+              ts: runnerEvent.ts,
+            }).catch(() => undefined);
           } else if (runnerEvent.type === 'error') {
             console.error(
               `${runPrefix} ${runnerEvent.message}${runnerEvent.stack ? `\n${runnerEvent.stack}` : ''}`,
             );
+            void this.env.COORDINATOR?.recordRunEvent(req.runId, {
+              runId: req.runId,
+              type: 'log',
+              line: `[error] ${runnerEvent.message}`,
+              ts: runnerEvent.ts,
+            }).catch(() => undefined);
           }
         },
         step,
-        { persistResultDatasets: !req.playCallGovernance, abortController },
+        {
+          persistResultDatasets: !req.playCallGovernance,
+          abortController,
+          waitUntil: (promise) => this.ctx.waitUntil(promise),
+        },
       )) as Record<string, unknown>;
     } catch (error) {
       // CF Workflows + the dynamic-workflows framework swallow the error