deepline 0.1.19 → 0.1.21

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -56,6 +56,7 @@ import {
   type ToolResultMetadataInput,
 } from '../../../shared_libs/play-runtime/tool-result';
 import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
+import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
 import type { PlayRuntimeManifestMap } from '../../../shared_libs/plays/compiler-manifest';
 import {
   derivePlayRowIdentity,
@@ -63,6 +64,7 @@ import {
 } from '../../../shared_libs/plays/row-identity';
 import {
   getCompiledPipelineSubsteps,
+  flattenStaticPipeline,
   resolveSheetContractForTableNamespace,
   sqlSafePlayColumnName,
   type PlayStaticPipeline,
@@ -79,6 +81,8 @@ import {
 // re-bundle harness internals into per-play. Keep that in mind.
 import {
   harnessFetchStagedFile,
+  harnessPersistCompletedSheetRows,
+  harnessPrewarmPostgresSessions,
   harnessStartSheetDataset,
   setHarnessBinding,
 } from '../../../sdk/src/plays/harness-stub';
@@ -128,6 +132,7 @@ type RunRequest = {
   childPlayManifests?: PlayRuntimeManifestMap | null;
   /** Internal ctx.runPlay lineage. Public SDK/users never see this. */
   playCallGovernance?: PlayCallGovernanceSnapshot | null;
+  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
   /** Cloudflare coordinator URL for direct Workflow control-plane signals. */
   coordinatorUrl?: string | null;
   /** Request-scoped coordinator auth token for preview/dev direct control calls. */
@@ -190,6 +195,10 @@ type WorkerEnv = {
       runId: string,
       payload: Record<string, unknown>,
     ): Promise<void>;
+    recordRunEvent(
+      runId: string,
+      event: Record<string, unknown>,
+    ): Promise<void>;
   };
   /**
    * Service binding to the long-lived Play Harness Worker
@@ -365,8 +374,9 @@ function cachedVercelProtectionBypassToken(): string | null {
 
 const WORKER_PLAY_CALL_LIMITS = {
   maxPlayCallDepth: 6,
-  maxPlayCallCount: 32,
-  maxChildPlayCallsPerParent: 16,
+  maxPlayCallCount: 1_000,
+  maxChildPlayCallsPerParent: 1_000,
+  maxConcurrentPlayCalls: 16,
 };
 
 /**
@@ -518,6 +528,34 @@ function nowMs(): number {
   return Date.now();
 }
 
+function recordRunnerPerfTrace(input: {
+  req: RunRequest;
+  phase: string;
+  ms?: number;
+  extra?: Record<string, unknown>;
+}): void {
+  if (!input.req.runId || !input.phase) return;
+  const payload = {
+    ts: Date.now(),
+    source: 'dynamic_worker' as const,
+    runId: input.req.runId,
+    phase: `runner.${input.phase}`,
+    ...(input.ms !== undefined ? { ms: input.ms } : {}),
+    ...(input.extra ?? {}),
+  };
+  console.log(
+    `[deepline-run:${input.req.runId}] [perf-trace] ${JSON.stringify(payload)}`,
+  );
+  cachedCoordinatorBinding
+    ?.recordPerfTrace(input.req.runId, payload)
+    .catch((error: unknown) => {
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(
+        `[deepline-run:${input.req.runId}] failed to forward runner perf trace: ${message}`,
+      );
+    });
+}
+
 function makeRequestId(): string {
   // Workers crypto.randomUUID is available without nodejs_compat.
   return crypto.randomUUID();
@@ -637,6 +675,7 @@ async function postRuntimeApiBestEffort(
 async function submitChildPlayThroughCoordinator(input: {
   req: RunRequest;
   body: unknown;
+  allowInline?: boolean;
 }): Promise<{
   workflowId?: string;
   runId?: string;
@@ -648,7 +687,7 @@ async function submitChildPlayThroughCoordinator(input: {
   logs?: string[];
   timings?: Array<{ phase: string; ms: number }>;
 }> {
-  if (cachedCoordinatorBinding) {
+  if (cachedCoordinatorBinding && input.allowInline !== false) {
     if (!isRecord(input.body)) {
       throw new Error('ctx.runPlay child submit requires an object body.');
     }
@@ -657,7 +696,7 @@ async function submitChildPlayThroughCoordinator(input: {
   const coordinatorUrl = input.req.coordinatorUrl?.trim();
   if (coordinatorUrl) {
     // Keep child plays on the same coordinator/Workflow submit path as
-    // top-level runs; the RPC binding remains only as a no-URL fallback.
+    // top-level runs when the coordinator URL is present.
     const res = await fetch(
       `${coordinatorUrl.replace(/\/$/, '')}/workflow/${encodeURIComponent(
         input.req.runId,
@@ -1414,7 +1453,23 @@ class WorkerToolBatchScheduler {
     workflowStep?: WorkflowStep,
   ): Promise<unknown> {
     return new Promise((resolve, reject) => {
-      this.queue.push({ id, toolId, input, workflowStep, resolve, reject });
+      const queuedAt = nowMs();
+      this.queue.push({
+        id,
+        toolId,
+        input,
+        workflowStep,
+        resolve: (value) => {
+          recordRunnerPerfTrace({
+            req: this.req,
+            phase: 'runner.tool.request',
+            ms: nowMs() - queuedAt,
+            extra: { id, toolId },
+          });
+          resolve(value);
+        },
+        reject,
+      });
       this.scheduleDrain();
     });
   }
@@ -1444,6 +1499,7 @@ class WorkerToolBatchScheduler {
     const requests = this.queue;
     this.queue = [];
     this.scheduled = false;
+    const drainStartedAt = nowMs();
     await Promise.all(
       [...groupWorkerToolRequestsByTool(requests).entries()].map(
         async ([toolId, groupedRequests]) => {
@@ -1451,6 +1507,12 @@ class WorkerToolBatchScheduler {
         },
       ),
     );
+    recordRunnerPerfTrace({
+      req: this.req,
+      phase: 'runner.tool.drain',
+      ms: nowMs() - drainStartedAt,
+      extra: { requests: requests.length },
+    });
     if (this.queue.length > 0) {
       this.scheduleDrain();
     }
@@ -1462,6 +1524,7 @@ class WorkerToolBatchScheduler {
   ): Promise<void> {
     const strategy = getDefaultPlayRuntimeBatchStrategy(toolId);
     if (!strategy || toolId === 'test_wait_for_event') {
+      const groupStartedAt = nowMs();
       await Promise.all(
         requests.map(async (request) => {
           try {
@@ -1477,14 +1540,27 @@ class WorkerToolBatchScheduler {
           }
         }),
       );
+      recordRunnerPerfTrace({
+        req: this.req,
+        phase: 'runner.tool.group',
+        ms: nowMs() - groupStartedAt,
+        extra: { toolId, requests: requests.length, batched: false },
+      });
       return;
     }
 
+    const batchStartedAt = nowMs();
     await executeBatchedWorkerToolGroup({
       req: this.req,
       requests,
       strategy,
     });
+    recordRunnerPerfTrace({
+      req: this.req,
+      phase: 'runner.tool.group',
+      ms: nowMs() - batchStartedAt,
+      extra: { toolId, requests: requests.length, batched: true },
+    });
   }
 }
 
@@ -2289,7 +2365,7 @@ function isStreamingDataset<T extends Record<string, unknown>>(
 
 /**
  * Resolve the sheet contract for a (play, namespace) from the request's
- * contractSnapshot. Required by both direct-Neon writes and the HTTP fallback.
+ * contractSnapshot. Required by direct runtime sheet persistence.
  */
 function resolveSheetContractFromReq(
   req: RunRequest,
@@ -2317,13 +2393,13 @@ function resolveSheetContractFromReq(
 }
 
 /**
- * Direct-Neon writes from the Workers harness. Resolves the org runtime
- * Postgres URL via `create_db_session` (cached by runtime-api per
- * runId+namespace), then runs the same mega-CTE SQL Daytona uses — but over
- * `@neondatabase/serverless` so it runs inside the V8 isolate.
+ * Direct-Neon writes from the shared harness Worker. Resolves the org runtime
+ * Postgres URL via `create_db_session` inside that long-lived Worker, then
+ * runs the same row persistence path Daytona uses without bundling Neon into
+ * every per-play dynamic Worker.
  *
- * No HTTP fallback: if the contract/session can't be obtained, we want a
- * loud failure rather than silently regressing to per-chunk Vercel hops.
+ * If the contract/session can't be obtained, fail loudly rather than silently
+ * regressing to per-chunk Vercel hops.
  */
 function requireSheetContract(
   req: RunRequest,
@@ -2348,12 +2424,9 @@ async function persistCompletedMapRows(input: {
   extraOutputFields?: string[];
 }): Promise<void> {
   if (input.rows.length === 0) return;
-  await postRuntimeApi<{
-    ok: true;
-    rowsWritten: number;
-    tableNamespace: string;
-  }>(input.req.baseUrl, input.req.executorToken, {
-    action: 'persist_completed_sheet_rows',
+  await harnessPersistCompletedSheetRows({
+    baseUrl: input.req.baseUrl,
+    executorToken: input.req.executorToken,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
     sheetContract: requireSheetContract(input.req, input.tableNamespace),
@@ -2365,6 +2438,7 @@ async function persistCompletedMapRows(input: {
       ),
     ],
     runId: input.req.runId,
+    userEmail: input.req.userEmail,
   });
 }
 
@@ -2381,19 +2455,15 @@ async function prepareMapRows(input: {
   if (input.rows.length === 0) {
     return { inserted: 0, skipped: 0, pendingRows: [], completedRows: [] };
   }
-  const result = await postRuntimeApi<{
-    inserted: number;
-    skipped: number;
-    pendingRows: Record<string, unknown>[];
-    completedRows: Record<string, unknown>[];
-    tableNamespace: string;
-  }>(input.req.baseUrl, input.req.executorToken, {
-    action: 'start_sheet_dataset',
+  const result = await harnessStartSheetDataset({
+    baseUrl: input.req.baseUrl,
+    executorToken: input.req.executorToken,
     playName: input.req.playName,
     tableNamespace: input.tableNamespace,
     sheetContract: requireSheetContract(input.req, input.tableNamespace),
     rows: input.rows.map((row) => ({ ...row })),
     runId: input.req.runId,
+    userEmail: input.req.userEmail,
   });
   return {
     inserted: result.inserted,
@@ -2461,6 +2531,18 @@ function childPipelineUsesCtxMap(
   );
 }
 
+function childPipelineNeedsWorkflowScheduler(
+  pipeline: PlayStaticPipeline | null | undefined,
+): boolean {
+  if (!pipeline) return false;
+  return flattenStaticPipeline(pipeline).some(
+    (substep) =>
+      substep.type === 'tool' &&
+      (substep.isEventWait === true ||
+        substep.toolId === 'test_wait_for_event'),
+  );
+}
+
 function releaseChildPlayConcurrency(
   inFlightByPlayName: Record<string, number>,
   playName: string,
@@ -2483,6 +2565,41 @@ function createMinimalWorkerCtx(
   let playCallCount = 0;
   const parentChildCalls: Record<string, number> = {};
   const inFlightChildCallsByPlayName: Record<string, number> = {};
+  let inFlightChildPlayCalls = 0;
+  const childPlaySlotWaiters: Array<() => void> = [];
+
+  const acquireChildPlaySlot = async (): Promise<() => void> => {
+    while (
+      inFlightChildPlayCalls >= WORKER_PLAY_CALL_LIMITS.maxConcurrentPlayCalls
+    ) {
+      await new Promise<void>((resolve, reject) => {
+        const waiter = () => {
+          abortSignal?.removeEventListener('abort', onAbort);
+          resolve();
+        };
+        const onAbort = () => {
+          const index = childPlaySlotWaiters.indexOf(waiter);
+          if (index >= 0) childPlaySlotWaiters.splice(index, 1);
+          reject(
+            abortSignal?.reason instanceof Error
+              ? abortSignal.reason
+              : new WorkflowAbortError(),
+          );
+        };
+        childPlaySlotWaiters.push(waiter);
+        abortSignal?.addEventListener('abort', onAbort, { once: true });
+      });
+      assertNotAborted(abortSignal);
+    }
+    inFlightChildPlayCalls += 1;
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      inFlightChildPlayCalls = Math.max(0, inFlightChildPlayCalls - 1);
+      childPlaySlotWaiters.shift()?.();
+    };
+  };
   const rootGovernance = req.playCallGovernance;
   const rootRunId = rootGovernance?.rootRunId ?? req.runId;
   // Local ancestry chain that always ENDS with the currently-executing play
@@ -2517,6 +2634,7 @@ function createMinimalWorkerCtx(
     >,
     opts?: WorkerMapOptions,
   ): Promise<unknown> => {
+    const mapStartedAt = nowMs();
     const sliced = rows;
     const baseOffset = 0;
     const fieldEntries = Object.entries(fieldsDef);
@@ -2579,6 +2697,7 @@ function createMinimalWorkerCtx(
           : JSON.stringify(normalizedParts);
       return keyValue;
     };
+    const mapLogicFingerprint = req.graphHash ?? null;
     const resolveRowKey = (
       row: Record<string, unknown>,
       index: number,
@@ -2586,8 +2705,12 @@ function createMinimalWorkerCtx(
       const inputRow = publicCsvInputRow(row);
       const explicitKeyValue = resolveExplicitKeyValue(row, index);
       return explicitKeyValue == null
-        ? derivePlayRowIdentity(inputRow, name)
-        : derivePlayRowIdentityFromKey(explicitKeyValue, name);
+        ? derivePlayRowIdentity(inputRow, name, mapLogicFingerprint)
+        : derivePlayRowIdentityFromKey(
+            explicitKeyValue,
+            name,
+            mapLogicFingerprint,
+          );
     };
     const assertUniqueExplicitRowKeys = (
       chunkRows: readonly Record<string, unknown>[],
@@ -2614,12 +2737,21 @@ function createMinimalWorkerCtx(
       chunkStart: number,
       chunkIndex: number,
     ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
+      const chunkStartedAt = nowMs();
       assertNotAborted(abortSignal);
+      const keyStartedAt = nowMs();
       const chunkEntries = chunkRows.map((row, localIndex) => {
         const absoluteIndex = baseOffset + chunkStart + localIndex;
         const rowKey = resolveRowKey(row, absoluteIndex);
         return { row, absoluteIndex, rowKey };
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.keys',
+        ms: nowMs() - keyStartedAt,
+        extra: { mapName: name, chunkIndex, rows: chunkRows.length },
+      });
+      const prepareStartedAt = nowMs();
       const prepared = await prepareMapRows({
         req,
         tableNamespace: name,
@@ -2628,6 +2760,20 @@ function createMinimalWorkerCtx(
           __deeplineRowKey: rowKey,
         })),
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.prepare_rows',
+        ms: nowMs() - prepareStartedAt,
+        extra: {
+          mapName: name,
+          chunkIndex,
+          rows: chunkRows.length,
+          inserted: prepared.inserted,
+          skipped: prepared.skipped,
+          pendingRows: prepared.pendingRows.length,
+          completedRows: prepared.completedRows.length,
+        },
+      });
       const pendingKeys = new Set<string>();
       const completedKeys = new Set<string>();
       const preparedKeys = new Set<string>();
@@ -2635,7 +2781,11 @@ function createMinimalWorkerCtx(
         const key =
           typeof row.__deeplineRowKey === 'string'
             ? row.__deeplineRowKey
-            : derivePlayRowIdentity(publicCsvInputRow(row), name);
+            : derivePlayRowIdentity(
+                publicCsvInputRow(row),
+                name,
+                mapLogicFingerprint,
+              );
         if (key) {
           pendingKeys.add(key);
           preparedKeys.add(key);
@@ -2645,7 +2795,11 @@ function createMinimalWorkerCtx(
         const key =
           typeof row.__deeplineRowKey === 'string'
             ? row.__deeplineRowKey
-            : derivePlayRowIdentity(publicCsvInputRow(row), name);
+            : derivePlayRowIdentity(
+                publicCsvInputRow(row),
+                name,
+                mapLogicFingerprint,
+              );
         if (key) {
           completedKeys.add(key);
           preparedKeys.add(key);
@@ -2838,8 +2992,56 @@ function createMinimalWorkerCtx(
           })),
         });
       };
+      const workersStartedAt = nowMs();
       const workerResults = await Promise.allSettled(workers);
-      await persistExecutedRows();
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.execute_workers',
+        ms: nowMs() - workersStartedAt,
+        extra: {
+          mapName: name,
+          chunkIndex,
+          rowsToExecute: rowsToExecute.length,
+          concurrency,
+        },
+      });
+      const persistRowsStartedAt = nowMs();
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.persist_rows_start',
+        ms: 0,
+        extra: {
+          mapName: name,
+          chunkIndex,
+          rowsToExecute: rowsToExecute.length,
+        },
+      });
+      try {
+        await persistExecutedRows();
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.map_chunk.persist_rows',
+          ms: nowMs() - persistRowsStartedAt,
+          extra: {
+            mapName: name,
+            chunkIndex,
+            rowsToExecute: rowsToExecute.length,
+          },
+        });
+      } catch (error) {
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.map_chunk.persist_rows_error',
+          ms: nowMs() - persistRowsStartedAt,
+          extra: {
+            mapName: name,
+            chunkIndex,
+            rowsToExecute: rowsToExecute.length,
+            error: error instanceof Error ? error.message : String(error),
+          },
+        });
+        throw error;
+      }
       const rejectedWorker = workerResults.find(
         (result): result is PromiseRejectedResult =>
           result.status === 'rejected',
@@ -2852,7 +3054,11 @@ function createMinimalWorkerCtx(
         const key =
           typeof completedRow.__deeplineRowKey === 'string'
             ? completedRow.__deeplineRowKey
-            : derivePlayRowIdentity(publicCsvInputRow(completedRow), name);
+            : derivePlayRowIdentity(
+                publicCsvInputRow(completedRow),
+                name,
+                mapLogicFingerprint,
+              );
         if (key) {
           const { __deeplineRowKey: _rowKey, ...cleanedRow } =
             publicCsvInputRow(completedRow);
@@ -2875,6 +3081,27 @@ function createMinimalWorkerCtx(
           return resultByKey.get(key);
         })
         .filter((row): row is T & Record<string, unknown> => Boolean(row));
+      const hashStartedAt = nowMs();
+      const hash = await hashJson(out);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.hash',
+        ms: nowMs() - hashStartedAt,
+        extra: { mapName: name, chunkIndex, rows: out.length },
+      });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map_chunk.total',
+        ms: nowMs() - chunkStartedAt,
+        extra: {
+          mapName: name,
+          chunkIndex,
+          rowsRead: chunkRows.length,
+          rowsWritten: out.length,
+          rowsExecuted: executedRows.length,
+          rowsCached: Math.max(0, out.length - executedRows.length),
+        },
+      });
       return {
         chunkIndex,
         rangeStart: baseOffset + chunkStart,
@@ -2887,7 +3114,7 @@ function createMinimalWorkerCtx(
         rowsInserted,
         rowsSkipped,
         outputDatasetId: `map:${name}`,
-        hash: await hashJson(out),
+        hash,
         preview: toWorkflowSerializableValue(out.slice(0, 5)),
       };
     };
@@ -2911,7 +3138,7 @@ function createMinimalWorkerCtx(
         workflowStep.do as unknown as (
           name: string,
           config: {
-            retries: { limit: number; delay: string; backoff: 'exponential' };
+            retries: { limit: number; delay: number; backoff: 'exponential' };
           },
           callback: () => Promise<
             WorkerMapChunkSummary<T & Record<string, unknown>>
@@ -2919,7 +3146,7 @@ function createMinimalWorkerCtx(
         ) => Promise<WorkerMapChunkSummary<T & Record<string, unknown>>>
       )(
         deterministicMapChunkStepName({ mapName: name, chunkIndex }),
-        { retries: { limit: 5, delay: '5 seconds', backoff: 'exponential' } },
+        { retries: { limit: 5, delay: 100, backoff: 'exponential' } },
         async () => await processChunk(chunkRows, chunkStart, chunkIndex),
       )) as WorkerMapChunkSummary<T & Record<string, unknown>>;
     };
@@ -2978,7 +3205,14 @@ function createMinimalWorkerCtx(
         chunkStart += chunkRows.length;
         chunkIndex += 1;
       }
-      return finalize(totalRowsWritten);
+      const dataset = finalize(totalRowsWritten);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map.total',
+        ms: nowMs() - mapStartedAt,
+        extra: { mapName: name, rowsWritten: totalRowsWritten, streaming: true },
+      });
+      return dataset;
     }
 
     if (workflowStep && sliced.length > rowsPerChunk) {
@@ -3000,7 +3234,14 @@ function createMinimalWorkerCtx(
           out.push(...chunkResult.preview.slice(0, 10 - out.length));
         }
       }
-      return finalize(totalRowsWritten);
+      const dataset = finalize(totalRowsWritten);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.map.total',
+        ms: nowMs() - mapStartedAt,
+        extra: { mapName: name, rowsWritten: totalRowsWritten, streaming: false },
+      });
+      return dataset;
     }
 
     assertUniqueExplicitRowKeys(sliced, 0);
@@ -3011,7 +3252,18 @@ function createMinimalWorkerCtx(
     totalRowsInserted = chunkResult.rowsInserted;
     totalRowsSkipped = chunkResult.rowsSkipped;
     out.push(...chunkResult.preview);
-    return finalize(chunkResult.rowsWritten);
+    const dataset = finalize(chunkResult.rowsWritten);
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.map.total',
+      ms: nowMs() - mapStartedAt,
+      extra: {
+        mapName: name,
+        rowsWritten: chunkResult.rowsWritten,
+        streaming: false,
+      },
+    });
+    return dataset;
   };
 
   class WorkerMapBuilder<T extends Record<string, unknown>> {
@@ -3098,28 +3350,43 @@ function createMinimalWorkerCtx(
       arg: unknown,
       options?: CsvRenameOptions,
     ): Promise<T[]> {
+      const csvStartedAt = nowMs();
       if (Array.isArray(arg)) {
         // Inline rows passed at call site — already in memory, keep the
         // legacy array-backed dataset shape.
-        return makeWorkerDataset(
+        const dataset = makeWorkerDataset(
           'csv',
           applyCsvRenameProjection(arg as T[], options),
           {
             datasetKind: 'csv',
           },
         ) as unknown as T[];
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.csv',
+          ms: nowMs() - csvStartedAt,
+          extra: { mode: 'inline_array', rows: arg.length },
+        });
+        return dataset;
       }
       const filename = String(arg ?? '');
       if (req.inlineCsv && filename === req.inlineCsv.name) {
         // Inline CSV pre-staged by the dispatcher (small files <1 MiB). Already
         // in memory; no streaming needed.
-        return makeWorkerDataset(
+        const dataset = makeWorkerDataset(
           'csv',
           applyCsvRenameProjection(req.inlineCsv.rows as T[], options),
           {
             datasetKind: 'csv',
           },
         ) as unknown as T[];
+        recordRunnerPerfTrace({
+          req,
+          phase: 'runner.csv',
+          ms: nowMs() - csvStartedAt,
+          extra: { mode: 'inline_csv', rows: req.inlineCsv.rows.length },
+        });
+        return dataset;
       }
       // Resolution order: explicit inputR2Keys (runtime input) → packaged
       // files (relative-path imports bundled with the play artifact).
@@ -3144,7 +3411,7 @@ function createMinimalWorkerCtx(
       // and switches its chunked execution loop to consume iterChunks
       // directly, so 2M-row CSVs never get fully materialized in memory.
       const storageKey = r2Key;
-      return makeStreamingCsvDataset<T>({
+      const dataset = makeStreamingCsvDataset<T>({
         name: filename,
         logicalPath: filename,
         renameOptions: options,
@@ -3156,6 +3423,13 @@ function createMinimalWorkerCtx(
             storageKey,
           }),
       }) as unknown as T[];
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.csv',
+        ms: nowMs() - csvStartedAt,
+        extra: { mode: 'streaming_r2', filename },
+      });
+      return dataset;
     },
     map<T extends Record<string, unknown>>(
       name: string,
@@ -3186,320 +3460,6 @@ function createMinimalWorkerCtx(
       throw new Error(
         'ctx.map(key, rows, fields, options) was removed. Use ctx.map(key, rows).step(...).run(options).',
       );
-      /*
-      const sliced = rows;
-      const baseOffset = 0;
-      const fieldEntries = Object.entries(fieldsDef);
-      const plan = req.executionPlan;
-      const planMap = plan?.maps.find(
-        (candidate) =>
-          candidate.mapName === name || candidate.tableNamespace === name,
-      );
-      const streaming = isStreamingDataset<T>(sliced);
-      // For streaming inputs we don't know the row count upfront — pass
-      // `totalRows: 0` so chooseMapChunkSize falls back to the preferred /
-      // default chunk size rather than trying to budget against an unknown.
-      const rowsPerChunk = chooseMapChunkSize({
-        totalRows: streaming ? 0 : sliced.length,
-        mapCount: Math.max(1, plan?.maps.length ?? 1),
-        stepsPerChunk: planMap?.stepsPerChunk ?? 1,
-        preferredChunkSize: planMap?.defaultChunkSize,
-        softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
-      });
-      const outputFields = fieldEntries.map(([field]) => field);
-
-      // Workflow steps have bounded CPU but unbounded wall time, so long
-      // network calls are OK here as long as we checkpoint by chunk and do not
-      // collapse a giant map into one monolithic step result.
-
-      const processChunk = async (
-        chunkRows: T[],
-        chunkStart: number,
-        chunkIndex: number,
-      ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
-        assertNotAborted(abortSignal);
-        const prepared = await prepareMapRows({
-          req,
-          tableNamespace: name,
-          rows: chunkRows,
-        });
-        const pendingKeys = new Set<string>();
-        const completedKeys = new Set<string>();
-        const preparedKeys = new Set<string>();
-        for (const row of prepared.pendingRows) {
-          const key = derivePlayRowIdentity(publicCsvInputRow(row), name);
-          if (key) {
-            pendingKeys.add(key);
-            preparedKeys.add(key);
-          }
-        }
-        for (const row of prepared.completedRows) {
-          const key =
-            typeof row.__deeplineRowKey === 'string'
-              ? row.__deeplineRowKey
-              : derivePlayRowIdentity(publicCsvInputRow(row), name);
-          if (key) {
-            completedKeys.add(key);
-            preparedKeys.add(key);
-          }
-        }
-        const missingPreparedRows = chunkRows.filter((row) => {
-          const key = derivePlayRowIdentity(publicCsvInputRow(row), name);
-          return !key || !preparedKeys.has(key);
-        });
-        const rowsToExecute = chunkRows.filter((row) => {
-          const key = derivePlayRowIdentity(publicCsvInputRow(row), name);
-          return !key || pendingKeys.has(key) || !completedKeys.has(key);
-        });
-        const rowsInserted = prepared.inserted + missingPreparedRows.length;
-        const rowsSkipped = Math.max(
-          0,
-          prepared.skipped - missingPreparedRows.length,
-        );
-        const concurrency = Math.max(1, Math.min(opts?.concurrency ?? 10, 100));
-        const executedRows: Array<T & Record<string, unknown>> = new Array(
-          rowsToExecute.length,
-        );
-        const generatedOutputFields = new Set<string>();
-        let idx = 0;
-        const workers: Array<Promise<void>> = [];
-        for (let w = 0; w < concurrency; w += 1) {
-          workers.push(
-            (async () => {
-              while (true) {
-                if (abortSignal?.aborted) return;
-                const myIndex = idx++;
-                if (myIndex >= rowsToExecute.length) return;
-                const row = rowsToExecute[myIndex]!;
-                const absoluteIndex = baseOffset + chunkStart + myIndex;
-                const enriched: Record<string, unknown> = cloneCsvAliasedRow(row);
-                const fieldOutputs: Record<string, unknown> = {};
-                const waterfallOutputs: RecordedWaterfallOutput[] = [];
-                const rowCtx = {
-                  ...(ctx as Record<string, unknown>),
-                  waterfall: (
-                    toolNameOrSpec: string | WorkerInlineWaterfallSpec,
-                    waterfallInput: Record<string, unknown>,
-                    waterfallOpts?: WorkerWaterfallOptions,
-                  ) =>
-                    executeWorkerWaterfall(
-                      req,
-                      waterfallOutputs,
-                      toolNameOrSpec,
-                      waterfallInput,
-                      waterfallOpts,
-                    ),
-                };
-                for (const [key, value] of fieldEntries) {
-                  if (typeof value === 'function') {
-                    const resolved = await (
-                      value as (
-                        r: T,
-                        c: unknown,
-                        f: Record<string, unknown>,
-                        i: number,
-                      ) => Promise<unknown> | unknown
-                    )(row, rowCtx, fieldOutputs, absoluteIndex);
-                    enriched[key] = resolved;
-                    fieldOutputs[key] = resolved;
-                  } else {
-                    enriched[key] = value;
-                    fieldOutputs[key] = value;
-                  }
-                }
-                for (const waterfallOutput of waterfallOutputs) {
-                  const columnName =
-                    `${sqlishIdentifierPart(waterfallOutput.waterfallId)}__` +
-                    sqlishIdentifierPart(waterfallOutput.stepId);
-                  enriched[columnName] = waterfallOutput.value;
-                  generatedOutputFields.add(columnName);
-                }
-                executedRows[myIndex] = enriched as T & Record<string, unknown>;
-              }
-            })(),
-          );
-        }
-        await Promise.all(workers);
-        if (executedRows.length > 0) {
-          await persistCompletedMapRows({
-            req,
-            tableNamespace: name,
-            outputFields,
-            extraOutputFields: Array.from(generatedOutputFields),
-            rows: executedRows.map((row, executedIndex) => ({
-              ...row,
-              __deeplineRowKey: derivePlayRowIdentity(
-                publicCsvInputRow(rowsToExecute[executedIndex]!),
-                name,
-              ),
-            })),
-          });
-        }
-        const resultByKey = new Map<string, T & Record<string, unknown>>();
-        for (const completedRow of prepared.completedRows) {
-          const key =
-            typeof completedRow.__deeplineRowKey === 'string'
-              ? completedRow.__deeplineRowKey
-              : derivePlayRowIdentity(publicCsvInputRow(completedRow), name);
-          if (key) {
-            const { __deeplineRowKey: _rowKey, ...cleanedRow } =
-              publicCsvInputRow(completedRow);
-            void _rowKey;
-            resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
-          }
-        }
-        for (
-          let executedIndex = 0;
-          executedIndex < executedRows.length;
-          executedIndex += 1
-        ) {
-          const executedRow = executedRows[executedIndex]!;
-          const key = derivePlayRowIdentity(
-            publicCsvInputRow(rowsToExecute[executedIndex]!),
-            name,
-          );
-          if (key) resultByKey.set(key, executedRow);
-        }
-        const out = chunkRows
-          .map((row) => {
-            const key = derivePlayRowIdentity(publicCsvInputRow(row), name);
-            return key ? resultByKey.get(key) : undefined;
-          })
-          .filter((row): row is T & Record<string, unknown> => Boolean(row));
-        return {
-          chunkIndex,
-          rangeStart: baseOffset + chunkStart,
-          rangeEnd: baseOffset + chunkStart + out.length,
-          rowsRead: chunkRows.length,
-          rowsWritten: out.length,
-          rowsExecuted: executedRows.length,
-          rowsCached: prepared.completedRows.length,
-          rowsInserted,
-          rowsSkipped,
-          outputDatasetId: `map:${name}`,
-          hash: await hashJson(out),
-          preview: toWorkflowSerializableValue(out.slice(0, 5)),
-        };
-      };
-
-      const out: Array<T & Record<string, unknown>> = [];
-      let totalRowsExecuted = 0;
-      let totalRowsCached = 0;
-      let totalRowsInserted = 0;
-      let totalRowsSkipped = 0;
-
-      const runChunkStep = async (
-        chunkRows: T[],
-        chunkStart: number,
-        chunkIndex: number,
-      ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
-        if (!workflowStep) {
-          return await processChunk(chunkRows, chunkStart, chunkIndex);
-        }
-        return (await (
-          workflowStep.do as unknown as (
-            name: string,
-            config: {
-              retries: { limit: number; delay: string; backoff: 'exponential' };
-            },
-            callback: () => Promise<
-              WorkerMapChunkSummary<T & Record<string, unknown>>
-            >,
-          ) => Promise<WorkerMapChunkSummary<T & Record<string, unknown>>>
-        )(
-          deterministicMapChunkStepName({ mapName: name, chunkIndex }),
-          { retries: { limit: 5, delay: '5 seconds', backoff: 'exponential' } },
-          async () => await processChunk(chunkRows, chunkStart, chunkIndex),
-        )) as WorkerMapChunkSummary<T & Record<string, unknown>>;
-      };
-
-      const finalize = (totalRowsWritten: number) => {
-        const cacheSummary =
-          `Map completed: ${totalRowsWritten} results ` +
-          `(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
-          `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
-        emitEvent({
-          type: 'log',
-          level: 'info',
-          message: cacheSummary,
-          ts: nowMs(),
-        });
-        return makeWorkerDataset(name, out, {
-          count: totalRowsWritten,
-          cacheSummary,
-          workProgress: {
-            total: totalRowsWritten,
-            executed: totalRowsExecuted,
-            reused: totalRowsCached,
-            skipped: totalRowsCached,
-            pending: 0,
-            failed: 0,
-          },
-        });
-      };
-
-      // Streaming path: input came from `ctx.csv` over R2 and we never
-      // materialized the full row array. Pull row chunks from the dataset's
-      // iterChunks() and run each through processChunk inside its own
-      // workflow step. Memory stays bounded by `rowsPerChunk`.
-      if (streaming) {
-        let totalRowsWritten = 0;
-        let chunkIndex = 0;
-        let chunkStart = 0;
-        const streamingDataset = sliced as unknown as StreamingCsvDataset<T>;
-        for await (const chunkRows of streamingDataset.iterChunks(
-          rowsPerChunk,
-        )) {
-          assertNotAborted(abortSignal);
-          if (chunkRows.length === 0) continue;
-          const chunkResult = await runChunkStep(
-            chunkRows,
-            chunkStart,
-            chunkIndex,
-          );
-          totalRowsWritten += chunkResult.rowsWritten;
-          totalRowsExecuted += chunkResult.rowsExecuted;
-          totalRowsCached += chunkResult.rowsCached;
-          totalRowsInserted += chunkResult.rowsInserted;
-          totalRowsSkipped += chunkResult.rowsSkipped;
-          if (out.length < 10) {
-            out.push(...chunkResult.preview.slice(0, 10 - out.length));
-          }
-          chunkStart += chunkRows.length;
-          chunkIndex += 1;
-        }
-        return finalize(totalRowsWritten);
-      }
-
-      // Materialized array path (inline rows or small CSV).
-      if (workflowStep && sliced.length > rowsPerChunk) {
-        let totalRowsWritten = 0;
-        for (let start = 0; start < sliced.length; start += rowsPerChunk) {
-          assertNotAborted(abortSignal);
-          const end = Math.min(sliced.length, start + rowsPerChunk);
-          const chunkRows = sliced.slice(start, end);
-          const chunkIndex = Math.floor(start / rowsPerChunk);
-          const chunkResult = await runChunkStep(chunkRows, start, chunkIndex);
-          totalRowsWritten += chunkResult.rowsWritten;
-          totalRowsExecuted += chunkResult.rowsExecuted;
-          totalRowsCached += chunkResult.rowsCached;
-          totalRowsInserted += chunkResult.rowsInserted;
-          totalRowsSkipped += chunkResult.rowsSkipped;
-          if (out.length < 10) {
-            out.push(...chunkResult.preview.slice(0, 10 - out.length));
-          }
-        }
-        return finalize(totalRowsWritten);
-      }
-
-      const chunkResult = await processChunk(sliced, 0, 0);
-      totalRowsExecuted = chunkResult.rowsExecuted;
-      totalRowsCached = chunkResult.rowsCached;
-      totalRowsInserted = chunkResult.rowsInserted;
-      totalRowsSkipped = chunkResult.rowsSkipped;
-      out.push(...chunkResult.preview);
-      return finalize(chunkResult.rowsWritten);
-      */
     },
     tool: async (
       key: string,
@@ -3620,7 +3580,11 @@ function createMinimalWorkerCtx(
       const childIsMapBacked = childPipelineUsesCtxMap(
         childManifest.staticPipeline,
       );
+      const childNeedsWorkflowScheduler = childPipelineNeedsWorkflowScheduler(
+        childManifest.staticPipeline,
+      );
       let childConcurrencyAcquired = false;
+      let releaseChildPlaySlot: (() => void) | null = null;
       if (childIsMapBacked) {
         const nextInFlight =
           (inFlightChildCallsByPlayName[resolvedName] ?? 0) + 1;
@@ -3635,11 +3599,21 @@ function createMinimalWorkerCtx(
         childConcurrencyAcquired = true;
       }
       try {
+        releaseChildPlaySlot = await acquireChildPlaySlot();
         const childSubmitStartedAt = nowMs();
-        let started: { workflowId?: string; runId?: string; error?: unknown };
+        let started: {
+          workflowId?: string;
+          runId?: string;
+          status?: string;
+          output?: unknown;
+          result?: unknown;
+          error?: unknown;
+        };
         try {
           started = await submitChildPlayThroughCoordinator({
             req,
+            allowInline:
+              options?.timeoutMs == null && !childNeedsWorkflowScheduler,
             body: {
               name: resolvedName,
               input: isRecord(input) ? input : {},
@@ -3709,6 +3683,27 @@ function createMinimalWorkerCtx(
           ms: nowMs() - childSubmitStartedAt,
           status: 'ok',
         });
+        const startedStatus = String(started.status ?? '').toLowerCase();
+        if (startedStatus === 'completed') {
+          emitEvent({
+            type: 'log',
+            level: 'info',
+            message: `Completed child play ${resolvedName} (${normalizedKey})`,
+            ts: nowMs(),
+          });
+          return started.output ?? extractChildPlayOutput(started);
+        }
+        if (startedStatus === 'failed') {
+          const startedError = isRecord(started.error)
+            ? started.error
+            : { message: started.error };
+          const startedErrorMessage =
+            typeof startedError.message === 'string' &&
+            startedError.message.trim()
+              ? startedError.message.trim()
+              : `Child play ${resolvedName} (${workflowId}) failed.`;
+          throw new Error(startedErrorMessage);
+        }
         const childWaitStartedAt = nowMs();
         let result: unknown;
         try {
@@ -3761,6 +3756,7 @@ function createMinimalWorkerCtx(
         });
         return result;
       } finally {
+        releaseChildPlaySlot?.();
         if (childConcurrencyAcquired) {
           releaseChildPlayConcurrency(
             inFlightChildCallsByPlayName,
@@ -3883,6 +3879,7 @@ async function executeRunRequest(
   workflowStep?: WorkflowStep,
   options?: {
     persistResultDatasets?: boolean;
+    waitUntil?: (promise: Promise<unknown>) => void;
     /**
      * Cooperative cancellation token. CF Workflows surfaces termination as a
      * thrown error from any in-progress step; the harness catches that, flips
@@ -3893,8 +3890,29 @@ async function executeRunRequest(
 ): Promise<WorkflowRunOutput> {
   installProcessExitTrap();
   const startedAt = nowMs();
+  recordRunnerPerfTrace({
+    req,
+    phase: 'runner.execute_start',
+    extra: {
+      persistResultDatasets: Boolean(options?.persistResultDatasets),
+      hasWorkflowStep: Boolean(workflowStep),
+    },
+  });
   const abortController = options?.abortController ?? new AbortController();
   const abortSignal = abortController.signal;
+  const postgresPrewarmStartedAt = nowMs();
+  await harnessPrewarmPostgresSessions({
+    executorToken: req.executorToken,
+    sessions: req.preloadedDbSessions ?? [],
+  });
+  recordRunnerPerfTrace({
+    req,
+    phase: 'runner.prewarm_postgres',
+    ms: nowMs() - postgresPrewarmStartedAt,
+    extra: {
+      sessions: req.preloadedDbSessions?.length ?? 0,
+    },
+  });
   // Maintain a rolling buffer of log lines emitted during the run. This is
   // what the play-page UI consumes via Convex polling + diffPlayRunStreamEvents
   // → play.run.log SSE events. Without periodic flushing, the play page only
@@ -3959,17 +3977,42 @@ async function executeRunRequest(
     abortSignal,
   );
   try {
+    const playStartedAt = nowMs();
     const result = await (
       playFn as (
         ctx: unknown,
         input: Record<string, unknown>,
       ) => Promise<unknown>
     )(ctx, req.runtimeInput);
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.play_function',
+      ms: nowMs() - playStartedAt,
+    });
+    const serializeStartedAt = nowMs();
     const serializedResult = serializePlayReturnValue(result);
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.serialize_result',
+      ms: nowMs() - serializeStartedAt,
+    });
     if (options?.persistResultDatasets) {
+      const persistStartedAt = nowMs();
       await liveLogFlushInFlight.catch(() => undefined);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.live_log_flush_wait',
+        ms: nowMs() - persistStartedAt,
+      });
+      const resultDatasetStartedAt = nowMs();
       await persistResultDatasets(req, serializedResult);
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.persist_result_datasets',
+        ms: nowMs() - resultDatasetStartedAt,
+      });
       const terminalResult = trimResultForStatus(serializedResult);
+      const terminalUpdateStartedAt = nowMs();
       await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
         action: 'update_run_status',
         playId: req.runId,
@@ -3982,12 +4025,24 @@ async function executeRunRequest(
         liveLogs,
         lastCheckpointAt: nowMs(),
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.terminal_status_update',
+        ms: nowMs() - terminalUpdateStartedAt,
+      });
+      const billingStartedAt = nowMs();
       await finalizeWorkerComputeBilling({
         req,
         success: true,
         actionEstimate: 4,
       });
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.compute_billing_finalize',
+        ms: nowMs() - billingStartedAt,
+      });
     }
+    const parentSignalStartedAt = nowMs();
     await signalParentPlayTerminal({
       req,
       status: 'completed',
@@ -3999,6 +4054,16 @@ async function executeRunRequest(
         }`,
       );
     });
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.parent_terminal_signal',
+      ms: nowMs() - parentSignalStartedAt,
+    });
+    recordRunnerPerfTrace({
+      req,
+      phase: 'runner.execute_total',
+      ms: nowMs() - startedAt,
+    });
     return {
       playName: req.playName,
       result: serializedResult,
@@ -4156,6 +4221,9 @@ function runRequestFromWorkflowParams(
     playCallGovernance: isPlayCallGovernanceSnapshot(params.playCallGovernance)
       ? params.playCallGovernance
       : null,
+    preloadedDbSessions: Array.isArray(params.preloadedDbSessions)
+      ? (params.preloadedDbSessions as PreloadedRuntimeDbSession[])
+      : null,
     coordinatorUrl:
       typeof params.coordinatorUrl === 'string' && params.coordinatorUrl.trim()
         ? params.coordinatorUrl.trim()
@@ -4396,6 +4464,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
   Record<string, unknown>
 > {
   declare readonly env: WorkerEnv;
+  declare readonly ctx: ExecutionContext;
 
   /**
    * Cancellation model:
@@ -4439,14 +4508,30 @@ export class TenantWorkflow extends WorkflowEntrypoint<
         (runnerEvent) => {
           if (runnerEvent.type === 'log') {
             console.log(`${runPrefix} ${runnerEvent.message}`);
+            void this.env.COORDINATOR?.recordRunEvent(req.runId, {
+              runId: req.runId,
+              type: 'log',
+              line: runnerEvent.message,
+              ts: runnerEvent.ts,
+            }).catch(() => undefined);
           } else if (runnerEvent.type === 'error') {
             console.error(
               `${runPrefix} ${runnerEvent.message}${runnerEvent.stack ? `\n${runnerEvent.stack}` : ''}`,
             );
+            void this.env.COORDINATOR?.recordRunEvent(req.runId, {
+              runId: req.runId,
+              type: 'log',
+              line: `[error] ${runnerEvent.message}`,
+              ts: runnerEvent.ts,
+            }).catch(() => undefined);
           }
         },
         step,
-        { persistResultDatasets: !req.playCallGovernance, abortController },
+        {
+          persistResultDatasets: !req.playCallGovernance,
+          abortController,
+          waitUntil: (promise) => this.ctx.waitUntil(promise),
+        },
       )) as Record<string, unknown>;
     } catch (error) {
       // CF Workflows + the dynamic-workflows framework swallow the error