deepline 0.1.108 → 0.1.110

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -42,7 +42,7 @@ import {
   executeChunkedRequests,
   type ChunkExecutionResult,
 } from '../../../shared_libs/play-runtime/batch-runtime';
-import { getDefaultPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/default-batch-strategies';
+import { getPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/play-runtime-batching-registry';
 import { STANDARD_PLAY_RUNTIME_LIMIT_SECONDS } from '../../../shared_libs/temporal/constants';
 import {
   createPlayExecutionGovernor,
@@ -59,6 +59,7 @@ import {
   type ChildPlayTerminalWaitResult,
   type WorkflowStepLike,
 } from './child-play-await';
+import { submitChildPlayThroughCoordinator } from './child-play-submit';
 import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
 import {
   parseToolExecuteResponse,
@@ -84,11 +85,12 @@ import {
   getCompiledPipelineSubsteps,
   flattenStaticPipeline,
   resolveSheetContractForTableNamespace,
-  sqlSafePlayColumnName,
   type PlayStaticSubstep,
   type PlayStaticPipeline,
   type PlaySheetContract,
 } from '../../../shared_libs/plays/static-pipeline';
+import { sqlSafePlayColumnName } from '../../../shared_libs/play-data-plane/column-names';
+import { augmentSheetContractWithDatasetFields } from '../../../shared_libs/play-data-plane/sheet-contract';
 import {
   PlayStepLifecycleTracker,
   type PlayStepLifecycleEvent,
@@ -131,10 +133,16 @@ import {
   setHarnessBinding,
 } from '../../../sdk/src/plays/harness-stub';
 import { createHarnessWorkerReceiptStore } from './runtime/harness-receipt-store';
+import {
+  runtimeCsvExecutionRow,
+  publicCsvInputRow,
+  publicCsvOutputRow,
+  publicCsvStorageRow,
+  runtimeCsvStorageRow,
+} from './runtime/csv-rows';
 import { chooseWorkerMapRowsPerChunk } from './runtime/map-chunk-plan';
 import {
   applyCsvRenameProjection,
-  stripCsvProjectionMetadata,
   cloneCsvAliasedRow,
   type CsvRenameOptions,
 } from '../../../shared_libs/play-runtime/csv-rename';
@@ -153,6 +161,12 @@ import {
   type SecretAwareRequestInit,
   type SecretHandle,
 } from '../../../shared_libs/play-runtime/secret-capability';
+import { safePublicFetch } from '../../../shared_libs/security/safe-fetch';
+import {
+  assertPublicHttpUrl,
+  isIpAddressLiteral,
+  UnsafeOutboundUrlError,
+} from '../../../shared_libs/security/outbound-url-policy';
 import type {
   LiveNodeProgressMap,
   LiveNodeProgressSnapshot,
@@ -174,7 +188,6 @@ import {
 } from '../../../shared_libs/play-runtime/step-program-dataset-builder';
 import {
   DEEPLINE_CELL_META_FIELD,
-  normalizeCellStalenessPolicy,
   previousCellFromValue,
   resolveCompletedCellStalenessMeta,
   shouldRecomputeCell,
@@ -183,6 +196,10 @@ import {
   type CellStalenessPolicyByField,
   type PreviousCell,
 } from '../../../shared_libs/play-runtime/cell-staleness';
+import {
+  authoredCellPoliciesFromDatasetSteps,
+  cellPoliciesFromDatasetSteps,
+} from '../../../shared_libs/play-data-plane/cell-policy';
 
 // The play's default export. The bundler injects this — see bundle-play-file.ts.
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -701,57 +718,6 @@ function makeRequestId(): string {
   return crypto.randomUUID();
 }
 
-function publicCsvInputRow<T extends Record<string, unknown>>(row: T): T {
-  const restored = stripCsvProjectionMetadata(row) as Record<string, unknown>;
-  const publicRow: Record<string, unknown> = {};
-  for (const fieldName of Reflect.ownKeys(restored)) {
-    if (typeof fieldName === 'string' && fieldName.startsWith('__deepline')) {
-      continue;
-    }
-    const descriptor = Object.getOwnPropertyDescriptor(restored, fieldName);
-    if (!descriptor) continue;
-    Object.defineProperty(publicRow, fieldName, descriptor);
-  }
-  return publicRow as T;
-}
-
-function publicCsvOutputRow<T extends Record<string, unknown>>(row: T): T {
-  const stripped = stripCsvProjectionMetadata(row) as Record<string, unknown>;
-  const publicRow: Record<string, unknown> = {};
-  for (const fieldName of Reflect.ownKeys(stripped)) {
-    if (typeof fieldName === 'string' && fieldName.startsWith('__deepline')) {
-      continue;
-    }
-    const descriptor = Object.getOwnPropertyDescriptor(stripped, fieldName);
-    if (!descriptor) continue;
-    Object.defineProperty(publicRow, fieldName, descriptor);
-  }
-  return publicRow as T;
-}
-
-function publicCsvStorageRow<T extends Record<string, unknown>>(row: T): T {
-  const publicRow = publicCsvInputRow(row) as Record<string, unknown>;
-  const storageRow: Record<string, unknown> = {};
-  for (const fieldName of Reflect.ownKeys(publicRow)) {
-    if (typeof fieldName !== 'string') continue;
-    const descriptor = Object.getOwnPropertyDescriptor(publicRow, fieldName);
-    if (!descriptor) continue;
-    storageRow[fieldName] =
-      'value' in descriptor ? descriptor.value : publicRow[fieldName];
-  }
-  for (const runtimeField of [
-    '__deeplineRowKey',
-    '__deeplineCellMetaPatch',
-    '__deeplineRowStatus',
-    '__deeplineRowError',
-  ]) {
-    if (Object.prototype.hasOwnProperty.call(row, runtimeField)) {
-      storageRow[runtimeField] = row[runtimeField];
-    }
-  }
-  return storageRow as T;
-}
-
 /**
  * Strip credentials and JWT-shaped tokens from any string before it lands in
  * a log buffer or upstream error message. The harness routinely echoes
@@ -893,90 +859,6 @@ async function postRuntimeApiBestEffort(
   }
 }
 
-async function submitChildPlayThroughCoordinator(input: {
-  req: RunRequest;
-  body: unknown;
-  allowInline?: boolean;
-}): Promise<{
-  workflowId?: string;
-  runId?: string;
-  status?: string;
-  mode?: string;
-  output?: unknown;
-  result?: unknown;
-  error?: unknown;
-  logs?: string[];
-  timings?: Array<{ phase: string; ms: number }>;
-}> {
-  if (cachedCoordinatorBinding && input.allowInline !== false) {
-    if (!isRecord(input.body)) {
-      throw new Error('ctx.runPlay child submit requires an object body.');
-    }
-    return cachedCoordinatorBinding.submitChild(input.req.runId, input.body);
-  }
-  if (cachedCoordinatorBinding?.submitWorkflowChild) {
-    if (!isRecord(input.body)) {
-      throw new Error('ctx.runPlay child submit requires an object body.');
-    }
-    return cachedCoordinatorBinding.submitWorkflowChild(
-      input.req.runId,
-      input.body,
-    );
-  }
-  const coordinatorUrl = input.req.coordinatorUrl?.trim();
-  if (coordinatorUrl) {
-    // Keep child plays on the same coordinator/Workflow submit path as
-    // top-level runs when the coordinator URL is present.
-    const res = await fetch(
-      `${coordinatorUrl.replace(/\/$/, '')}/workflow/${encodeURIComponent(
-        input.req.runId,
-      )}/submit-child`,
-      {
-        method: 'POST',
-        headers: {
-          'x-deepline-request-id': makeRequestId(),
-          ...coordinatorRequestHeaders({
-            runId: input.req.runId,
-            contentType: 'application/json',
-            internalToken: input.req.coordinatorInternalToken,
-          }),
-        },
-        body: JSON.stringify(input.body),
-      },
-    );
-    const text = await res.text().catch(() => '');
-    let parsed: {
-      workflowId?: string;
-      runId?: string;
-      status?: string;
-      mode?: string;
-      output?: unknown;
-      result?: unknown;
-      error?: unknown;
-      logs?: string[];
-      timings?: Array<{ phase: string; ms: number }>;
-    } = {};
-    try {
-      parsed = text ? JSON.parse(text) : {};
-    } catch {
-      parsed = { error: text };
-    }
-    if (!res.ok) {
-      const error = isRecord(parsed.error) ? parsed.error : null;
-      const message =
-        (typeof error?.message === 'string' && error.message.trim()) ||
-        (typeof parsed.error === 'string' && parsed.error.trim()) ||
-        text.slice(0, 800) ||
-        `Coordinator child submit failed with ${res.status}.`;
-      throw new Error(message);
-    }
-    return parsed;
-  }
-  throw new Error(
-    'ctx.runPlay child submit requires a coordinator binding in the cf-workflows runtime.',
-  );
-}
-
 function workflowEventType(name: string): string {
   const normalized = name
     .trim()
@@ -1155,10 +1037,18 @@ async function executeToolWithLifecycle(
   args: { id: string; toolId: string; input: Record<string, unknown> },
   workflowStep: WorkflowStep | undefined,
   callbacks: WorkerCtxCallbacks | undefined,
+  onProviderBackpressure?: (retryAfterMs: number) => void,
+  onRetryAttempt?: () => void,
 ): Promise<ToolExecuteResult> {
   callbacks?.onToolCalled?.(args.toolId, nowMs());
   try {
-    return await executeTool(req, args, workflowStep);
+    return await executeTool(
+      req,
+      args,
+      workflowStep,
+      onProviderBackpressure,
+      onRetryAttempt,
+    );
   } catch (error) {
     callbacks?.onToolFailed?.(args.toolId, nowMs());
     throw error;
@@ -1302,17 +1192,38 @@ async function callToolDirect(
     attempt <= WORKER_TOOL_RATE_LIMIT_MAX_ATTEMPTS;
     attempt += 1
   ) {
-    const res = await fetchRuntimeApi(req.baseUrl, path, {
-      method: 'POST',
-      headers: {
-        'content-type': 'application/json',
-        authorization: `Bearer ${req.executorToken}`,
-        'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${attempt}`,
-        [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
-        [EXECUTE_TOOL_METADATA_HEADER]: 'true',
-      },
-      body: JSON.stringify({ payload: input }),
-    });
+    let res: Response;
+    try {
+      res = await fetchRuntimeApi(req.baseUrl, path, {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          authorization: `Bearer ${req.executorToken}`,
+          'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${attempt}`,
+          [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
+          [EXECUTE_TOOL_METADATA_HEADER]: 'true',
+        },
+        body: JSON.stringify({ payload: input }),
+      });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      lastError = new Error(
+        `Tool ${toolId} transport failed calling ${path} for run ${req.runId} on attempt ${attempt}/${WORKER_TOOL_TRANSPORT_MAX_ATTEMPTS}: ${message}`,
+      );
+      if (
+        attempt >= WORKER_TOOL_TRANSPORT_MAX_ATTEMPTS ||
+        !isRetryableRuntimeApiError(error)
+      ) {
+        throw lastError;
+      }
+      onRetryAttempt?.();
+      const delayMs = WORKER_TOOL_TRANSPORT_RETRY_DELAY_MS * attempt;
+      console.warn(
+        `[deepline-run:${req.runId}] tool transport retry tool=${toolId} path=${path} attempt=${attempt}/${WORKER_TOOL_TRANSPORT_MAX_ATTEMPTS} retryAfterMs=${delayMs} error=${redactSecretsFromLogString(message)}`,
+      );
+      await sleepWorkerMs(delayMs);
+      continue;
+    }
     if (res.ok) {
       const body = (await res.json()) as Record<string, unknown>;
       const parsed = parseToolExecuteResponse(toolId, body);
@@ -1547,6 +1458,8 @@ const WORKER_RETRY_SAFE_5XX_TOOLS = new Set(['test_transient_500']);
  * retry budget, so a runaway storm stays bounded and loud.
  */
 const WORKER_TOOL_RATE_LIMIT_MAX_ATTEMPTS = 8;
+const WORKER_TOOL_TRANSPORT_MAX_ATTEMPTS = 3;
+const WORKER_TOOL_TRANSPORT_RETRY_DELAY_MS = 1_000;
 
 function sleepWorkerMs(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -1566,6 +1479,7 @@ class WorkerToolBatchScheduler {
     private readonly resolvePacing: WorkerPacingResolver,
     private readonly abortSignal?: AbortSignal,
     private readonly onRequestsSettled?: (count: number) => void,
+    private readonly callbacks?: WorkerCtxCallbacks,
   ) {}
 
   /**
@@ -1632,7 +1546,7 @@ class WorkerToolBatchScheduler {
     return this.queue.some(
       (request) =>
         request.toolId !== 'test_wait_for_event' &&
-        getDefaultPlayRuntimeBatchStrategy(request.toolId) !== null,
+        getPlayRuntimeBatchStrategy(request.toolId) !== null,
     );
   }
 
@@ -1663,8 +1577,8 @@ class WorkerToolBatchScheduler {
     toolId: string,
     requests: WorkerToolBatchRequest[],
   ): Promise<void> {
-    const strategy = getDefaultPlayRuntimeBatchStrategy(toolId);
-    if (!strategy || toolId === 'test_wait_for_event') {
+    const strategy = getPlayRuntimeBatchStrategy(toolId);
+    if (!strategy || toolId === 'test_wait_for_event' || requests.length < 2) {
       const groupStartedAt = nowMs();
       await Promise.all(
         requests.map(async (request) => {
@@ -1676,10 +1590,11 @@ class WorkerToolBatchScheduler {
           });
           try {
             request.resolve(
-              await executeTool(
+              await executeToolWithLifecycle(
                 this.req,
                 { id: request.id, toolId, input: request.input },
                 request.workflowStep,
+                this.callbacks,
                 (retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
                 () => this.governor.chargeBudget('retry'),
               ),
@@ -1715,6 +1630,7 @@ class WorkerToolBatchScheduler {
       reportBackpressure: (retryAfterMs) =>
         this.reportBackpressure(toolId, retryAfterMs),
       onRequestsSettled: this.onRequestsSettled,
+      callbacks: this.callbacks,
     });
     recordRunnerPerfTrace({
       req: this.req,
@@ -1749,6 +1665,7 @@ async function executeBatchedWorkerToolGroup(input: {
   abortSignal?: AbortSignal;
   reportBackpressure: (retryAfterMs: number) => void;
   onRequestsSettled?: (count: number) => void;
+  callbacks?: WorkerCtxCallbacks;
 }): Promise<void> {
   const compiledBatches = compileRequestsWithStrategy({
     requests: input.requests,
@@ -1783,6 +1700,7 @@ async function executeBatchedWorkerToolGroup(input: {
         signal: input.abortSignal,
       });
       try {
+        input.callbacks?.onToolCalled?.(batch.batchOperation, nowMs());
         return await executeTool(
           input.req,
           {
@@ -1794,6 +1712,9 @@ async function executeBatchedWorkerToolGroup(input: {
           input.reportBackpressure,
           () => input.governor.chargeBudget('retry'),
         );
+      } catch (error) {
+        input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
+        throw error;
       } finally {
         slot.release();
       }
@@ -1995,47 +1916,16 @@ type WorkerMapOptions = {
   onRowError?: 'isolate' | 'fail';
 };
 
-function authoredCellPolicyForWorkerStep(
-  step: WorkerStepProgramStep,
-): {
-  recompute?: true;
-  recomputeOnError?: true;
-  staleAfterSeconds?: AuthoredStaleAfterSeconds;
-} | null {
-  const policy = {
-    ...(step.recompute === true ? { recompute: true as const } : {}),
-    ...(step.recomputeOnError === true
-      ? { recomputeOnError: true as const }
-      : {}),
-    ...(step.staleAfterSeconds !== undefined
-      ? { staleAfterSeconds: step.staleAfterSeconds }
-      : {}),
-  };
-  return Object.keys(policy).length > 0 ? policy : null;
-}
-
 function workerCellPoliciesFromSteps(
   steps: readonly WorkerStepProgramStep[],
 ): CellStalenessPolicyByField {
-  return Object.fromEntries(
-    steps.flatMap((step) => {
-      const policy = authoredCellPolicyForWorkerStep(step);
-      return policy
-        ? [[step.name, normalizeCellStalenessPolicy(policy)]]
-        : [];
-    }),
-  ) as CellStalenessPolicyByField;
+  return cellPoliciesFromDatasetSteps(steps);
 }
 
 function authoredWorkerCellPoliciesFromSteps(
   steps: readonly WorkerStepProgramStep[],
 ): AuthoredCellStalenessPolicyByField {
-  return Object.fromEntries(
-    steps.flatMap((step) => {
-      const policy = authoredCellPolicyForWorkerStep(step);
-      return policy ? [[step.name, policy]] : [];
-    }),
-  ) as AuthoredCellStalenessPolicyByField;
+  return authoredCellPoliciesFromDatasetSteps(steps);
 }
 
 /**
@@ -2431,6 +2321,46 @@ function parseFetchJsonOrNull(bodyText: string): unknown | null {
   }
 }
 
+async function safeWorkerPublicFetch(
+  input: string | URL,
+  init: RequestInit,
+  options: {
+    allowedOrigins: Iterable<string>;
+    sensitiveHeaders: Iterable<string>;
+  },
+): Promise<Response> {
+  const allowedOrigins = new Set(options.allowedOrigins);
+  return safePublicFetch(input, init, {
+    sensitiveHeaders: options.sensitiveHeaders,
+    fetchImpl: async (nextInput, nextInit) => {
+      const url = assertPublicHttpUrl(nextInput);
+      if (
+        !isIpAddressLiteral(url.hostname) &&
+        !allowedOrigins.has(url.origin)
+      ) {
+        throw new UnsafeOutboundUrlError(
+          'workers_edge ctx.fetch requires a public IP literal target or Deepline runtime origin. Use a Deepline integration tool for other hostname URLs.',
+        );
+      }
+      return fetch(url, nextInit);
+    },
+  });
+}
+
+function normalizeAllowedWorkerFetchOrigin(rawUrl: string): string | null {
+  try {
+    return assertPublicHttpUrl(rawUrl).origin;
+  } catch {
+    return null;
+  }
+}
+
+function getAllowedWorkerFetchOrigins(req: RunRequest): string[] {
+  return [req.baseUrl, req.callbackUrl]
+    .map(normalizeAllowedWorkerFetchOrigin)
+    .filter((origin): origin is string => origin !== null);
+}
+
 // ---------------------------------------------------------------------------
 // Streaming CSV parser. Pipes a `ReadableStream<Uint8Array>` from R2 through
 // a TextDecoder + line buffer + RFC-4180-ish state machine, yielding chunks
@@ -2964,96 +2894,6 @@ function requireSheetContract(
   return contract;
 }
 
-function isDatasetPayloadField(field: string): boolean {
-  return (
-    field.length > 0 &&
-    !field.startsWith('__deepline') &&
-    field !== '_key' &&
-    field !== '_status' &&
-    field !== '_run_id' &&
-    field !== '_error' &&
-    field !== '_stage' &&
-    field !== '_provider' &&
-    field !== '_input_index' &&
-    field !== '_created_at' &&
-    field !== '_updated_at' &&
-    field !== '_cell_meta'
-  );
-}
-
-function augmentSheetContractWithDatasetFields(input: {
-  contract: PlaySheetContract;
-  rows: readonly Record<string, unknown>[];
-  outputFields?: readonly string[];
-}): PlaySheetContract {
-  const outputFields = new Set(input.outputFields ?? []);
-  const candidateFields = new Set<string>();
-  for (const row of input.rows) {
-    for (const field of Object.keys(row)) {
-      if (isDatasetPayloadField(field)) {
-        candidateFields.add(field);
-      }
-    }
-  }
-  for (const field of outputFields) {
-    if (isDatasetPayloadField(field)) {
-      candidateFields.add(field);
-    }
-  }
-
-  const existingFields = new Set<string>();
-  const existingSqlNames = new Set<string>();
-  const inputColumns: PlaySheetContract['columns'] = [];
-  const outputColumns: PlaySheetContract['columns'] = [];
-  const appendColumn = (
-    target: PlaySheetContract['columns'],
-    column: PlaySheetContract['columns'][number],
-  ) => {
-    const field = typeof column.field === 'string' ? column.field : column.id;
-    const sqlName = column.sqlName.trim();
-    if (
-      !field ||
-      !sqlName ||
-      existingFields.has(field) ||
-      existingSqlNames.has(sqlName)
-    ) {
-      return;
-    }
-    existingFields.add(field);
-    existingSqlNames.add(sqlName);
-    target.push(column);
-  };
-
-  for (const column of input.contract.columns) {
-    const field = typeof column.field === 'string' ? column.field : column.id;
-    if (
-      column.source === 'input' &&
-      ((field === input.contract.tableNamespace && !candidateFields.has(field)) ||
-        outputFields.has(field))
-    ) {
-      continue;
-    }
-    appendColumn(
-      column.source === 'input' ? inputColumns : outputColumns,
-      column,
-    );
-  }
-
-  for (const field of candidateFields) {
-    if (existingFields.has(field)) continue;
-    const sqlName = sqlSafePlayColumnName(field);
-    if (existingSqlNames.has(sqlName)) continue;
-    appendColumn(outputFields.has(field) ? outputColumns : inputColumns, {
-      id: `runtime:${input.contract.tableNamespace}:${field}`,
-      sqlName,
-      source: outputFields.has(field) ? 'datasetColumn' : 'input',
-      field,
-    });
-  }
-
-  return { ...input.contract, columns: [...inputColumns, ...outputColumns] };
-}
-
 async function persistCompletedMapRows(input: {
   req: RunRequest;
   tableNamespace: string;
@@ -3112,10 +2952,10 @@ async function prepareMapRows(input: {
     tableNamespace: input.tableNamespace,
     sheetContract: augmentSheetContractWithDatasetFields({
       contract: requireSheetContract(input.req, input.tableNamespace),
-      rows: input.rows.map((row) => publicCsvStorageRow(row)),
+      rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
       outputFields: input.outputFields,
     }),
-    rows: input.rows.map((row) => publicCsvStorageRow(row)),
+    rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
     runId: input.req.runId,
     inputOffset: input.inputOffset,
     userEmail: input.req.userEmail,
@@ -3493,7 +3333,8 @@ function createMinimalWorkerCtx(
   const executeWithRuntimeReceipt = async <T>(
     key: string,
     execute: () => Promise<T> | T,
-    repairRunningReceiptForSameRun = false,
+    repairRunningReceiptForSameRun = true,
+    reclaimRunning = false,
   ): Promise<T> => {
     const serialized = await runWorkerRuntimeReceiptBoundary<unknown>({
       orgId: req.orgId,
@@ -3503,6 +3344,7 @@ function createMinimalWorkerCtx(
       receiptStore,
       execute: async () => serializeDurableStepValue(await execute()),
       repairRunningReceiptForSameRun,
+      reclaimRunning,
     });
     return deserializeDurableStepValue(serialized) as T;
   };
@@ -3524,7 +3366,7 @@ function createMinimalWorkerCtx(
         )(name, async () => serializeDurableStepValue(await execute()));
         return deserializeDurableStepValue(serialized) as T;
       },
-      true,
+      false,
     );
   };
   const nextCtxStepReceiptKey = (name: string): string => {
@@ -3545,6 +3387,14 @@ function createMinimalWorkerCtx(
     }
     return `:stale:${staleAfterSeconds}:${Math.floor(nowMs() / (staleAfterSeconds * 1000))}`;
   };
+  const rootToolBatchScheduler = new WorkerToolBatchScheduler(
+    req,
+    governor,
+    resolveToolPacing,
+    abortSignal,
+    undefined,
+    callbacks,
+  );
   // Local ancestry chain that always ENDS with the currently-executing play
   // (req.playName). The /api/v2/plays/run lineage validator requires the
   // submitted ancestry's tail to equal the executor token's play name (i.e.
@@ -3682,7 +3532,7 @@ function createMinimalWorkerCtx(
       if (parts.some((part) => part === null || part === undefined)) {
         throw new Error(
           `ctx.dataset("${name}") key returned null or undefined for row ${index}. ` +
-            'Return a non-empty string or number derived from a stable input column.',
+            'Use a stable non-empty key.',
         );
       }
       const normalizedParts = parts.map((part) => {
@@ -3694,7 +3544,7 @@ function createMinimalWorkerCtx(
       if (normalizedParts.some((part) => !part)) {
         throw new Error(
           `ctx.dataset("${name}") key returned an empty value for row ${index}. ` +
-            'Return a non-empty string or finite number derived from a stable input column.',
+            'Use a stable non-empty finite key.',
         );
       }
       const keyValue =
@@ -3991,19 +3841,8 @@ function createMinimalWorkerCtx(
                 reportExecutionHeartbeat(false);
                 const entry = uniqueRowsToExecuteEntries[myIndex]!;
                 const pendingRow = pendingRowsByKey.get(entry.rowKey);
-                const row = pendingRow
-                  ? ({
-                      ...entry.row,
-                      ...publicCsvInputRow(pendingRow),
-                      ...(pendingRow[DEEPLINE_CELL_META_FIELD] &&
-                      typeof pendingRow[DEEPLINE_CELL_META_FIELD] === 'object'
-                        ? {
-                            [DEEPLINE_CELL_META_FIELD]:
-                              pendingRow[DEEPLINE_CELL_META_FIELD],
-                          }
-                        : {}),
-                    } as T & Record<string, unknown>)
-                  : entry.row;
+                const row = runtimeCsvExecutionRow(entry.row, pendingRow) as T &
+                  Record<string, unknown>;
                 const absoluteIndex = entry.absoluteIndex;
                 const enriched: Record<string, unknown> =
                   cloneCsvAliasedRow(row);
@@ -4597,7 +4436,10 @@ function createMinimalWorkerCtx(
     const failFastRowErrors = opts?.onRowError === 'fail';
     let chunkIndex = 0;
     let chunkStart = 0;
-    for await (const rawChunkRows of iterDatasetChunks(inputRows, rowsPerChunk)) {
+    for await (const rawChunkRows of iterDatasetChunks(
+      inputRows,
+      rowsPerChunk,
+    )) {
       assertNotAborted(abortSignal);
       if (rawChunkRows.length === 0) continue;
       // Drop duplicate explicit-key rows before anything downstream observes
@@ -5004,7 +4846,13 @@ function createMinimalWorkerCtx(
             toolId: request.toolId,
             requestInput: request.input,
           })}${staleRuntimeSuffix(request.staleAfterSeconds)}`,
-          () => executeToolWithLifecycle(req, request, workflowStep, callbacks),
+          () =>
+            rootToolBatchScheduler.execute(
+              request.id,
+              request.toolId,
+              request.input,
+              workflowStep,
+            ),
         );
       },
     },
@@ -5157,6 +5005,9 @@ function createMinimalWorkerCtx(
           try {
             started = await submitChildPlayThroughCoordinator({
               req,
+              coordinatorBinding: cachedCoordinatorBinding,
+              makeRequestId,
+              coordinatorRequestHeaders,
               allowInline:
                 options?.timeoutMs == null && !childNeedsWorkflowScheduler,
               body: {
@@ -5462,7 +5313,10 @@ function createMinimalWorkerCtx(
         };
         const fetchInit = { ...init, headers };
         delete fetchInit.auth;
-        const response = await fetch(url, fetchInit);
+        const response = await safeWorkerPublicFetch(url, fetchInit, {
+          allowedOrigins: getAllowedWorkerFetchOrigins(req),
+          sensitiveHeaders: Object.keys(secretHeaderMarkers),
+        });
         assertNotAborted(abortSignal);
         const bodyText = await response.text();
         const redactedBodyText = secretRedactor.redactString(bodyText);
@@ -5748,12 +5602,10 @@ async function executeRunRequest(
   let runLogBuffer: string[] = [];
   let pendingRunLogLines: string[] = [];
   // Monotonic count of every line ever appended to this run's worker log
-  // channel. runLogBuffer/pendingRunLogLines are rotating tails of those
-  // lines (RUN_LOG_BUFFER_LIMIT is the coordinator transport cache only), so
-  // each log.appended batch can carry the absolute channelOffset of its first
-  // line: totalEmittedLogLines - pendingRunLogLines.length. Run Log Stream
-  // ingestion skips re-sent prefixes positionally (exactly-once, repeated
-  // identical lines preserved) instead of text-deduping.
+  // channel. runLogBuffer is only the rotating live/coordinator transport
+  // cache; pendingRunLogLines is the durable unsent suffix and must not rotate,
+  // otherwise a flush already in flight can let fresh lines fall out before
+  // Run Log Stream ingestion ever sees them.
   let totalEmittedLogLines = 0;
   let stepProgressByNodeId: LiveNodeProgressMap = {};
   let dirtyProgressNodeIds = new Set<string>();
@@ -5779,9 +5631,7 @@ async function executeRunRequest(
     if (!trimmed) return;
     totalEmittedLogLines += 1;
     runLogBuffer = [...runLogBuffer, trimmed].slice(-RUN_LOG_BUFFER_LIMIT);
-    pendingRunLogLines = [...pendingRunLogLines, trimmed].slice(
-      -RUN_LOG_BUFFER_LIMIT,
-    );
+    pendingRunLogLines = [...pendingRunLogLines, trimmed];
   };
 
   const updateStepProgress = (input: {
@@ -5970,9 +5820,7 @@ async function executeRunRequest(
         lines: pendingRunLogLines,
         // Positional cursor: pendingRunLogLines always holds the LAST
         // pending lines emitted on this channel, so the offset of its first
-        // line is total-emitted minus pending length. This also covers the
-        // terminal full-buffer re-send (pending = runLogBuffer), which
-        // ingestion then skips positionally instead of via text dedupe.
+        // line is total-emitted minus pending length.
         channelOffset: totalEmittedLogLines - pendingRunLogLines.length,
       });
       pendingRunLogLines = [];
@@ -6068,18 +5916,13 @@ async function executeRunRequest(
     terminalEvent: PlayRunLedgerEvent,
   ): Promise<void> => {
     if (!options?.persistResultDatasets) return;
+    await ledgerFlushInFlight;
     const now = nowMs();
-    // Terminal re-send of the full retained buffer. drainPendingLedgerEvents
-    // stamps it with channelOffset = totalEmitted - buffer length, so Run Log
-    // Stream ingestion drops the already-ingested prefix positionally.
-    pendingRunLogLines = runLogBuffer;
     dirtyProgressNodeIds = new Set([
       ...dirtyProgressNodeIds,
       ...Object.keys(stepProgressByNodeId),
     ]);
-    pendingLedgerEvents = [...pendingLedgerEvents, terminalEvent];
-    await ledgerFlushInFlight;
-    const events = drainPendingLedgerEvents(now);
+    const events = [...drainPendingLedgerEvents(now), terminalEvent];
     if (events.length === 0) return;
     try {
       await postRuntimeApi(req.baseUrl, req.executorToken, {
@@ -6294,7 +6137,6 @@ async function executeRunRequest(
       playName: req.playName,
       result: serializedResult,
       outputRows: inferOutputRows(serializedResult),
-      liveLogs: runLogBuffer,
       liveNodeProgress: stepProgressSnapshot(),
       durationMs: nowMs() - startedAt,
     };