deepline 0.1.12 → 0.1.19

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -204,11 +204,18 @@ type WorkerEnv = {
    * loud error. Loud failures > silent fallbacks.
    */
   HARNESS?: import('../../play-harness-worker/src/rpc-types').PlayHarnessRpc;
+  VERCEL_PROTECTION_BYPASS_TOKEN?: string;
 };
 
 let cachedRuntimeApiBinding: WorkerEnv['RUNTIME_API'] | null = null;
+let cachedRuntimeApiVercelBypassToken: string | null = null;
 function captureRuntimeApiBinding(env: WorkerEnv): void {
   cachedRuntimeApiBinding = env.RUNTIME_API ?? null;
+  cachedRuntimeApiVercelBypassToken =
+    typeof env.VERCEL_PROTECTION_BYPASS_TOKEN === 'string' &&
+    env.VERCEL_PROTECTION_BYPASS_TOKEN.trim()
+      ? env.VERCEL_PROTECTION_BYPASS_TOKEN.trim()
+      : null;
 }
 
 let cachedCoordinatorBinding: WorkerEnv['COORDINATOR'] | null = null;
@@ -288,22 +295,20 @@ async function probeHarnessOnce(
   }
 }
 /**
- * Routes runtime API requests through the in-process RUNTIME_API binding.
- * The coordinator MUST provide this binding — there is no public-fetch fallback.
+ * Routes runtime API requests through the in-process RUNTIME_API binding when
+ * Cloudflare exposes the coordinator WorkerEntrypoint export. Some workflow
+ * execution paths do not expose those exports; there we keep the older public
+ * fetch transport so the play still reaches the same authenticated handler.
  */
 const RUNTIME_API_TIMEOUT_MS = 30_000;
 const RUNTIME_API_PLAY_RUN_TIMEOUT_MS = 75_000;
+let loggedMissingRuntimeApiBinding = false;
 
 async function fetchRuntimeApi(
   baseUrl: string,
   path: string,
   init: RequestInit,
 ): Promise<Response> {
-  if (!cachedRuntimeApiBinding) {
-    throw new Error(
-      `[play-harness] RUNTIME_API binding missing — coordinator did not wire it before invoking the play. path=${path}`,
-    );
-  }
   const timeoutMs =
     path === '/api/v2/plays/run'
       ? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
@@ -313,16 +318,25 @@ async function fetchRuntimeApi(
   try {
     const mergedInit: RequestInit = {
       ...init,
+      headers: runtimeApiHeaders(init.headers, cachedRuntimeApiBinding == null),
       signal: controller.signal,
     };
-    const res = await cachedRuntimeApiBinding.fetch(
-      new Request(`${baseUrl}${path}`, mergedInit),
+    if (!cachedRuntimeApiBinding) {
+      if (!loggedMissingRuntimeApiBinding) {
+        loggedMissingRuntimeApiBinding = true;
+        console.warn(
+          `[play-harness] RUNTIME_API binding missing; using public runtime API transport. path=${path}`,
+        );
+      }
+      return await fetch(`${baseUrl.replace(/\/$/, '')}${path}`, mergedInit);
+    }
+    return await cachedRuntimeApiBinding.fetch(
+      new Request(`${baseUrl.replace(/\/$/, '')}${path}`, mergedInit),
     );
-    return res;
   } catch (err) {
     if (err instanceof Error && err.name === 'AbortError') {
       throw new Error(
-        `[play-harness] RUNTIME_API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
+        `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
       );
     }
     throw err;
@@ -331,6 +345,24 @@ async function fetchRuntimeApi(
   }
 }
 
+function runtimeApiHeaders(
+  headers: HeadersInit | undefined,
+  includeVercelBypass: boolean,
+): Headers {
+  const next = new Headers(headers);
+  if (includeVercelBypass) {
+    const bypassToken = cachedVercelProtectionBypassToken();
+    if (bypassToken) {
+      next.set('x-vercel-protection-bypass', bypassToken);
+    }
+  }
+  return next;
+}
+
+function cachedVercelProtectionBypassToken(): string | null {
+  return cachedRuntimeApiVercelBypassToken;
+}
+
 const WORKER_PLAY_CALL_LIMITS = {
   maxPlayCallDepth: 6,
   maxPlayCallCount: 32,
@@ -350,6 +382,20 @@ function makeWorkerDataset<T extends Record<string, unknown>>(
     count?: number;
     datasetKind?: 'csv' | 'map';
     cacheSummary?: string | null;
+    workProgress?: {
+      total: number;
+      executed: number;
+      reused: number;
+      skipped: number;
+      pending: number;
+      failed: number;
+      degraded?: boolean;
+      duplicates?: {
+        exact?: number;
+        semantic?: number;
+        rejected?: number;
+      };
+    };
   },
 ): T[] & {
   count(): Promise<number>;
@@ -363,6 +409,7 @@ function makeWorkerDataset<T extends Record<string, unknown>>(
   const count = Math.max(0, Math.floor(options?.count ?? rows.length));
   const datasetKind = options?.datasetKind ?? 'map';
   const cacheSummary = options?.cacheSummary ?? null;
+  const workProgress = options?.workProgress;
   // Build the array result. JSON.stringify on arrays calls toJSON only if
   // present on the array itself — we attach below. The dataset metadata is
   // also exposed via own properties so plays can `enriched.count()` etc.
@@ -415,6 +462,10 @@ function makeWorkerDataset<T extends Record<string, unknown>>(
     value: cacheSummary,
     enumerable: false,
   });
+  Object.defineProperty(arr, '__deeplineWorkProgress', {
+    value: workProgress,
+    enumerable: false,
+  });
   // Plays often `return { rows: dataset, count: N }`. JSON.stringify on the
   // array would normally produce `[row, row, ...]` — we want the dataset
   // envelope shape instead so assertions seeing `result.rows.columns` pass.
@@ -435,6 +486,9 @@ function makeWorkerDataset<T extends Record<string, unknown>>(
         preview: plainRows,
         tableNamespace: name,
         ...(cacheSummary ? { cacheSummary } : {}),
+        ...(workProgress
+          ? { _metadata: { workProgress } }
+          : {}),
       };
     },
     enumerable: false,
@@ -454,6 +508,7 @@ type RunnerEvent =
   | { type: 'error'; message: string; stack?: string; ts: number };
 
 type WorkflowRunOutput = {
+  playName: string;
   result: unknown;
   outputRows: number;
   durationMs: number;
@@ -469,7 +524,12 @@ function makeRequestId(): string {
 }
 
 function publicCsvInputRow<T extends Record<string, unknown>>(row: T): T {
-  return stripCsvProjectedFields(row) as T;
+  const stripped = stripCsvProjectedFields(row) as Record<string, unknown>;
+  return Object.fromEntries(
+    Object.entries(stripped).filter(
+      ([fieldName]) => !fieldName.startsWith('__deepline'),
+    ),
+  ) as T;
 }
 
 /**
@@ -714,6 +774,11 @@ function childPlayEventKey(input: { key: string; workflowId: string }): string {
   return `child_play_${hashChildPlayEventKey(`${input.key}:${input.workflowId}`)}_${readableKey}`;
 }
 
+function workflowTimeoutFromMs(timeoutMs: number): string {
+  const seconds = Math.max(1, Math.ceil(timeoutMs / 1000));
+  return `${seconds} second${seconds === 1 ? '' : 's'}`;
+}
+
 async function waitForChildPlayTerminalEvent(input: {
   req: RunRequest;
   workflowStep?: WorkflowStep;
@@ -734,11 +799,11 @@ async function waitForChildPlayTerminalEvent(input: {
   const event = (await (
     input.workflowStep.waitForEvent as unknown as (
       name: string,
-      options: { type: string; timeout: number },
+      options: { type: string; timeout: string },
     ) => Promise<{ payload: unknown }>
   )(`child_play_terminal:${eventKey}`, {
     type: integrationEventType(eventKey),
-    timeout: input.timeoutMs,
+    timeout: workflowTimeoutFromMs(input.timeoutMs),
   })) as { payload: unknown };
   const rawPayload = isRecord(event.payload) ? event.payload : {};
   const payload = isRecord(rawPayload.data) ? rawPayload.data : rawPayload;
@@ -909,15 +974,25 @@ async function waitForSyntheticIntegrationEvent(
     typeof input.timeout_ms === 'number' && Number.isFinite(input.timeout_ms)
       ? Math.max(1, Math.round(input.timeout_ms))
       : 30_000;
+  await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
+    action: 'update_run_status',
+    playId: req.runId,
+    status: 'running',
+    runtimeBackend: 'cf_workflows_dynamic_worker',
+    waitKind: 'integration_event_batch',
+    waitUntil: nowMs() + timeoutMs,
+    activeBoundaryId: `integration_event:${eventKey}`,
+    lastCheckpointAt: nowMs(),
+  });
   try {
     const event = (await (
       workflowStep.waitForEvent as unknown as (
         name: string,
-        options: { type: string; timeout: number },
+        options: { type: string; timeout: string },
       ) => Promise<{ payload: unknown }>
     )(`integration_event:${eventKey}`, {
       type: integrationEventType(eventKey),
-      timeout: timeoutMs,
+      timeout: workflowTimeoutFromMs(timeoutMs),
     })) as { payload: unknown };
     const payload =
       event.payload &&
@@ -983,7 +1058,7 @@ async function callToolDirect(
       headers: {
         'content-type': 'application/json',
         authorization: `Bearer ${req.executorToken}`,
-        'x-deepline-request-id': `${req.runId}:${toolId}:${id}`,
+        'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${attempt}`,
         [EXECUTE_TOOL_METADATA_HEADER]: 'true',
       },
       body: JSON.stringify({ payload: input }),
@@ -1455,7 +1530,9 @@ async function executeBatchedWorkerToolGroup(input: {
     ) => {
       for (const entry of chunkResults) {
         const batchResult = isToolExecuteResult(entry.result)
-          ? entry.result.result.data
+          ? isRecordLike(entry.result.result)
+            ? entry.result.result.data
+            : undefined
           : entry.result;
         const splitResults =
           batchResult != null
@@ -1516,6 +1593,7 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
   rowsWritten: number;
   rowsExecuted: number;
   rowsCached: number;
+  rowsDuplicateReused: number;
   rowsInserted: number;
   rowsSkipped: number;
   outputDatasetId: string;
@@ -1641,22 +1719,32 @@ async function executeWorkerStepProgram(
     path: string[];
     outputs: RecordedStepProgramOutput[];
   },
+  workflowStep?: WorkflowStep,
 ): Promise<unknown> {
   let currentRow: Record<string, unknown> = cloneCsvAliasedRow(inputRow);
   for (const step of program.steps) {
     const stepPath = [...(recorder?.path ?? []), step.name];
-    const resolution = await executeWorkerStepResolver(
-      step.resolver,
-      currentRow,
-      ctx,
-      index,
-      recorder
-        ? {
-            ...recorder,
-            path: stepPath,
-          }
-        : undefined,
-    );
+    const runStep = async () =>
+      await executeWorkerStepResolver(
+        step.resolver,
+        currentRow,
+        ctx,
+        index,
+        recorder
+          ? {
+              ...recorder,
+              path: stepPath,
+            }
+          : undefined,
+      );
+    const resolution = workflowStep
+      ? await (
+          workflowStep.do as unknown as (
+            name: string,
+            callback: () => Promise<WorkerStepResolution>,
+          ) => Promise<WorkerStepResolution>
+        )(stepPath.join('.'), runStep)
+      : await runStep();
     const value = resolution.value;
     currentRow = cloneCsvAliasedRow(currentRow, { [step.name]: value });
     if (recorder) {
@@ -2569,7 +2657,17 @@ function createMinimalWorkerCtx(
       const rowsToExecuteEntries = chunkEntries.filter(
         ({ rowKey }) => pendingKeys.has(rowKey) || !completedKeys.has(rowKey),
       );
-      const rowsToExecute = rowsToExecuteEntries.map(({ row }) => row);
+      const uniqueRowsToExecuteEntries = [
+        ...new Map(
+          rowsToExecuteEntries.map((entry) => [entry.rowKey, entry]),
+        ).values(),
+      ];
+      const duplicateInputReuseCount = Math.max(
+        0,
+        chunkEntries.length -
+          new Set(chunkEntries.map((entry) => entry.rowKey)).size,
+      );
+      const rowsToExecute = uniqueRowsToExecuteEntries.map(({ row }) => row);
       const rowsInserted = prepared.inserted + missingPreparedRows.length;
       const rowsSkipped = Math.max(
         0,
@@ -2593,7 +2691,7 @@ function createMinimalWorkerCtx(
               if (abortSignal?.aborted) return;
               const myIndex = idx++;
               if (myIndex >= rowsToExecute.length) return;
-              const entry = rowsToExecuteEntries[myIndex]!;
+              const entry = uniqueRowsToExecuteEntries[myIndex]!;
               const row = entry.row;
               const absoluteIndex = entry.absoluteIndex;
               const enriched: Record<string, unknown> = cloneCsvAliasedRow(row);
@@ -2701,14 +2799,33 @@ function createMinimalWorkerCtx(
           })(),
         );
       }
-      await Promise.all(workers);
-      if (executedRows.length > 0) {
+      const persistExecutedRows = async () => {
+        const rowsToPersist = executedRows
+          .map((row, executedIndex) =>
+            row
+              ? {
+                  row,
+                  executedIndex,
+                }
+              : null,
+          )
+          .filter(
+            (
+              entry,
+            ): entry is {
+              row: T & Record<string, unknown>;
+              executedIndex: number;
+            } => entry !== null,
+          );
+        if (rowsToPersist.length === 0) {
+          return;
+        }
         await persistCompletedMapRows({
           req,
           tableNamespace: name,
           outputFields,
           extraOutputFields: Array.from(generatedOutputFields),
-          rows: executedRows.map((row, executedIndex) => ({
+          rows: rowsToPersist.map(({ row, executedIndex }) => ({
             ...row,
             ...(executedCellMetaPatches[executedIndex]
               ? {
@@ -2716,9 +2833,19 @@ function createMinimalWorkerCtx(
                     executedCellMetaPatches[executedIndex],
                 }
               : {}),
-            __deeplineRowKey: rowsToExecuteEntries[executedIndex]!.rowKey,
+            __deeplineRowKey:
+              uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
           })),
         });
+      };
+      const workerResults = await Promise.allSettled(workers);
+      await persistExecutedRows();
+      const rejectedWorker = workerResults.find(
+        (result): result is PromiseRejectedResult =>
+          result.status === 'rejected',
+      );
+      if (rejectedWorker) {
+        throw rejectedWorker.reason;
       }
       const resultByKey = new Map<string, T & Record<string, unknown>>();
       for (const completedRow of prepared.completedRows) {
@@ -2739,7 +2866,7 @@ function createMinimalWorkerCtx(
         executedIndex += 1
       ) {
         const executedRow = executedRows[executedIndex]!;
-        const key = rowsToExecuteEntries[executedIndex]!.rowKey;
+        const key = uniqueRowsToExecuteEntries[executedIndex]!.rowKey;
         if (key) resultByKey.set(key, executedRow);
       }
       const out = chunkRows
@@ -2755,7 +2882,8 @@ function createMinimalWorkerCtx(
         rowsRead: chunkRows.length,
         rowsWritten: out.length,
         rowsExecuted: executedRows.length,
-        rowsCached: prepared.completedRows.length,
+        rowsCached: Math.max(0, out.length - executedRows.length),
+        rowsDuplicateReused: duplicateInputReuseCount,
         rowsInserted,
         rowsSkipped,
         outputDatasetId: `map:${name}`,
@@ -2767,6 +2895,7 @@ function createMinimalWorkerCtx(
     const out: Array<T & Record<string, unknown>> = [];
     let totalRowsExecuted = 0;
     let totalRowsCached = 0;
+    let totalRowsDuplicateReused = 0;
     let totalRowsInserted = 0;
     let totalRowsSkipped = 0;
 
@@ -2809,6 +2938,17 @@ function createMinimalWorkerCtx(
       return makeWorkerDataset(name, out, {
         count: totalRowsWritten,
         cacheSummary,
+        workProgress: {
+          total: totalRowsWritten,
+          executed: totalRowsExecuted,
+          reused: totalRowsCached,
+          skipped: totalRowsCached,
+          pending: 0,
+          failed: 0,
+          ...(totalRowsDuplicateReused > 0
+            ? { duplicates: { exact: totalRowsDuplicateReused } }
+            : {}),
+        },
       });
     };
 
@@ -2829,6 +2969,7 @@ function createMinimalWorkerCtx(
         totalRowsWritten += chunkResult.rowsWritten;
         totalRowsExecuted += chunkResult.rowsExecuted;
         totalRowsCached += chunkResult.rowsCached;
+        totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
         totalRowsInserted += chunkResult.rowsInserted;
         totalRowsSkipped += chunkResult.rowsSkipped;
         if (out.length < 10) {
@@ -2852,6 +2993,7 @@ function createMinimalWorkerCtx(
         totalRowsWritten += chunkResult.rowsWritten;
         totalRowsExecuted += chunkResult.rowsExecuted;
         totalRowsCached += chunkResult.rowsCached;
+        totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
         totalRowsInserted += chunkResult.rowsInserted;
         totalRowsSkipped += chunkResult.rowsSkipped;
         if (out.length < 10) {
@@ -2865,6 +3007,7 @@ function createMinimalWorkerCtx(
     const chunkResult = await runChunkStep(sliced, 0, 0);
     totalRowsExecuted = chunkResult.rowsExecuted;
     totalRowsCached = chunkResult.rowsCached;
+    totalRowsDuplicateReused = chunkResult.rowsDuplicateReused;
     totalRowsInserted = chunkResult.rowsInserted;
     totalRowsSkipped = chunkResult.rowsSkipped;
     out.push(...chunkResult.preview);
@@ -2917,18 +3060,39 @@ function createMinimalWorkerCtx(
     },
     async step<T>(name: string, callback: () => Promise<T> | T): Promise<T> {
       assertNotAborted(abortSignal);
-      if (!workflowStep) {
-        return await callback();
+      if (!name.trim()) {
+        throw new Error('ctx.step(name, callback) requires a name.');
       }
-      return await (
-        workflowStep.do as unknown as (
-          name: string,
-          callback: () => Promise<T>,
-        ) => Promise<T>
-      )(name, async () => {
-        assertNotAborted(abortSignal);
-        return await callback();
-      });
+      // Static pipeline JS blocks are already Workflow steps in the Workers
+      // backend. Nesting another `step.do` here can leave preview runs parked
+      // inside the JS stage before they reach subsequent event waits.
+      return await callback();
+    },
+    async runSteps<T>(
+      program: WorkerStepProgram,
+      input: Record<string, unknown>,
+      opts?: { description?: string },
+    ): Promise<T> {
+      assertNotAborted(abortSignal);
+      if (!isWorkerStepProgram(program)) {
+        throw new Error('ctx.runSteps(program, input) requires steps().');
+      }
+      if (opts?.description) {
+        emitEvent({
+          type: 'log',
+          level: 'info',
+          message: String(opts.description),
+          ts: nowMs(),
+        });
+      }
+      return (await executeWorkerStepProgram(
+        program,
+        input,
+        ctx,
+        0,
+        undefined,
+        workflowStep,
+      )) as T;
     },
     async csv<T extends Record<string, unknown> = Record<string, unknown>>(
       arg: unknown,
@@ -3263,6 +3427,14 @@ function createMinimalWorkerCtx(
         return makeWorkerDataset(name, out, {
           count: totalRowsWritten,
           cacheSummary,
+          workProgress: {
+            total: totalRowsWritten,
+            executed: totalRowsExecuted,
+            reused: totalRowsCached,
+            skipped: totalRowsCached,
+            pending: 0,
+            failed: 0,
+          },
         });
       };
 
@@ -3619,11 +3791,11 @@ function createMinimalWorkerCtx(
       const event = (await (
         workflowStep.waitForEvent as unknown as (
           name: string,
-          options: { type: string; timeout: number },
+          options: { type: string; timeout: string },
         ) => Promise<{ payload: unknown }>
       )(`wait_for_event:${workflowEventType(eventType)}`, {
         type: workflowEventType(eventType),
-        timeout: timeoutMs,
+        timeout: workflowTimeoutFromMs(timeoutMs),
       })) as { payload: unknown };
       return event.payload ?? null;
     },
@@ -3804,6 +3976,9 @@ async function executeRunRequest(
         status: 'completed',
         result: terminalResult,
         runtimeBackend: 'cf_workflows_dynamic_worker',
+        waitKind: null,
+        waitUntil: null,
+        activeBoundaryId: null,
         liveLogs,
         lastCheckpointAt: nowMs(),
       });
@@ -3825,6 +4000,7 @@ async function executeRunRequest(
       );
     });
     return {
+      playName: req.playName,
       result: serializedResult,
       outputRows: inferOutputRows(serializedResult),
       durationMs: nowMs() - startedAt,
@@ -3847,6 +4023,9 @@ async function executeRunRequest(
         status: aborted ? 'cancelled' : 'failed',
         error: message,
         runtimeBackend: 'cf_workflows_dynamic_worker',
+        waitKind: null,
+        waitUntil: null,
+        activeBoundaryId: null,
         liveLogs,
         lastCheckpointAt: nowMs(),
       });
@@ -4121,6 +4300,14 @@ function serializeValue(value: unknown, depth: number): unknown {
         ? (value as unknown as { __deeplineCacheSummary: string })
             .__deeplineCacheSummary
         : null;
+    const workProgress =
+      isRecord(
+        (value as unknown as { __deeplineWorkProgress?: unknown })
+          .__deeplineWorkProgress,
+      )
+        ? (value as unknown as { __deeplineWorkProgress: Record<string, unknown> })
+            .__deeplineWorkProgress
+        : null;
     const previewRows = value
       .slice(0, 5)
       .map((row) => serializeValue(row, depth + 1))
@@ -4138,6 +4325,9 @@ function serializeValue(value: unknown, depth: number): unknown {
         preview: previewRows,
         tableNamespace,
         ...(cacheSummary ? { cacheSummary } : {}),
+        ...(workProgress
+          ? { _metadata: { workProgress } }
+          : {}),
       };
     }
     return value.map((entry) => serializeValue(entry, depth + 1));