deepline 0.1.22 → 0.1.24

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -93,6 +93,10 @@ import {
   type CsvRenameOptions,
 } from '../../../shared_libs/play-runtime/csv-rename';
 import { coordinatorRequestHeaders } from '../../../shared_libs/play-runtime/coordinator-headers';
+import type {
+  LiveNodeProgressMap,
+  LiveNodeProgressSnapshot,
+} from './runtime/live-progress';
 
 // The play's default export. The bundler injects this — see bundle-play-file.ts.
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -496,9 +500,7 @@ function makeWorkerDataset<T extends Record<string, unknown>>(
         preview: plainRows,
         tableNamespace: name,
         ...(cacheSummary ? { cacheSummary } : {}),
-        ...(workProgress
-          ? { _metadata: { workProgress } }
-          : {}),
+        ...(workProgress ? { _metadata: { workProgress } } : {}),
       };
     },
     enumerable: false,
@@ -521,22 +523,11 @@ type WorkflowRunOutput = {
   playName: string;
   result: unknown;
   outputRows: number;
+  liveLogs?: string[];
+  liveNodeProgress?: LiveNodeProgressMap;
   durationMs: number;
 };
 
-type LiveNodeProgressSnapshot = {
-  completed?: number;
-  total?: number;
-  failed?: number;
-  message?: string;
-  updatedAt?: number;
-  startedAt?: number;
-  completedAt?: number;
-  artifactTableNamespace?: string | null;
-};
-
-type LiveNodeProgressMap = Record<string, LiveNodeProgressSnapshot>;
-
 type WorkerCtxCallbacks = {
   onNodeProgress?: (input: {
     nodeId: string;
@@ -555,6 +546,12 @@ function recordRunnerPerfTrace(input: {
   extra?: Record<string, unknown>;
 }): void {
   if (!input.req.runId || !input.phase) return;
+  // Tool-level traces can fire once per row/provider step. Forwarding each one
+  // through the coordinator binding can consume Cloudflare's subrequest budget
+  // before large batched maps finish.
+  if (input.phase.startsWith('runner.tool.')) {
+    return;
+  }
   const payload = {
     ts: Date.now(),
     source: 'dynamic_worker' as const,
@@ -946,7 +943,8 @@ async function signalParentPlayTerminal(input: {
     }
     const text = await res.text().catch(() => '');
     throw new Error(
-      text.slice(0, 800) || `Coordinator parent signal failed with ${res.status}.`,
+      text.slice(0, 800) ||
+        `Coordinator parent signal failed with ${res.status}.`,
     );
   }
   throw new Error(
@@ -2517,6 +2515,7 @@ async function persistCompletedMapRows(input: {
     ],
     runId: input.req.runId,
     userEmail: input.req.userEmail,
+    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
   });
 }
 
@@ -2542,6 +2541,7 @@ async function prepareMapRows(input: {
     rows: input.rows.map((row) => ({ ...row })),
     runId: input.req.runId,
     userEmail: input.req.userEmail,
+    preloadedDbSessions: input.req.preloadedDbSessions ?? null,
   });
   return {
     inserted: result.inserted,
@@ -3091,8 +3091,7 @@ function createMinimalWorkerCtx(
                     executedCellMetaPatches[executedIndex],
                 }
               : {}),
-            __deeplineRowKey:
-              uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
+            __deeplineRowKey: uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
           })),
         });
       };
@@ -3326,7 +3325,11 @@ function createMinimalWorkerCtx(
         req,
         phase: 'runner.map.total',
         ms: nowMs() - mapStartedAt,
-        extra: { mapName: name, rowsWritten: totalRowsWritten, streaming: true },
+        extra: {
+          mapName: name,
+          rowsWritten: totalRowsWritten,
+          streaming: true,
+        },
       });
       return dataset;
     }
@@ -3360,7 +3363,11 @@ function createMinimalWorkerCtx(
         req,
         phase: 'runner.map.total',
         ms: nowMs() - mapStartedAt,
-        extra: { mapName: name, rowsWritten: totalRowsWritten, streaming: false },
+        extra: {
+          mapName: name,
+          rowsWritten: totalRowsWritten,
+          streaming: false,
+        },
       });
       return dataset;
     }
@@ -4176,6 +4183,7 @@ async function executeRunRequest(
         phase: 'runner.terminal_status_update',
         ms: nowMs() - terminalUpdateStartedAt,
       });
+
       const billingStartedAt = nowMs();
       await finalizeWorkerComputeBilling({
         req,
@@ -4214,6 +4222,8 @@ async function executeRunRequest(
       playName: req.playName,
       result: serializedResult,
       outputRows: inferOutputRows(serializedResult),
+      liveLogs,
+      liveNodeProgress: liveNodeProgressSnapshot(),
       durationMs: nowMs() - startedAt,
     };
   } catch (error) {
@@ -4429,6 +4439,7 @@ async function persistResultDatasets(
       rows: dataset.rows,
       runId: req.runId,
       userEmail: req.userEmail,
+      preloadedDbSessions: req.preloadedDbSessions ?? null,
     });
   }
 }
@@ -4515,14 +4526,16 @@ function serializeValue(value: unknown, depth: number): unknown {
         ? (value as unknown as { __deeplineCacheSummary: string })
             .__deeplineCacheSummary
         : null;
-    const workProgress =
-      isRecord(
-        (value as unknown as { __deeplineWorkProgress?: unknown })
-          .__deeplineWorkProgress,
-      )
-        ? (value as unknown as { __deeplineWorkProgress: Record<string, unknown> })
-            .__deeplineWorkProgress
-        : null;
+    const workProgress = isRecord(
+      (value as unknown as { __deeplineWorkProgress?: unknown })
+        .__deeplineWorkProgress,
+    )
+      ? (
+          value as unknown as {
+            __deeplineWorkProgress: Record<string, unknown>;
+          }
+        ).__deeplineWorkProgress
+      : null;
     const previewRows = value
       .slice(0, 5)
       .map((row) => serializeValue(row, depth + 1))
@@ -4540,9 +4553,7 @@ function serializeValue(value: unknown, depth: number): unknown {
         preview: previewRows,
         tableNamespace,
         ...(cacheSummary ? { cacheSummary } : {}),
-        ...(workProgress
-          ? { _metadata: { workProgress } }
-          : {}),
+        ...(workProgress ? { _metadata: { workProgress } } : {}),
       };
     }
     return value.map((entry) => serializeValue(entry, depth + 1));

package/dist/repo/apps/play-runner-workers/src/runtime/live-progress.ts

@@ -0,0 +1,18 @@
+export type LiveNodeProgressSnapshot = {
+  completed?: number;
+  total?: number;
+  failed?: number;
+  message?: string;
+  updatedAt?: number;
+  startedAt?: number;
+  completedAt?: number;
+  artifactTableNamespace?: string | null;
+};
+
+export type LiveNodeProgressMap = Record<string, LiveNodeProgressSnapshot>;
+
+export function sanitizeLiveLogLines(value: unknown): string[] | null {
+  return Array.isArray(value)
+    ? value.filter((line): line is string => typeof line === 'string')
+    : null;
+}

package/dist/repo/sdk/src/client.ts

@@ -67,6 +67,26 @@ import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-mani
 
 const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
 const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
+const COMPILE_MANIFEST_RETRY_DELAYS_MS = [250, 1_000];
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isTransientCompileManifestError(error: unknown): boolean {
+  if (error instanceof DeeplineError && typeof error.statusCode === 'number') {
+    return (
+      error.statusCode === 408 ||
+      error.statusCode === 425 ||
+      error.statusCode === 499 ||
+      (error.statusCode >= 500 && error.statusCode < 600)
+    );
+  }
+  const message = error instanceof Error ? error.message : String(error);
+  return /fetch failed|connection (?:closed|reset|terminated)|socket hang up|econnreset|etimedout|eai_again|abort/i.test(
+    message,
+  );
+}
 
 type ExecuteToolRawOptions = {
   includeToolMetadata?: boolean;
@@ -759,10 +779,24 @@ export class DeeplineClient {
     artifact: Record<string, unknown>;
     importedPlayDependencies?: PlayCompilerManifest[];
   }): Promise<PlayCompilerManifest> {
-    const response = await this.http.post<{
-      compilerManifest: PlayCompilerManifest;
-    }>('/api/v2/plays/compile-manifest', input);
-    return response.compilerManifest;
+    const retryDelays = COMPILE_MANIFEST_RETRY_DELAYS_MS.slice(
+      0,
+      Math.max(0, this.config.maxRetries),
+    );
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        const response = await this.http.post<{
+          compilerManifest: PlayCompilerManifest;
+        }>('/api/v2/plays/compile-manifest', input);
+        return response.compilerManifest;
+      } catch (error) {
+        const delayMs = retryDelays[attempt];
+        if (delayMs === undefined || !isTransientCompileManifestError(error)) {
+          throw error;
+        }
+        await sleep(delayMs);
+      }
+    }
   }
 
   /**

package/dist/repo/sdk/src/plays/harness-stub.ts

@@ -146,6 +146,7 @@ export async function harnessStartSheetDataset(input: {
   rows: Array<Record<string, unknown>>;
   runId: string;
   userEmail?: string | null;
+  preloadedDbSessions?: PreloadedRuntimeDbSessionInput[] | null;
 }): Promise<{
   inserted: number;
   skipped: number;
@@ -170,6 +171,7 @@ export async function harnessPersistCompletedSheetRows(input: {
   outputFields: string[];
   runId: string;
   userEmail?: string | null;
+  preloadedDbSessions?: PreloadedRuntimeDbSessionInput[] | null;
 }): Promise<{ ok: true; rowsWritten: number; tableNamespace: string }> {
   return requireBinding().persistCompletedMapRows(input);
 }

package/dist/repo/sdk/src/version.ts

@@ -1,2 +1,2 @@
-export const SDK_VERSION = "0.1.22";
+export const SDK_VERSION = "0.1.24";
 export const SDK_API_CONTRACT = "2026-05-runs-v2";

package/dist/repo/shared_libs/play-runtime/execution-plan.ts

@@ -7,6 +7,7 @@ import {
 export const EXECUTION_PLAN_DEFAULTS = {
   inlineRowsLimit: 1_000,
   largeMapChunkSize: 5_000,
+  complexMapChunkSize: 1_000,
   workflowSoftStepBudget: 20_000,
   workflowHardStepBudget: 25_000,
   ingestStepCount: 1,
@@ -183,6 +184,10 @@ function extractPlanMaps(
     (substep): substep is Extract<PlayStaticSubstep, { type: 'waterfall' }> =>
       substep.type === 'waterfall',
   );
+  const fallbackStepSuites = substeps.filter(
+    (substep): substep is Extract<PlayStaticSubstep, { type: 'step_suite' }> =>
+      substep.type === 'step_suite',
+  );
   return substeps
     .filter(
       (substep): substep is Extract<PlayStaticSubstep, { type: 'map' }> =>
@@ -200,13 +205,33 @@ function extractPlanMaps(
         waterfallId: waterfall.id ?? waterfall.field,
         stageIds: waterfall.steps?.map((step) => step.id) ?? [],
       }));
+      const stepSuites = fallbackStepSuites.filter((stepSuite) => {
+        if (!mapSubstep.waterfallIds?.length) return true;
+        return mapSubstep.waterfallIds.includes(stepSuite.field);
+      });
+      const stepSuiteStepsPerChunk = stepSuites.reduce(
+        (max, stepSuite) => Math.max(max, stepSuite.steps.length),
+        0,
+      );
+      const waterfallStepsPerChunk = waterfallStages.reduce(
+        (max, waterfall) => Math.max(max, waterfall.stageIds.length),
+        0,
+      );
+      const stepsPerChunk = Math.max(
+        1,
+        waterfallStepsPerChunk,
+        stepSuiteStepsPerChunk,
+      );
       return {
         mapName: mapSubstep.name ?? mapSubstep.field,
         tableNamespace: mapSubstep.tableNamespace ?? mapSubstep.field,
         outputFields: mapSubstep.outputFields ?? [],
         waterfallStages,
-        defaultChunkSize: EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
-        stepsPerChunk: 1,
+        defaultChunkSize:
+          stepsPerChunk > 1
+            ? EXECUTION_PLAN_DEFAULTS.complexMapChunkSize
+            : EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
+        stepsPerChunk,
       };
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.22",
+  "version": "0.1.24",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {