deepline 0.1.191 → 0.1.193

package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts

@@ -12,6 +12,8 @@
  * is a coordinator-side safety net for missed signals.
  */
 
+import { isRetryableWorkerRuntimeApiError } from './runtime-api-retry';
+
 export type WorkflowStepLike = {
   waitForEvent: (
     name: string,
@@ -77,12 +79,6 @@ const CHILD_TERMINAL_DURABLE_VERIFY_GRACE_MS = 5_000;
 const CHILD_TERMINAL_DURABLE_VERIFY_POLL_MS = 250;
 const CHILD_TERMINAL_DURABLE_VERIFY_READ_TIMEOUT_MS = 3_000;
 const CHILD_TERMINAL_DURABLE_VERIFY_MIN_READ_TIMEOUT_MS = 2_000;
-const CHILD_TERMINAL_PERMANENT_RUNTIME_API_STATUS_RE =
-  /runtime API (400|401|403|404):/i;
-const CHILD_TERMINAL_RETRYABLE_RUNTIME_API_STATUS_RE =
-  /runtime API (408|429|500|502|503|504|530):/i;
-const CHILD_TERMINAL_RETRYABLE_ERROR_RE =
-  /timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT/i;
 
 export interface AwaitChildTerminalInput {
   parentRunId: string;
@@ -140,17 +136,6 @@ function terminalPayloadFromState(
   return isRecord(state?.data) ? state.data : null;
 }
 
-function isRetryableChildTerminalSnapshotError(error: unknown): boolean {
-  const message = error instanceof Error ? error.message : String(error);
-  if (CHILD_TERMINAL_PERMANENT_RUNTIME_API_STATUS_RE.test(message)) {
-    return false;
-  }
-  return (
-    CHILD_TERMINAL_RETRYABLE_RUNTIME_API_STATUS_RE.test(message) ||
-    CHILD_TERMINAL_RETRYABLE_ERROR_RE.test(message)
-  );
-}
-
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -259,7 +244,7 @@ async function readVerifiedChildTerminalPayload(input: {
       if (payload) return { payload, lastState, lastError };
     } catch (error) {
       lastError = error;
-      if (!isRetryableChildTerminalSnapshotError(error)) {
+      if (!isRetryableWorkerRuntimeApiError(error)) {
         throw error;
       }
     }

package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts

@@ -23,7 +23,10 @@ import {
 } from '@cloudflare/dynamic-workflows';
 import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
 import { buildChildRunId as buildSharedChildRunId } from '../../../shared_libs/play-runtime/child-run-id';
-import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
+import type {
+  PlayCallGovernanceSnapshot,
+  PlayRunInputPayload,
+} from '../../../shared_libs/play-runtime/scheduler-backend';
 import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runtime/test-runtime-seams';
 import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
 import {
@@ -96,7 +99,7 @@ export type PlayWorkflowParams = {
   artifactStorageKey: string;
   artifactHash: string;
   graphHash: string;
-  input: Record<string, unknown>;
+  input: PlayRunInputPayload;
   force?: boolean;
   runAttempt?: number;
   inputFile?: {
@@ -131,6 +134,8 @@ export type PlayWorkflowParams = {
   dynamicWorkerCode?: string | null;
   executorToken: string;
   baseUrl: string;
+  runtimeApiBaseUrl?: string | null;
+  callbackUrl?: string | null;
   integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
   orgId: string;
   userEmail: string;
@@ -402,6 +407,15 @@ function recordCoordinatorPerfTrace(event: CoordinatorPerfTraceInput): void {
   logCoordinatorPerfTrace(payload);
 }
 
+function safeOrigin(value: string | null | undefined): string | null {
+  if (!value?.trim()) return null;
+  try {
+    return new URL(value).origin;
+  } catch {
+    return '<invalid>';
+  }
+}
+
 async function appendCoordinatorPerfTrace(
   env: CoordinatorEnv,
   payload: CoordinatorPerfTracePayload,
@@ -1330,9 +1344,23 @@ async function markWorkflowRuntimeFailure(input: {
   const backoffMs = [200, 500, 1500];
   let lastError: unknown = null;
   for (let attempt = 0; attempt <= backoffMs.length; attempt += 1) {
+    const requestId = crypto.randomUUID();
     try {
+      headers.set('x-deepline-request-id', requestId);
       const response = await fetch(url, { method: 'POST', headers, body });
-      if (response.ok) return;
+      if (response.ok) {
+        if (attempt > 0) {
+          console.info('[coordinator.runtime_api.retry]', {
+            action: 'append_run_events',
+            status: 'ok',
+            attempt,
+            runId,
+            requestId,
+            baseOrigin: safeOrigin(baseUrl),
+          });
+        }
+        return;
+      }
       lastError = new Error(
         `runtime API responded ${response.status}: ${(await response.text().catch(() => '')).slice(0, 400)}`,
       );
@@ -1348,6 +1376,19 @@ async function markWorkflowRuntimeFailure(input: {
       lastError = error;
     }
     if (attempt < backoffMs.length) {
+      console.warn('[coordinator.runtime_api.retry]', {
+        action: 'append_run_events',
+        status: 'retry',
+        attempt: attempt + 1,
+        retryDelayMs: backoffMs[attempt],
+        runId,
+        requestId,
+        baseOrigin: safeOrigin(baseUrl),
+        error:
+          lastError instanceof Error
+            ? lastError.message.slice(0, 300)
+            : String(lastError).slice(0, 300),
+      });
       await new Promise((resolve) => setTimeout(resolve, backoffMs[attempt]));
     }
   }
@@ -2203,6 +2244,26 @@ async function callRuntimeApiFromCoordinator(input: {
   });
 
   const fetchStartedAt = Date.now();
+  const requestId = crypto.randomUUID();
+  const action =
+    body && typeof body === 'object' && !Array.isArray(body)
+      ? String((body as { action?: unknown }).action ?? 'unknown')
+      : 'unknown';
+  const runId =
+    body && typeof body === 'object' && !Array.isArray(body)
+      ? String(
+          (body as { runId?: unknown; playId?: unknown }).runId ??
+            (body as { runId?: unknown; playId?: unknown }).playId ??
+            '',
+        ) || null
+      : null;
+  console.info('[coordinator.runtime_api.breadcrumb]', {
+    phase: 'request',
+    action,
+    runId,
+    requestId,
+    baseOrigin: safeOrigin(input.baseUrl),
+  });
   const response = await input.env.HARNESS.runtimeApiCall({
     contract: PLAY_RUNTIME_CONTRACT,
     executorToken: input.executorToken,
@@ -2210,11 +2271,20 @@ async function callRuntimeApiFromCoordinator(input: {
     path: PLAY_RUNTIME_API_COMPAT_PATH,
     body,
     headers: {
-      'x-deepline-request-id': crypto.randomUUID(),
+      'x-deepline-request-id': requestId,
       [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
     },
   });
   recordTiming('coordinator.runtime_api.fetch', fetchStartedAt);
+  console.info('[coordinator.runtime_api.breadcrumb]', {
+    phase: 'response',
+    action,
+    runId,
+    requestId,
+    status: response.status,
+    ms: Date.now() - fetchStartedAt,
+    baseOrigin: safeOrigin(input.baseUrl),
+  });
 
   const bodyStartedAt = Date.now();
   const responseBody = response.body;
@@ -2329,7 +2399,7 @@ function buildChildRunId(input: {
         ? governance.parentPlayName
         : null,
     key: typeof governance?.key === 'string' ? governance.key : null,
-    input: isRecord(body.input) ? body.input : {},
+    input: isRecord(body.input) || Array.isArray(body.input) ? body.input : {},
     graphHash: isRecord(body.manifest)
       ? typeof body.manifest.graphHash === 'string'
         ? body.manifest.graphHash
@@ -2361,12 +2431,82 @@ function normalizeRuntimeBaseUrl(value: unknown): string | null {
   return parsed.toString().replace(/\/$/, '');
 }
 
-function resolveRuntimeBaseUrl(
+const DEEPLINE_PRODUCTION_APP_HOST = 'code.deepline.com';
+const DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE =
+  /^deepline-api(?:-[a-z0-9-]+)?-aero-team-0f650658\.vercel\.app$/i;
+const DEEPLINE_VERCEL_PROJECT_HOST_RE =
+  /^deepline-[a-z0-9]+-aero-team-0f650658\.vercel\.app$/i;
+const DEEPLINE_WORKERS_DEV_SUBDOMAINS = new Set(['chirag-d94']);
+
+function isDeeplineCoordinatorWorkerHost(hostname: string): boolean {
+  const labels = hostname.toLowerCase().split('.');
+  return (
+    labels.length === 4 &&
+    labels[0]?.startsWith('deepline-play-coordinator-') &&
+    labels[2] === 'workers' &&
+    labels[3] === 'dev' &&
+    DEEPLINE_WORKERS_DEV_SUBDOMAINS.has(labels[1] ?? '')
+  );
+}
+
+function shouldCanonicalizeRuntimeApiBaseUrl(input: {
+  configuredBaseUrl: URL;
+  overrideBaseUrl: URL;
+}): boolean {
+  if (input.overrideBaseUrl.host === input.configuredBaseUrl.host) {
+    return false;
+  }
+  const configuredHostname = input.configuredBaseUrl.hostname.toLowerCase();
+  const overrideHostname = input.overrideBaseUrl.hostname.toLowerCase();
+  if (isDeeplineCoordinatorWorkerHost(overrideHostname)) {
+    return true;
+  }
+  const overrideIsDeeplineVercelHost =
+    DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE.test(overrideHostname) ||
+    DEEPLINE_VERCEL_PROJECT_HOST_RE.test(overrideHostname);
+  if (!overrideIsDeeplineVercelHost) {
+    return false;
+  }
+  const configuredIsDeeplineVercelHost =
+    DEEPLINE_VERCEL_DEPLOYMENT_HOST_RE.test(configuredHostname) ||
+    DEEPLINE_VERCEL_PROJECT_HOST_RE.test(configuredHostname);
+  return (
+    configuredHostname === DEEPLINE_PRODUCTION_APP_HOST ||
+    !configuredIsDeeplineVercelHost
+  );
+}
+
+function resolveRuntimeApiBaseUrl(
+  env: CoordinatorEnv,
+  body: Record<string, unknown>,
+): string {
+  const configuredBaseUrl =
+    normalizeRuntimeBaseUrl(env.DEEPLINE_API_BASE_URL) ??
+    env.DEEPLINE_API_BASE_URL.replace(/\/$/, '');
+  const overrideBaseUrl =
+    normalizeRuntimeBaseUrl(body.runtimeApiBaseUrl) ??
+    normalizeRuntimeBaseUrl(body.baseUrl);
+  if (!overrideBaseUrl) {
+    return configuredBaseUrl;
+  }
+  if (
+    shouldCanonicalizeRuntimeApiBaseUrl({
+      configuredBaseUrl: new URL(configuredBaseUrl),
+      overrideBaseUrl: new URL(overrideBaseUrl),
+    })
+  ) {
+    return configuredBaseUrl;
+  }
+  return overrideBaseUrl;
+}
+
+function resolveWorkflowCallbackBaseUrl(
   env: CoordinatorEnv,
   body: Record<string, unknown>,
 ): string {
   return (
     normalizeRuntimeBaseUrl(body.callbackBaseUrl) ??
+    normalizeRuntimeBaseUrl(body.callbackUrl) ??
     normalizeRuntimeBaseUrl(body.baseUrl) ??
     env.DEEPLINE_API_BASE_URL.replace(/\/$/, '')
   );
@@ -2388,7 +2528,8 @@ function validateChildSubmitBody(input: {
   const { parentRunId, body } = input;
   const manifest = body.manifest as PlayRuntimeManifest | undefined;
   const governance = body.internalRunPlay as
-    PlayCallGovernanceSnapshot | undefined;
+    | PlayCallGovernanceSnapshot
+    | undefined;
   const childPlayName =
     typeof body.name === 'string' && body.name.trim()
       ? body.name.trim()
@@ -2480,7 +2621,8 @@ function buildChildWorkflowParams(input: {
     dynamicWorkerCode,
     preloadedDbSessions,
   } = input;
-  const baseUrl = resolveRuntimeBaseUrl(env, body);
+  const baseUrl = resolveRuntimeApiBaseUrl(env, body);
+  const callbackUrl = resolveWorkflowCallbackBaseUrl(env, body);
   return {
     runId: childRunId,
     playId: childRunId,
@@ -2488,7 +2630,7 @@ function buildChildWorkflowParams(input: {
     artifactStorageKey: manifest.artifactStorageKey,
     artifactHash: manifest.artifactHash,
     graphHash: manifest.graphHash,
-    input: isRecord(body.input) ? body.input : {},
+    input: isRecord(body.input) || Array.isArray(body.input) ? body.input : {},
     contractSnapshot: {
       source: 'published',
       revisionVersion: null,
@@ -2517,7 +2659,9 @@ function buildChildWorkflowParams(input: {
     preloadedDbSessions,
     dynamicWorkerCode,
     executorToken: childToken,
-    baseUrl,
+    baseUrl: callbackUrl,
+    runtimeApiBaseUrl: baseUrl,
+    callbackUrl,
     integrationMode: normalizeIntegrationMode(body.integrationMode),
     force: body.force === true || body.forceToolRefresh === true,
     forceToolRefresh: body.forceToolRefresh === true,
@@ -2545,9 +2689,9 @@ function runRequestFromPlayWorkflowParams(
     params.inputFile?.r2Key ?? params.inputFile?.storageKey ?? null;
   return {
     runId: params.runId,
-    callbackUrl: params.baseUrl,
+    callbackUrl: params.callbackUrl ?? params.baseUrl,
     executorToken: params.executorToken,
-    baseUrl: params.baseUrl,
+    baseUrl: params.runtimeApiBaseUrl ?? params.baseUrl,
     integrationMode: normalizeIntegrationMode(params.integrationMode),
     orgId: params.orgId,
     playName: params.playName,
@@ -2742,7 +2886,7 @@ async function executeChildInline(input: {
     input.body.parentPlayName.trim()
       ? input.body.parentPlayName.trim()
       : governance.parentPlayName;
-  const baseUrl = resolveRuntimeBaseUrl(input.env, input.body);
+  const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
   const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
     await prepareInlineChildRunWithRuntime({
       env: input.env,
@@ -3072,7 +3216,7 @@ async function submitChildWorkflowThroughCoordinator(input: {
     parentRunId: input.parentRunId,
     body: input.body,
   });
-  const baseUrl = resolveRuntimeBaseUrl(input.env, input.body);
+  const baseUrl = resolveRuntimeApiBaseUrl(input.env, input.body);
 
   const tokenStartedAt = Date.now();
   const childToken = await mintChildWorkflowExecutorToken({

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -65,6 +65,7 @@ import {
   type WorkflowStepLike,
 } from './child-play-await';
 import { submitChildPlayThroughCoordinator } from './child-play-submit';
+import { isRetryableWorkerRuntimeApiError } from './runtime-api-retry';
 import {
   attachToolResultListDataset,
   createToolExecuteResult,
@@ -307,6 +308,7 @@ import {
   type PreviousCell,
 } from '../../../shared_libs/play-runtime/cell-staleness';
 import type { PlayRowUpdate } from '../../../shared_libs/play-runtime/ctx-types';
+import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/scheduler-backend';
 
 // The play's default export. The bundler injects this — see bundle-play-file.ts.
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -323,7 +325,7 @@ type RunRequest = {
   playName: string;
   graphHash?: string | null;
   userEmail: string | null;
-  runtimeInput: Record<string, unknown>;
+  runtimeInput: PlayRunInputPayload;
   /** Start a fresh run graph and recompute runtime-sheet rows. */
   force?: boolean;
   /** Explicit durable tool receipt refresh that can re-execute completed provider calls. */
@@ -648,8 +650,15 @@ type WorkerEnv = {
 
 let cachedRuntimeApiBinding: WorkerEnv['RUNTIME_API'] | null = null;
 let cachedRuntimeApiVercelBypassToken: string | null = null;
+
+function runtimeApiBindingFromEnv(
+  env: WorkerEnv,
+): WorkerEnv['RUNTIME_API'] | null {
+  return env.RUNTIME_API ?? env.HARNESS ?? null;
+}
+
 function captureRuntimeApiBinding(env: WorkerEnv): void {
-  cachedRuntimeApiBinding = env.RUNTIME_API ?? null;
+  cachedRuntimeApiBinding = runtimeApiBindingFromEnv(env);
   cachedRuntimeApiVercelBypassToken =
     typeof env.VERCEL_PROTECTION_BYPASS_TOKEN === 'string' &&
     env.VERCEL_PROTECTION_BYPASS_TOKEN.trim()
@@ -831,7 +840,9 @@ async function fetchRuntimeApi(
       signal: controller.signal,
     };
     if (!cachedRuntimeApiBinding) {
-      throw new Error('[play-harness] RUNTIME_API service binding is required');
+      throw new Error(
+        '[play-harness] RUNTIME_API/HARNESS service binding is required',
+      );
     }
     if (abortSignal?.aborted) {
       throw abortError();
@@ -1360,7 +1371,7 @@ async function postRuntimeApi<T>(
       lastError = error;
       if (
         attempt >= RUNTIME_API_RETRY_DELAYS_MS.length ||
-        !isRetryableRuntimeApiError(error)
+        !isRetryableWorkerRuntimeApiError(error)
       ) {
         throw error;
       }
@@ -1386,13 +1397,6 @@ async function postRuntimeApi<T>(
   throw lastError instanceof Error ? lastError : new Error(String(lastError));
 }
 
-function isRetryableRuntimeApiError(error: unknown): boolean {
-  const message = error instanceof Error ? error.message : String(error);
-  return /timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT/i.test(
-    message,
-  );
-}
-
 function isRetryableRuntimePersistenceError(error: unknown): boolean {
   const message = error instanceof Error ? error.message : String(error);
   return /deadlock detected|Runtime Postgres connection timed out|could not serialize access|tuple concurrently updated/i.test(
@@ -1401,21 +1405,17 @@ function isRetryableRuntimePersistenceError(error: unknown): boolean {
 }
 
 function isRetryableRuntimeApiResponse(status: number, body: string): boolean {
-  if (
+  return (
     status === 408 ||
     status === 429 ||
-    status === 502 ||
-    status === 503 ||
-    status === 504 ||
-    status === 530
-  ) {
-    return true;
-  }
-  return (
-    status === 500 &&
-    /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|UND_ERR_CONNECT_TIMEOUT/i.test(
-      body,
-    )
+    (status >= 502 && status <= 504) ||
+    status === 530 ||
+    (status === 404 &&
+      /DEPLOYMENT_NOT_FOUND|DeploymentNotFound|deployment .*not found|requested deployment .*exist/i.test(
+        body,
+      )) ||
+    (status === 500 &&
+      isRetryableWorkerRuntimeApiError(`runtime API ${status}: ${body}`))
   );
 }
 
@@ -2076,7 +2076,7 @@ async function callToolDirect(
       );
       if (
         transportAttempt >= TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS ||
-        !isRetryableRuntimeApiError(error)
+        !isRetryableWorkerRuntimeApiError(error)
       ) {
         throw lastError;
       }
@@ -2360,7 +2360,10 @@ function isRuntimeReceiptPersistenceError(error: unknown): boolean {
 }
 
 function isRunFatalWorkerRowError(error: unknown): boolean {
+  const failure = normalizePlayRunFailure(error);
   return (
+    failure.code === 'PLATFORM_DEPLOY_INTERRUPTED' ||
+    failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED' ||
     isCoordinatorProgressAuthorizationError(error) ||
     isRowIsolationExemptError(error) ||
     isRuntimeReceiptPersistenceError(error) ||
@@ -6891,7 +6894,7 @@ function createMinimalWorkerCtx(
                   childIdempotencyKey: receiptKey,
                   input: isRecord(input) ? input : {},
                   orgId: req.orgId,
-                  callbackBaseUrl: req.callbackUrl,
+                  callbackUrl: req.callbackUrl,
                   baseUrl: req.baseUrl,
                   integrationMode: req.integrationMode ?? null,
                   parentExecutorToken: req.executorToken,
@@ -7381,10 +7384,7 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
       await probeHarnessOnce(env, runPrefix);
       const ctx = createMinimalWorkerCtx(req, emit, env);
       const result = await (
-        playFn as (
-          ctx: unknown,
-          input: Record<string, unknown>,
-        ) => Promise<unknown>
+        playFn as (ctx: unknown, input: PlayRunInputPayload) => Promise<unknown>
       )(ctx, req.runtimeInput);
       emit({
         type: 'result',
@@ -7913,10 +7913,7 @@ async function executeRunRequest(
   try {
     const playStartedAt = nowMs();
     const result = await (
-      playFn as (
-        ctx: unknown,
-        input: Record<string, unknown>,
-      ) => Promise<unknown>
+      playFn as (ctx: unknown, input: PlayRunInputPayload) => Promise<unknown>
     )(ctx, req.runtimeInput);
     recordRunnerPerfTrace({
       req,
@@ -8305,9 +8302,9 @@ function runRequestFromWorkflowParams(
         : null;
   return {
     runId: String(params.runId ?? ''),
-    callbackUrl: String(params.baseUrl ?? ''),
+    callbackUrl: String(params.callbackUrl ?? params.baseUrl ?? ''),
     executorToken: String(params.executorToken ?? ''),
-    baseUrl: String(params.baseUrl ?? ''),
+    baseUrl: String(params.runtimeApiBaseUrl ?? params.baseUrl ?? ''),
     integrationMode: normalizeIntegrationMode(params.integrationMode),
     orgId: String(params.orgId ?? ''),
     playName: String(params.playName ?? ''),
@@ -8315,9 +8312,8 @@ function runRequestFromWorkflowParams(
     force: params.force === true,
     forceToolRefresh: params.forceToolRefresh === true,
     runAttempt: normalizeWorkerRunAttempt(params.runAttempt),
-    runtimeInput: isRecord(params.input)
-      ? (params.input as Record<string, unknown>)
-      : {},
+    runtimeInput:
+      isRecord(params.input) || Array.isArray(params.input) ? params.input : {},
     inlineCsv: isInlineCsv(params.inlineCsv) ? params.inlineCsv : null,
     inputFiles:
       inputFile && inputStorageKey

package/dist/bundling-sources/apps/play-runner-workers/src/runtime-api-retry.ts

@@ -0,0 +1,23 @@
+const PERMANENT_RUNTIME_API_STATUS_RE = /runtime API 40[0134]:/i;
+const RETRYABLE_RUNTIME_API_ERROR_RE =
+  /runtime API (408|429|500|50[2-4]|530):|time(?:d out|out)|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|The deployment could not be found on Vercel|Authentication infrastructure/i;
+
+function isVercelDeploymentMissingRuntimeApi404(message: string): boolean {
+  return (
+    /runtime API 404:/i.test(message) &&
+    /DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|The deployment could not be found on Vercel/i.test(
+      message,
+    )
+  );
+}
+
+export function isRetryableWorkerRuntimeApiError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  if (
+    PERMANENT_RUNTIME_API_STATUS_RE.test(message) &&
+    !isVercelDeploymentMissingRuntimeApi404(message)
+  ) {
+    return false;
+  }
+  return RETRYABLE_RUNTIME_API_ERROR_RE.test(message);
+}

package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts

@@ -1,5 +1,8 @@
 import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
-import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
+import type {
+  PlayCallGovernanceSnapshot,
+  PlayRunInputPayload,
+} from '../../../shared_libs/play-runtime/scheduler-backend';
 import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
 import type {
   PlayRuntimeManifest,
@@ -33,7 +36,7 @@ export type WorkflowRetryPlayParams = {
   artifactStorageKey: string;
   artifactHash: string;
   graphHash: string;
-  input: Record<string, unknown>;
+  input: PlayRunInputPayload;
   inputFile?: {
     name?: string;
     r2Key?: string;
@@ -65,6 +68,8 @@ export type WorkflowRetryPlayParams = {
   dynamicWorkerCode?: string | null;
   executorToken: string;
   baseUrl: string;
+  runtimeApiBaseUrl?: string | null;
+  callbackUrl?: string | null;
   orgId: string;
   userEmail: string;
   userId?: string | null;

package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts

@@ -24,14 +24,19 @@ export function decideWorkflowPlatformRetry(input: {
     return { action: 'fail', reason: null, message: null };
   }
   const failure = normalizePlayRunFailure(input.error);
-  const retryablePlatformReset = failure.code === 'PLATFORM_DEPLOY_INTERRUPTED';
+  const retryablePlatformReset =
+    failure.code === 'PLATFORM_DEPLOY_INTERRUPTED' ||
+    failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED';
   if (
     retryablePlatformReset &&
     input.retryAttempts < PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT
   ) {
     return {
       action: 'retry',
-      reason: 'cloudflare_durable_object_reset',
+      reason:
+        failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
+          ? 'cloudflare_worker_subrequest_limit'
+          : 'cloudflare_durable_object_reset',
       nextAttempt: input.retryAttempts + 1,
       message: failure.message,
     };
@@ -39,7 +44,9 @@ export function decideWorkflowPlatformRetry(input: {
   return {
     action: 'fail',
     reason: retryablePlatformReset
-      ? 'cloudflare_durable_object_reset_retry_exhausted'
+      ? failure.code === 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED'
+        ? 'cloudflare_worker_subrequest_limit_retry_exhausted'
+        : 'cloudflare_durable_object_reset_retry_exhausted'
       : null,
     message: failure.message,
   };

package/dist/bundling-sources/sdk/src/release.ts

@@ -105,10 +105,10 @@ export const SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: '0.1.191',
+  version: '0.1.193',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.191',
+    latest: '0.1.193',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts

@@ -49,7 +49,7 @@ export interface ChildRunIdInputs {
    */
   runPlaySemanticKey?: string | null;
   /** Child input payload (semantic-fallback input). */
-  input?: Record<string, unknown> | null;
+  input?: unknown;
   /**
    * Compiled child graph hash. Only present on workers_edge; `null` in-process.
    */
@@ -89,11 +89,16 @@ export function buildChildRunId(inputs: ChildRunIdInputs): string {
     stableStringify({
       parentRunId: inputs.parentRunId,
       parentPlayName:
-        typeof inputs.parentPlayName === 'string' ? inputs.parentPlayName : null,
+        typeof inputs.parentPlayName === 'string'
+          ? inputs.parentPlayName
+          : null,
       childPlayName: inputs.childPlayName,
       key: typeof inputs.key === 'string' ? inputs.key : null,
       ...(runPlaySemanticKey ? { runPlaySemanticKey } : {}),
-      input: isRecord(inputs.input) ? inputs.input : {},
+      input:
+        isRecord(inputs.input) || Array.isArray(inputs.input)
+          ? inputs.input
+          : {},
       graphHash: typeof inputs.graphHash === 'string' ? inputs.graphHash : null,
     });
   const digest = sha256Hex(semanticKey).slice(0, 24);

package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts

@@ -11,6 +11,7 @@ import {
   isPlayExecutionSuspendedError,
   isPlayRowExecutionSuspendedError,
 } from './suspension';
+import { getToolHttpErrorReceiptFailureKind } from './tool-http-errors';
 import type { WorkReceiptFailureKind } from './work-receipts';
 import {
   PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
@@ -48,6 +49,8 @@ function isInFlightRuntimeReceipt(
 export function runtimeReceiptFailureKindForError(
   error: unknown,
 ): WorkReceiptFailureKind {
+  const toolFailureKind = getToolHttpErrorReceiptFailureKind(error);
+  if (toolFailureKind) return toolFailureKind;
   return isPlayExecutionSuspendedError(error) ||
     isPlayRowExecutionSuspendedError(error)
     ? 'repairable'

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -1,8 +1,12 @@
 const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
-  /Durable Object.*(?:code was updated|storage caused object)/;
+  /Durable Object.*(?:code (?:was|has been) updated|storage caused object)/;
+const CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE =
+  /Too many subrequests by single Worker invocation/i;
 
 export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
   'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.';
+export const PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE =
+  'Worker subrequest limit hit. Deepline retries automatically; if still visible, re-run.';
 
 export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
   'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
@@ -64,6 +68,15 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
       cause,
     };
   }
+  if (CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE.test(cause)) {
+    return {
+      code: 'PLATFORM_WORKER_SUBREQUEST_INTERRUPTED',
+      phase: 'runtime',
+      message: PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE,
+      retryable: true,
+      cause,
+    };
+  }
   const playDepthBudgetMatch = cause.match(
     /Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./,
   );

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -36,6 +36,8 @@ export const PLAY_SCHEDULER_BACKENDS = {
 export type PlaySchedulerBackendId =
   (typeof PLAY_SCHEDULER_BACKENDS)[keyof typeof PLAY_SCHEDULER_BACKENDS];
 
+export type PlayRunInputPayload = Record<string, unknown> | unknown[];
+
 export type PlayCallGovernanceSnapshot = {
   rootRunId: string;
   parentRunId: string;
@@ -83,7 +85,7 @@ export type PlaySchedulerSubmitInput = {
   runtimeArtifact?: unknown;
   artifactHash: string;
   graphHash: string;
-  input: Record<string, unknown>;
+  input: PlayRunInputPayload;
   /** Start a fresh run graph and recompute runtime-sheet rows. */
   force?: boolean;
   /** Explicit cache bypass for completed ctx.tools.execute receipts. */
@@ -128,6 +130,11 @@ export type PlaySchedulerSubmitInput = {
   /** Optional immutable Worker module source for local Dynamic Worker loading. */
   dynamicWorkerCode?: string | null;
   executorToken: string;
+  /**
+   * Public app origin used by Workers to call Deepline runtime API routes.
+   * When omitted, legacy schedulers fall back to baseUrl.
+   */
+  runtimeApiBaseUrl?: string | null;
   baseUrl: string;
   /** Optional per-run provider execution mode. Defaults to server policy when omitted. */
   integrationMode?: 'live' | 'eval_stub' | 'fixture';

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -204,9 +204,7 @@ export function classifyToolExecuteHttpFailure(input: {
   });
   const shouldRetry =
     retryDecision.retryable && input.attempt < retryDecision.attemptCap;
-  error.receiptFailureKind = retryDecision.retryable
-    ? 'repairable'
-    : 'terminal';
+  if (retryDecision.retryable) error.receiptFailureKind = 'repairable';
   const retryAfterMs = parseToolExecuteRetryAfterMs(
     input.retryAfterHeader,
     input.nowMs,

package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts

@@ -4,13 +4,13 @@ export class ToolHttpError extends Error {
   readonly billing: Record<string, unknown> | null;
   /** HTTP status of the failed tool-execute response (e.g. 429, 502). */
   readonly status: number;
-  receiptFailureKind: WorkReceiptFailureKind | null;
+  receiptFailureKind: WorkReceiptFailureKind;
 
   constructor(
     message: string,
     billing: Record<string, unknown> | null,
     status: number,
-    receiptFailureKind: WorkReceiptFailureKind | null = null,
+    receiptFailureKind: WorkReceiptFailureKind = 'terminal',
   ) {
     super(message);
     this.name = 'ToolHttpError';

package/dist/cli/index.js

@@ -623,10 +623,10 @@ var SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: "0.1.191",
+  version: "0.1.193",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.191",
+    latest: "0.1.193",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [
@@ -19385,6 +19385,30 @@ function coalesceExportableSheetRows(rows, sourceRowStart = 0) {
     bySourceRowIndex.set(rowIndex, next);
     mergedRows.push(next);
   }
+  if (bySourceRowIndex.size > 0) {
+    mergedRows.sort((left, right) => {
+      const leftIndex = sourceRowIndexFromEnrichRow(
+        left,
+        sourceRowStart,
+        Number.MAX_SAFE_INTEGER
+      );
+      const rightIndex = sourceRowIndexFromEnrichRow(
+        right,
+        sourceRowStart,
+        Number.MAX_SAFE_INTEGER
+      );
+      if (leftIndex !== null && rightIndex !== null) {
+        return leftIndex - rightIndex;
+      }
+      if (leftIndex !== null) {
+        return -1;
+      }
+      if (rightIndex !== null) {
+        return 1;
+      }
+      return 0;
+    });
+  }
   return mergedRows;
 }
 function sourceRowIndexFromEnrichRow(row, start, end) {
@@ -21227,7 +21251,9 @@ function registerEnrichCommand(program) {
               ...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
             });
             if (chunk.exportResult) {
-              currentChunkCommittedExportResult = await commitInPlaceOutput(chunk.exportResult);
+              currentChunkCommittedExportResult = await commitInPlaceOutput(
+                chunk.exportResult
+              );
               finalExportResult = currentChunkCommittedExportResult;
             }
             const reportedExportResult = currentChunkCommittedExportResult ?? finalExportResult;

package/dist/cli/index.mjs

@@ -608,10 +608,10 @@ var SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: "0.1.191",
+  version: "0.1.193",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.191",
+    latest: "0.1.193",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [
@@ -19414,6 +19414,30 @@ function coalesceExportableSheetRows(rows, sourceRowStart = 0) {
     bySourceRowIndex.set(rowIndex, next);
     mergedRows.push(next);
   }
+  if (bySourceRowIndex.size > 0) {
+    mergedRows.sort((left, right) => {
+      const leftIndex = sourceRowIndexFromEnrichRow(
+        left,
+        sourceRowStart,
+        Number.MAX_SAFE_INTEGER
+      );
+      const rightIndex = sourceRowIndexFromEnrichRow(
+        right,
+        sourceRowStart,
+        Number.MAX_SAFE_INTEGER
+      );
+      if (leftIndex !== null && rightIndex !== null) {
+        return leftIndex - rightIndex;
+      }
+      if (leftIndex !== null) {
+        return -1;
+      }
+      if (rightIndex !== null) {
+        return 1;
+      }
+      return 0;
+    });
+  }
   return mergedRows;
 }
 function sourceRowIndexFromEnrichRow(row, start, end) {
@@ -21256,7 +21280,9 @@ function registerEnrichCommand(program) {
               ...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
             });
             if (chunk.exportResult) {
-              currentChunkCommittedExportResult = await commitInPlaceOutput(chunk.exportResult);
+              currentChunkCommittedExportResult = await commitInPlaceOutput(
+                chunk.exportResult
+              );
               finalExportResult = currentChunkCommittedExportResult;
             }
             const reportedExportResult = currentChunkCommittedExportResult ?? finalExportResult;

package/dist/index.js

@@ -422,10 +422,10 @@ var SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: "0.1.191",
+  version: "0.1.193",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.191",
+    latest: "0.1.193",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [

package/dist/index.mjs

@@ -352,10 +352,10 @@ var SDK_RELEASE = {
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
-  version: "0.1.191",
+  version: "0.1.193",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.191",
+    latest: "0.1.193",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.191",
+  "version": "0.1.193",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {