deepline 0.1.198 → 0.1.200

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -38,6 +38,7 @@ import {
 } from '@shared_libs/play-runtime/runtime-contract';
 import { PLAY_RUNTIME_API_COMPAT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
 import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
+import { vercelProtectionBypassHeaders } from '@shared_libs/play-runtime/vercel-protection';
 
 export type StoredPlayArtifactPayload = {
   sourceCode: string;
@@ -65,7 +66,7 @@ export type RuntimeStatusUpdate = {
   liveLogTotalCount?: number;
   /**
    * Explicit terminal-output replay for final runner logs when the caller must
-   * keep `status` nonterminal. Hatchet/Postgres progress finalizers do this so
+   * keep `status` nonterminal. Direct-worker/Postgres progress finalizers do this so
    * scheduler projection remains the only terminal run writer, while the Run
    * Log Stream still gets terminal transport recovery semantics.
    */
@@ -85,10 +86,6 @@ export type RuntimeSaveResults = {
     error?: string;
     publicResult?: Record<string, unknown> | null;
     maxCreditsPerRun?: number | null;
-    temporalActionEstimate?: {
-      totalActions?: number;
-      [key: string]: unknown;
-    } | null;
   };
   userId?: string | null;
 };
@@ -127,7 +124,6 @@ type RuntimeApiRequest =
       maxCreditsPerRun?: number | null;
       staticPipeline?: unknown;
       source?: 'published' | 'ad_hoc' | 'draft';
-      activeRunLimit?: number | null;
     }
   | ({
       action: 'save_results';
@@ -136,7 +132,8 @@ type RuntimeApiRequest =
       action: 'apply_row_updates';
       playName: string;
       tableNamespace: string;
-      sheetContract: PlaySheetContract;
+      sheetContract?: PlaySheetContract | null;
+      contractSnapshot?: unknown;
       runId: string;
       userEmail: string;
       updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
@@ -249,6 +246,36 @@ type RuntimeApiRequest =
       forceRefresh?: boolean;
       forceFailedRefresh?: boolean;
     }
+  | {
+      action: 'mark_runtime_step_receipt_running';
+      playName: string;
+      runId: string;
+      key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
+    }
+  | {
+      action: 'mark_runtime_step_receipts_running';
+      playName: string;
+      runId: string;
+      receipts: Array<{
+        key: string;
+        runId: string;
+        runAttempt?: number | null;
+        leaseId?: string | null;
+      }>;
+    }
+  | {
+      action: 'mark_runtime_step_receipts_queued';
+      playName: string;
+      runId: string;
+      receipts: Array<{
+        key: string;
+        runId: string;
+        runAttempt?: number | null;
+        leaseId?: string | null;
+      }>;
+    }
   | {
       action: 'complete_runtime_step_receipts';
       playName: string;
@@ -304,7 +331,6 @@ const runStatusLedgerSnapshots = new Map<string, PlayRunLedgerSnapshot>();
 // positionally (exactly-once on the channel, repeated identical lines kept).
 const runLogChannelSentCounts = new Map<string, number>();
 const runStatusUpdateChains = new Map<string, Promise<void>>();
-const vercelProtectionCookieCache = new Map<string, Promise<string | null>>();
 
 function isTerminalRuntimeStatus(status: string): boolean {
   return (
@@ -353,12 +379,53 @@ function isRetryableAppRuntimeResponse(input: {
   }
   return (
     input.status === 429 ||
+    (input.status >= 520 && input.status <= 524) ||
     input.status === 502 ||
     input.status === 503 ||
     input.status === 504
   );
 }
 
+function summarizeAppRuntimeErrorBody(body: string): string {
+  const trimmed = body.trim();
+  if (!trimmed) {
+    return 'empty response body';
+  }
+  if (/<!doctype\s+html|<html[\s>]/i.test(trimmed)) {
+    const title = trimmed.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+    const normalizedTitle = title
+      ?.replace(/<[^>]+>/g, ' ')
+      .replace(/\s+/g, ' ')
+      .trim();
+    return normalizedTitle
+      ? `HTML error page: ${normalizedTitle}`
+      : 'HTML error page';
+  }
+  try {
+    const parsed = JSON.parse(trimmed) as unknown;
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+      const record = parsed as Record<string, unknown>;
+      const message =
+        typeof record.debug_error === 'string'
+          ? record.debug_error
+          : typeof record.details === 'string'
+            ? record.details
+            : typeof record.error === 'string'
+              ? record.error
+              : typeof record.message === 'string'
+                ? record.message
+                : null;
+      if (message?.trim()) {
+        return message.trim();
+      }
+    }
+  } catch {}
+  if (/<[A-Za-z][\s\S]*>/.test(trimmed)) {
+    return 'HTML error page';
+  }
+  return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
+}
+
 function isTransientAppRuntimeFailureBody(body: string): boolean {
   return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated/i.test(
     body,
@@ -384,20 +451,32 @@ function isRetryableAppRuntimeAction(
 ): boolean {
   return (
     action === 'append_run_events' ||
+    action === 'apply_row_updates' ||
+    action === 'compute_billing_finalize' ||
+    action === 'compute_billing_record_item' ||
+    action === 'compute_billing_upsert' ||
     action === 'create_db_session' ||
     action === 'create_signed_artifact_url' ||
     action === 'create_signed_staged_file_url' ||
     action === 'get_runtime_step_receipt' ||
     action === 'get_runtime_step_receipts' ||
+    action === 'heartbeat_runtime_step_receipts' ||
     action === 'claim_runtime_step_receipt' ||
     action === 'claim_runtime_step_receipts' ||
+    action === 'mark_runtime_step_receipt_running' ||
+    action === 'mark_runtime_step_receipts_running' ||
+    action === 'mark_runtime_step_receipts_queued' ||
     action === 'complete_runtime_step_receipt' ||
     action === 'complete_runtime_step_receipts' ||
     action === 'fail_runtime_step_receipt' ||
     action === 'fail_runtime_step_receipts' ||
     action === 'rate_state_acquire' ||
     action === 'rate_state_release' ||
-    action === 'rate_state_penalize'
+    action === 'rate_state_penalize' ||
+    action === 'release_runtime_step_receipt' ||
+    action === 'save_results' ||
+    action === 'skip_runtime_step_receipt' ||
+    action === 'start_inline_child_run'
   );
 }
 
@@ -420,62 +499,38 @@ function appRuntimeRetryDelayMs(
   return appRuntimeRetryDelaysForAction(action)[attempt - 1] ?? 1_000;
 }
 
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
-  const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
-  const url = new URL(`${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`);
-  const bypassToken = context.vercelProtectionBypassToken?.trim();
-  if (bypassToken) {
-    url.searchParams.set('x-vercel-set-bypass-cookie', 'true');
-    url.searchParams.set('x-vercel-protection-bypass', bypassToken);
-  }
-  return url.toString();
-}
+function appRuntimeRetryAfterMs(
+  response: Response,
+  body: string,
+): number | null {
+  try {
+    const parsed = JSON.parse(body) as unknown;
+    if (
+      parsed &&
+      typeof parsed === 'object' &&
+      !Array.isArray(parsed) &&
+      typeof (parsed as Record<string, unknown>).retry_after_ms === 'number'
+    ) {
+      return Math.max(
+        0,
+        Math.min((parsed as Record<string, number>).retry_after_ms, 30_000),
+      );
+    }
+  } catch {}
 
-function setCookieHeaders(headers: Headers): string[] {
-  const getter = (headers as Headers & { getSetCookie?: () => string[] })
-    .getSetCookie;
-  if (typeof getter === 'function') {
-    return getter.call(headers);
-  }
-  const single = headers.get('set-cookie');
-  return single ? [single] : [];
+  const retryAfterSeconds = Number(response.headers.get('retry-after'));
+  return Number.isFinite(retryAfterSeconds)
+    ? Math.max(0, Math.min(retryAfterSeconds * 1_000, 30_000))
+    : null;
 }
 
-function cookieHeaderFromSetCookie(headers: Headers): string | null {
-  const cookies = setCookieHeaders(headers)
-    .flatMap((header) => header.split(/,(?=\s*[^;,=]+=[^;,]+)/g))
-    .map((header) => header.split(';', 1)[0]?.trim() ?? '')
-    .filter(Boolean);
-  return cookies.length > 0 ? cookies.join('; ') : null;
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-function resolveVercelProtectionCookie(
-  context: WorkerRuntimeApiContext,
-): Promise<string | null> {
-  const bypassToken = context.vercelProtectionBypassToken?.trim();
-  if (!bypassToken) return Promise.resolve(null);
-
+function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
   const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
-  const cacheKey = `${baseUrl}\n${bypassToken}`;
-  const cached = vercelProtectionCookieCache.get(cacheKey);
-  if (cached) return cached;
-
-  const promise = (async () => {
-    const url = new URL(`${baseUrl}/api/v2/health`);
-    url.searchParams.set('x-vercel-set-bypass-cookie', 'true');
-    url.searchParams.set('x-vercel-protection-bypass', bypassToken);
-    const runtimeFetch = context.fetch ?? fetch;
-    const response = await runtimeFetch(url.toString(), {
-      headers: { 'x-vercel-protection-bypass': bypassToken },
-    }).catch(() => null);
-    return response ? cookieHeaderFromSetCookie(response.headers) : null;
-  })();
-  vercelProtectionCookieCache.set(cacheKey, promise);
-  return promise;
+  return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
 }
 
 async function postAppRuntimeApi<TResponse>(
@@ -490,33 +545,31 @@ async function postAppRuntimeApi<TResponse>(
   if (!token) {
     throw new Error('Worker runtime API requires executorToken.');
   }
-  const vercelProtectionCookie = await resolveVercelProtectionCookie(context);
+  const runtimeFetch = context.fetch ?? fetch;
+  const vercelHeaders = await vercelProtectionBypassHeaders({
+    baseUrl,
+    token: context.vercelProtectionBypassToken,
+    fetchImpl: runtimeFetch,
+  });
 
   const maxAttempts = appRuntimeMaxAttempts(body.action);
 
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     let response: Response;
     try {
-      const runtimeFetch = context.fetch ?? fetch;
       response = await runtimeFetch(resolveAppRuntimeApiUrl(context), {
         method: 'POST',
         headers: {
           'content-type': 'application/json',
           authorization: `Bearer ${token}`,
           [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
-          ...(context.vercelProtectionBypassToken?.trim()
-            ? {
-                'x-vercel-protection-bypass':
-                  context.vercelProtectionBypassToken.trim(),
-              }
-            : {}),
+          ...vercelHeaders,
           ...(context.runtimeTestFaultHeader?.trim()
             ? {
                 [PLAY_RUNTIME_TEST_FAULT_HEADER]:
                   context.runtimeTestFaultHeader.trim(),
               }
             : {}),
-          ...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
         },
         body: JSON.stringify(body),
       });
@@ -542,11 +595,16 @@ async function postAppRuntimeApi<TResponse>(
         body: responseText,
       })
     ) {
-      await sleep(appRuntimeRetryDelayMs(body.action, attempt));
+      await sleep(
+        Math.max(
+          appRuntimeRetryDelayMs(body.action, attempt),
+          appRuntimeRetryAfterMs(response, responseText) ?? 0,
+        ),
+      );
       continue;
     }
     throw new Error(
-      `App runtime API ${body.action} failed with status ${response.status}: ${responseText}`,
+      `App runtime API ${body.action} failed with status ${response.status}: ${summarizeAppRuntimeErrorBody(responseText)}`,
     );
   }
 
@@ -774,13 +832,13 @@ async function updateRunStatusViaAppRuntimeUnlocked(
     },
     previousSnapshot,
     now: update.lastCheckpointAt ?? Date.now(),
-    source: 'temporal',
+    source: 'worker',
   });
   if (terminalLogReplay && terminalLogReplay.lines.length > 0) {
     events.push(
       ...buildTerminalLogReplayEvents({
         runId: update.playId,
-        source: 'temporal',
+        source: 'worker',
         occurredAt: update.lastCheckpointAt ?? Date.now(),
         lines: terminalLogReplay.lines,
         liveLogTotalCount: terminalLogReplay.totalCount,
@@ -834,7 +892,6 @@ export async function startRunViaAppRuntime(
     maxCreditsPerRun?: number | null;
     staticPipeline?: unknown;
     source?: 'published' | 'ad_hoc' | 'draft';
-    activeRunLimit?: number | null;
   },
 ): Promise<void> {
   await postAppRuntimeApi<{ ok: true }>(context, {
@@ -969,6 +1026,48 @@ export async function claimRuntimeStepReceiptsViaAppRuntime(
   });
 }
 
+export async function markRuntimeStepReceiptRunningViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<RuntimeApiRequest, { action: 'mark_runtime_step_receipt_running' }>,
+    'action'
+  >,
+): Promise<RuntimeStepReceipt | null> {
+  return await postAppRuntimeApi<RuntimeStepReceipt | null>(context, {
+    action: 'mark_runtime_step_receipt_running',
+    ...input,
+  });
+}
+
+export async function markRuntimeStepReceiptsRunningViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<
+      RuntimeApiRequest,
+      { action: 'mark_runtime_step_receipts_running' }
+    >,
+    'action'
+  >,
+): Promise<Array<RuntimeStepReceipt | null>> {
+  return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
+    action: 'mark_runtime_step_receipts_running',
+    ...input,
+  });
+}
+
+export async function markRuntimeStepReceiptsQueuedViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<RuntimeApiRequest, { action: 'mark_runtime_step_receipts_queued' }>,
+    'action'
+  >,
+): Promise<Array<RuntimeStepReceipt | null>> {
+  return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
+    action: 'mark_runtime_step_receipts_queued',
+    ...input,
+  });
+}
+
 export async function completeRuntimeStepReceiptViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: Omit<