deepline 0.1.227 → 0.1.229

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -129,13 +129,10 @@ type RuntimeApiRequest =
       idempotencyKey?: string;
     }
   | {
-      action: 'start_inline_child_run';
+      action: 'start_run';
       idempotencyKey?: string;
       playName: string;
       runId: string;
-      parentRunId?: string | null;
-      rootRunId?: string | null;
-      workflowFamilyKey?: string | null;
       artifactStorageKey?: string | null;
       artifactHash?: string | null;
       graphHash?: string | null;
@@ -380,6 +377,8 @@ function applyRetryJitter(delayMs: number): number {
   return Math.round(half + Math.random() * half);
 }
 const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+const APP_RUNTIME_RECEIPT_RETRY_TELEMETRY_TAG =
+  '[perf][worker.receipt_api.transport]';
 const RUN_STATUS_LEDGER_SNAPSHOT_CACHE_LIMIT = 1_000;
 const runStatusLedgerSnapshots = new Map<string, PlayRunLedgerSnapshot>();
 // Positional log cursor per run attempt: count of liveLogs lines already
@@ -454,7 +453,7 @@ async function appendRunEventBatchesViaAppRuntime(
   input: { playId: string; events: readonly PlayRunLedgerEvent[] },
 ): Promise<void> {
   for (const events of partitionRunEventsForAppRuntime(input.events)) {
-    await appendRunEventsViaAppRuntime(context, {
+    await appendRunEventsViaAppRuntimeRaw(context, {
       playId: input.playId,
       events,
     });
@@ -504,6 +503,20 @@ function isRetryableAppRuntimeResponse(input: {
   if (!isRetryableAppRuntimeAction(input.action)) {
     return false;
   }
+  // The app runtime may explicitly classify an otherwise-client-error status
+  // as transient. Keep the structured delivery contract authoritative rather
+  // than reducing every 4xx to permanent at the worker boundary.
+  try {
+    const parsed = JSON.parse(input.body) as unknown;
+    if (
+      parsed &&
+      typeof parsed === 'object' &&
+      !Array.isArray(parsed) &&
+      (parsed as { retryable?: unknown }).retryable === true
+    ) {
+      return true;
+    }
+  } catch {}
   if (
     input.action === 'append_run_events' &&
     input.status === 500 &&
@@ -583,7 +596,7 @@ function appRuntimeErrorCode(response: Response, body: string): string | null {
 }
 
 export function isTransientAppRuntimeFailureBody(body: string): boolean {
-  return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open|OptimisticConcurrencyControlFailure|changed while this mutation was being run|Documents read from or written to the .* table changed while this mutation/i.test(
+  return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_(?:SATURATED|UNAVAILABLE)|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open|OptimisticConcurrencyControlFailure|changed while this mutation was being run|Documents read from or written to the .* table changed while this mutation/i.test(
     body,
   );
 }
@@ -657,7 +670,7 @@ function isRetryableAppRuntimeAction(
     action === 'release_runtime_step_receipt' ||
     action === 'save_results' ||
     action === 'skip_runtime_step_receipt' ||
-    action === 'start_inline_child_run'
+    action === 'start_run'
   );
 }
 
@@ -733,14 +746,146 @@ function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+type AppRuntimeRetryFailureKind =
+  | 'connection_refused'
+  | 'connection_reset'
+  | 'connection_timeout'
+  | 'dns'
+  | 'fetch_failed'
+  | 'http_error'
+  | 'request_timeout'
+  | 'response_body_error'
+  | 'response_body_timeout'
+  | 'retryable_http'
+  | 'transport_error';
+
+type AppRuntimeRetryFailure = {
+  attempt: number;
+  failureKind: AppRuntimeRetryFailureKind;
+  attemptElapsedMs: number;
+  retryDelayMs: number;
+  httpStatus?: number;
+};
+
+type AppRuntimeRetryTelemetry = {
+  enabled: boolean;
+  startedAt: number;
+  retryCount: number;
+  retrySleepMs: number;
+  failures: AppRuntimeRetryFailure[];
+};
+
+function appRuntimeTransportFailureKind(
+  error: unknown,
+): AppRuntimeRetryFailureKind {
+  if (isRequestTimeoutAbort(error)) return 'request_timeout';
+  const message = error instanceof Error ? error.message : String(error ?? '');
+  if (/\b(?:ENOTFOUND|EAI_AGAIN)\b|getaddrinfo|\bdns\b/i.test(message)) {
+    return 'dns';
+  }
+  if (/\bECONNRESET\b|connection reset/i.test(message)) {
+    return 'connection_reset';
+  }
+  if (/\bECONNREFUSED\b|connection refused/i.test(message)) {
+    return 'connection_refused';
+  }
+  if (
+    /\bETIMEDOUT\b|UND_ERR_CONNECT_TIMEOUT|connection (?:timeout|timed out)/i.test(
+      message,
+    )
+  ) {
+    return 'connection_timeout';
+  }
+  return /fetch failed/i.test(message) ? 'fetch_failed' : 'transport_error';
+}
+
+function recordAppRuntimeRetryFailure(input: {
+  telemetry: AppRuntimeRetryTelemetry;
+  attempt: number;
+  failureKind: AppRuntimeRetryFailureKind;
+  attemptStartedAt: number;
+  retryDelayMs: number;
+  httpStatus?: number;
+}): void {
+  if (!input.telemetry.enabled) return;
+  // Receipt actions have at most eight attempts. Keep the terminal eighth
+  // failure instead of truncating the causally important final observation.
+  if (input.telemetry.failures.length < 8) {
+    input.telemetry.failures.push({
+      attempt: input.attempt,
+      failureKind: input.failureKind,
+      attemptElapsedMs: Date.now() - input.attemptStartedAt,
+      retryDelayMs: input.retryDelayMs,
+      ...(input.httpStatus === undefined
+        ? {}
+        : { httpStatus: input.httpStatus }),
+    });
+  }
+  if (input.retryDelayMs > 0) {
+    input.telemetry.retryCount += 1;
+    input.telemetry.retrySleepMs += input.retryDelayMs;
+  }
+}
+
+function emitAppRuntimeRetrySummary(input: {
+  telemetry: AppRuntimeRetryTelemetry;
+  action: RuntimeApiRequest['action'];
+  outcome: 'failed_after_retry' | 'recovered_after_retry';
+  attempts: number;
+  maxAttempts: number;
+  finalAttemptStartedAt: number;
+}): void {
+  if (!input.telemetry.enabled || input.telemetry.retryCount === 0) return;
+  const payload = {
+    action: input.action,
+    outcome: input.outcome,
+    attempts: input.attempts,
+    maxAttempts: input.maxAttempts,
+    totalElapsedMs: Date.now() - input.telemetry.startedAt,
+    retrySleepMs: input.telemetry.retrySleepMs,
+    finalAttemptElapsedMs: Date.now() - input.finalAttemptStartedAt,
+    failures: input.telemetry.failures,
+  };
+  // Observability must never alter receipt execution. The payload contains
+  // primitives from a fixed taxonomy only: no URL, token, body, raw error,
+  // run/play/org id, receipt key, input, or output.
+  try {
+    const serialized = JSON.stringify(payload);
+    if (input.outcome === 'failed_after_retry') {
+      console.warn(APP_RUNTIME_RECEIPT_RETRY_TELEMETRY_TAG, serialized);
+    } else {
+      console.info(APP_RUNTIME_RECEIPT_RETRY_TELEMETRY_TAG, serialized);
+    }
+  } catch {}
+}
+
 async function retryAppRuntimeBodyTimeoutOrThrow(input: {
   action: RuntimeApiRequest['action'];
   attempt: number;
   maxAttempts: number;
   error: unknown;
+  telemetry: AppRuntimeRetryTelemetry;
+  attemptStartedAt: number;
+  httpStatus?: number;
   retryAfterMs?: number | null;
 }): Promise<void> {
   if (!isRequestTimeoutAbort(input.error)) {
+    recordAppRuntimeRetryFailure({
+      telemetry: input.telemetry,
+      attempt: input.attempt,
+      failureKind: 'response_body_error',
+      attemptStartedAt: input.attemptStartedAt,
+      retryDelayMs: 0,
+      httpStatus: input.httpStatus,
+    });
+    emitAppRuntimeRetrySummary({
+      telemetry: input.telemetry,
+      action: input.action,
+      outcome: 'failed_after_retry',
+      attempts: input.attempt,
+      maxAttempts: input.maxAttempts,
+      finalAttemptStartedAt: input.attemptStartedAt,
+    });
     throw input.error;
   }
   if (
@@ -750,14 +895,37 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
       error: input.error,
     })
   ) {
-    await sleep(
-      Math.max(
-        applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
-        input.retryAfterMs ?? 0,
-      ),
+    const retryDelayMs = Math.max(
+      applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
+      input.retryAfterMs ?? 0,
     );
+    recordAppRuntimeRetryFailure({
+      telemetry: input.telemetry,
+      attempt: input.attempt,
+      failureKind: 'response_body_timeout',
+      attemptStartedAt: input.attemptStartedAt,
+      retryDelayMs,
+      httpStatus: input.httpStatus,
+    });
+    await sleep(retryDelayMs);
     return;
   }
+  recordAppRuntimeRetryFailure({
+    telemetry: input.telemetry,
+    attempt: input.attempt,
+    failureKind: 'response_body_timeout',
+    attemptStartedAt: input.attemptStartedAt,
+    retryDelayMs: 0,
+    httpStatus: input.httpStatus,
+  });
+  emitAppRuntimeRetrySummary({
+    telemetry: input.telemetry,
+    action: input.action,
+    outcome: 'failed_after_retry',
+    attempts: input.attempt,
+    maxAttempts: input.maxAttempts,
+    finalAttemptStartedAt: input.attemptStartedAt,
+  });
   throw new AppRuntimeApiTransportError({
     action: input.action,
     attempts: input.attempt,
@@ -786,6 +954,40 @@ export class AppRuntimeApiTransportError extends Error {
   }
 }
 
+/**
+ * A non-2xx response from the app runtime boundary.
+ *
+ * Keep the response contract structured here so durable callers can decide
+ * whether to retry or block a delivery without parsing an error message.
+ */
+export class AppRuntimeApiResponseError extends Error {
+  readonly status: number;
+  readonly code: string | null;
+  readonly requestId: string | null;
+  readonly retryable: boolean;
+
+  constructor(input: {
+    action: RuntimeApiRequest['action'];
+    status: number;
+    code?: string | null;
+    requestId?: string | null;
+    retryable: boolean;
+    detail: string;
+  }) {
+    super(
+      `App runtime API ${input.action} failed with status ${input.status}` +
+        `${input.code ? ` code=${input.code}` : ''}` +
+        `${input.requestId ? ` request_id=${input.requestId}` : ''}: ` +
+        input.detail,
+    );
+    this.name = 'AppRuntimeApiResponseError';
+    this.status = input.status;
+    this.code = input.code?.trim() || null;
+    this.requestId = input.requestId?.trim() || null;
+    this.retryable = input.retryable;
+  }
+}
+
 function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
   const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
   return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
@@ -811,6 +1013,13 @@ async function postAppRuntimeApi<TResponse>(
   });
 
   const maxAttempts = appRuntimeMaxAttempts(body.action);
+  const retryTelemetry: AppRuntimeRetryTelemetry = {
+    enabled: isRuntimeStepReceiptAction(body.action),
+    startedAt: Date.now(),
+    retryCount: 0,
+    retrySleepMs: 0,
+    failures: [],
+  };
   const requestTimeoutMs = Math.max(
     1,
     Math.floor(
@@ -819,6 +1028,7 @@ async function postAppRuntimeApi<TResponse>(
   );
 
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    const attemptStartedAt = Date.now();
     let response: Response;
     try {
       response = await runtimeFetch(resolveAppRuntimeApiUrl(context), {
@@ -843,11 +1053,34 @@ async function postAppRuntimeApi<TResponse>(
         attempt < maxAttempts &&
         isRetryableAppRuntimeFetchError({ action: body.action, error })
       ) {
-        await sleep(
-          applyRetryJitter(appRuntimeRetryDelayMs(body.action, attempt)),
+        const retryDelayMs = applyRetryJitter(
+          appRuntimeRetryDelayMs(body.action, attempt),
         );
+        recordAppRuntimeRetryFailure({
+          telemetry: retryTelemetry,
+          attempt,
+          failureKind: appRuntimeTransportFailureKind(error),
+          attemptStartedAt,
+          retryDelayMs,
+        });
+        await sleep(retryDelayMs);
         continue;
       }
+      recordAppRuntimeRetryFailure({
+        telemetry: retryTelemetry,
+        attempt,
+        failureKind: appRuntimeTransportFailureKind(error),
+        attemptStartedAt,
+        retryDelayMs: 0,
+      });
+      emitAppRuntimeRetrySummary({
+        telemetry: retryTelemetry,
+        action: body.action,
+        outcome: 'failed_after_retry',
+        attempts: attempt,
+        maxAttempts,
+        finalAttemptStartedAt: attemptStartedAt,
+      });
       throw new AppRuntimeApiTransportError({
         action: body.action,
         attempts: attempt,
@@ -856,13 +1089,25 @@ async function postAppRuntimeApi<TResponse>(
     }
     if (response.ok) {
       try {
-        return (await response.json()) as TResponse;
+        const parsed = (await response.json()) as TResponse;
+        emitAppRuntimeRetrySummary({
+          telemetry: retryTelemetry,
+          action: body.action,
+          outcome: 'recovered_after_retry',
+          attempts: attempt,
+          maxAttempts,
+          finalAttemptStartedAt: attemptStartedAt,
+        });
+        return parsed;
       } catch (error) {
         await retryAppRuntimeBodyTimeoutOrThrow({
           action: body.action,
           attempt,
           maxAttempts,
           error,
+          telemetry: retryTelemetry,
+          attemptStartedAt,
+          httpStatus: response.status,
         });
         continue;
       }
@@ -875,6 +1120,22 @@ async function postAppRuntimeApi<TResponse>(
         isRequestTimeoutAbort(error) &&
         !isRetryableAppRuntimeResponseStatus(response.status)
       ) {
+        recordAppRuntimeRetryFailure({
+          telemetry: retryTelemetry,
+          attempt,
+          failureKind: 'response_body_timeout',
+          attemptStartedAt,
+          retryDelayMs: 0,
+          httpStatus: response.status,
+        });
+        emitAppRuntimeRetrySummary({
+          telemetry: retryTelemetry,
+          action: body.action,
+          outcome: 'failed_after_retry',
+          attempts: attempt,
+          maxAttempts,
+          finalAttemptStartedAt: attemptStartedAt,
+        });
         throw new Error(
           `App runtime API ${body.action} failed with status ${response.status}: response body timed out`,
           { cause: error },
@@ -885,6 +1146,9 @@ async function postAppRuntimeApi<TResponse>(
         attempt,
         maxAttempts,
         error,
+        telemetry: retryTelemetry,
+        attemptStartedAt,
+        httpStatus: response.status,
         retryAfterMs: appRuntimeRetryAfterMs(response, ''),
       });
       continue;
@@ -897,22 +1161,61 @@ async function postAppRuntimeApi<TResponse>(
         body: responseText,
       })
     ) {
-      await sleep(
-        Math.max(
-          applyRetryJitter(appRuntimeRetryDelayMs(body.action, attempt)),
-          appRuntimeRetryAfterMs(response, responseText) ?? 0,
-        ),
+      const retryDelayMs = Math.max(
+        applyRetryJitter(appRuntimeRetryDelayMs(body.action, attempt)),
+        appRuntimeRetryAfterMs(response, responseText) ?? 0,
       );
+      recordAppRuntimeRetryFailure({
+        telemetry: retryTelemetry,
+        attempt,
+        failureKind: 'retryable_http',
+        attemptStartedAt,
+        retryDelayMs,
+        httpStatus: response.status,
+      });
+      await sleep(retryDelayMs);
       continue;
     }
+    recordAppRuntimeRetryFailure({
+      telemetry: retryTelemetry,
+      attempt,
+      failureKind: isRetryableAppRuntimeResponse({
+        action: body.action,
+        status: response.status,
+        body: responseText,
+      })
+        ? 'retryable_http'
+        : 'http_error',
+      attemptStartedAt,
+      retryDelayMs: 0,
+      httpStatus: response.status,
+    });
+    emitAppRuntimeRetrySummary({
+      telemetry: retryTelemetry,
+      action: body.action,
+      outcome: 'failed_after_retry',
+      attempts: attempt,
+      maxAttempts,
+      finalAttemptStartedAt: attemptStartedAt,
+    });
     const code = appRuntimeErrorCode(response, responseText);
     const requestId = response.headers.get('x-deepline-request-id')?.trim();
-    throw new Error(
-      `App runtime API ${body.action} failed with status ${response.status}` +
-        `${code ? ` code=${code}` : ''}` +
-        `${requestId ? ` request_id=${requestId}` : ''}: ` +
-        summarizeAppRuntimeErrorBody(responseText),
-    );
+    throw new AppRuntimeApiResponseError({
+      action: body.action,
+      status: response.status,
+      code,
+      requestId,
+      // Immediate request retries are intentionally selective, but durable
+      // delivery must never classify an unrecognized upstream 5xx as poison.
+      retryable:
+        response.status >= 500 ||
+        isRetryableAppRuntimeResponse({
+          action: body.action,
+          status: response.status,
+          body: responseText,
+        }),
+      detail: summarizeAppRuntimeErrorBody(responseText),
+    });
   }
 
   throw new Error(`App runtime API ${body.action} failed after retries.`);
@@ -1239,7 +1542,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
   );
 }
 
-export async function appendRunEventsViaAppRuntime(
+async function appendRunEventsViaAppRuntimeRaw(
   context: WorkerRuntimeApiContext,
   input: {
     playId: string;
@@ -1253,14 +1556,62 @@ export async function appendRunEventsViaAppRuntime(
   });
 }
 
+/**
+ * Append Run Ledger events through the bounded transport used by every runtime
+ * producer. Scheduler terminal outbox events can carry a complete log replay,
+ * so this boundary must never forward an arbitrarily large event array or log
+ * event directly to Convex.
+ */
+export async function appendRunEventsViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: {
+    playId: string;
+    events: PlayRunLedgerEvent[];
+    idempotencyKey?: string;
+  },
+): Promise<void> {
+  const batches = partitionRunEventsForAppRuntime(input.events);
+  if (batches.length <= 1) {
+    await appendRunEventsViaAppRuntimeRaw(context, input);
+    return;
+  }
+
+  // Terminal idempotency keys record a completed delivery. Sending the same
+  // key on an earlier log-only batch would make Convex drop the terminal batch,
+  // so attach it to the batch that contains the final terminal event only.
+  let terminalBatchIndex = -1;
+  for (let index = batches.length - 1; index >= 0; index -= 1) {
+    if (
+      batches[index]!.some(
+        (event) =>
+          event.type === 'run.completed' ||
+          event.type === 'run.failed' ||
+          event.type === 'run.cancelled',
+      )
+    ) {
+      terminalBatchIndex = index;
+      break;
+    }
+  }
+  const idempotencyBatchIndex =
+    terminalBatchIndex >= 0 ? terminalBatchIndex : batches.length - 1;
+
+  for (let index = 0; index < batches.length; index += 1) {
+    await appendRunEventsViaAppRuntimeRaw(context, {
+      playId: input.playId,
+      events: batches[index]!,
+      ...(input.idempotencyKey && index === idempotencyBatchIndex
+        ? { idempotencyKey: input.idempotencyKey }
+        : {}),
+    });
+  }
+}
+
 export async function startRunViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: {
     playName: string;
     runId: string;
-    parentRunId?: string | null;
-    rootRunId?: string | null;
-    workflowFamilyKey?: string | null;
     artifactStorageKey?: string | null;
     artifactHash?: string | null;
     graphHash?: string | null;
@@ -1275,7 +1626,7 @@ export async function startRunViaAppRuntime(
   },
 ): Promise<void> {
   await postAppRuntimeApi<{ ok: true }>(context, {
-    action: 'start_inline_child_run',
+    action: 'start_run',
     ...input,
   });
 }