deepline 0.1.227 → 0.1.228

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -380,6 +380,8 @@ function applyRetryJitter(delayMs: number): number {
   return Math.round(half + Math.random() * half);
 }
 const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+const APP_RUNTIME_RECEIPT_RETRY_TELEMETRY_TAG =
+  '[perf][worker.receipt_api.transport]';
 const RUN_STATUS_LEDGER_SNAPSHOT_CACHE_LIMIT = 1_000;
 const runStatusLedgerSnapshots = new Map<string, PlayRunLedgerSnapshot>();
 // Positional log cursor per run attempt: count of liveLogs lines already
@@ -454,7 +456,7 @@ async function appendRunEventBatchesViaAppRuntime(
   input: { playId: string; events: readonly PlayRunLedgerEvent[] },
 ): Promise<void> {
   for (const events of partitionRunEventsForAppRuntime(input.events)) {
-    await appendRunEventsViaAppRuntime(context, {
+    await appendRunEventsViaAppRuntimeRaw(context, {
       playId: input.playId,
       events,
     });
@@ -583,7 +585,7 @@ function appRuntimeErrorCode(response: Response, body: string): string | null {
 }
 
 export function isTransientAppRuntimeFailureBody(body: string): boolean {
-  return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open|OptimisticConcurrencyControlFailure|changed while this mutation was being run|Documents read from or written to the .* table changed while this mutation/i.test(
+  return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_(?:SATURATED|UNAVAILABLE)|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open|OptimisticConcurrencyControlFailure|changed while this mutation was being run|Documents read from or written to the .* table changed while this mutation/i.test(
     body,
   );
 }
@@ -733,14 +735,146 @@ function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+type AppRuntimeRetryFailureKind =
+  | 'connection_refused'
+  | 'connection_reset'
+  | 'connection_timeout'
+  | 'dns'
+  | 'fetch_failed'
+  | 'http_error'
+  | 'request_timeout'
+  | 'response_body_error'
+  | 'response_body_timeout'
+  | 'retryable_http'
+  | 'transport_error';
+
+type AppRuntimeRetryFailure = {
+  attempt: number;
+  failureKind: AppRuntimeRetryFailureKind;
+  attemptElapsedMs: number;
+  retryDelayMs: number;
+  httpStatus?: number;
+};
+
+type AppRuntimeRetryTelemetry = {
+  enabled: boolean;
+  startedAt: number;
+  retryCount: number;
+  retrySleepMs: number;
+  failures: AppRuntimeRetryFailure[];
+};
+
+function appRuntimeTransportFailureKind(
+  error: unknown,
+): AppRuntimeRetryFailureKind {
+  if (isRequestTimeoutAbort(error)) return 'request_timeout';
+  const message = error instanceof Error ? error.message : String(error ?? '');
+  if (/\b(?:ENOTFOUND|EAI_AGAIN)\b|getaddrinfo|\bdns\b/i.test(message)) {
+    return 'dns';
+  }
+  if (/\bECONNRESET\b|connection reset/i.test(message)) {
+    return 'connection_reset';
+  }
+  if (/\bECONNREFUSED\b|connection refused/i.test(message)) {
+    return 'connection_refused';
+  }
+  if (
+    /\bETIMEDOUT\b|UND_ERR_CONNECT_TIMEOUT|connection (?:timeout|timed out)/i.test(
+      message,
+    )
+  ) {
+    return 'connection_timeout';
+  }
+  return /fetch failed/i.test(message) ? 'fetch_failed' : 'transport_error';
+}
+
+function recordAppRuntimeRetryFailure(input: {
+  telemetry: AppRuntimeRetryTelemetry;
+  attempt: number;
+  failureKind: AppRuntimeRetryFailureKind;
+  attemptStartedAt: number;
+  retryDelayMs: number;
+  httpStatus?: number;
+}): void {
+  if (!input.telemetry.enabled) return;
+  // Receipt actions have at most eight attempts. Keep the terminal eighth
+  // failure instead of truncating the causally important final observation.
+  if (input.telemetry.failures.length < 8) {
+    input.telemetry.failures.push({
+      attempt: input.attempt,
+      failureKind: input.failureKind,
+      attemptElapsedMs: Date.now() - input.attemptStartedAt,
+      retryDelayMs: input.retryDelayMs,
+      ...(input.httpStatus === undefined
+        ? {}
+        : { httpStatus: input.httpStatus }),
+    });
+  }
+  if (input.retryDelayMs > 0) {
+    input.telemetry.retryCount += 1;
+    input.telemetry.retrySleepMs += input.retryDelayMs;
+  }
+}
+
+function emitAppRuntimeRetrySummary(input: {
+  telemetry: AppRuntimeRetryTelemetry;
+  action: RuntimeApiRequest['action'];
+  outcome: 'failed_after_retry' | 'recovered_after_retry';
+  attempts: number;
+  maxAttempts: number;
+  finalAttemptStartedAt: number;
+}): void {
+  if (!input.telemetry.enabled || input.telemetry.retryCount === 0) return;
+  const payload = {
+    action: input.action,
+    outcome: input.outcome,
+    attempts: input.attempts,
+    maxAttempts: input.maxAttempts,
+    totalElapsedMs: Date.now() - input.telemetry.startedAt,
+    retrySleepMs: input.telemetry.retrySleepMs,
+    finalAttemptElapsedMs: Date.now() - input.finalAttemptStartedAt,
+    failures: input.telemetry.failures,
+  };
+  // Observability must never alter receipt execution. The payload contains
+  // primitives from a fixed taxonomy only: no URL, token, body, raw error,
+  // run/play/org id, receipt key, input, or output.
+  try {
+    const serialized = JSON.stringify(payload);
+    if (input.outcome === 'failed_after_retry') {
+      console.warn(APP_RUNTIME_RECEIPT_RETRY_TELEMETRY_TAG, serialized);
+    } else {
+      console.info(APP_RUNTIME_RECEIPT_RETRY_TELEMETRY_TAG, serialized);
+    }
+  } catch {}
+}
+
 async function retryAppRuntimeBodyTimeoutOrThrow(input: {
   action: RuntimeApiRequest['action'];
   attempt: number;
   maxAttempts: number;
   error: unknown;
+  telemetry: AppRuntimeRetryTelemetry;
+  attemptStartedAt: number;
+  httpStatus?: number;
   retryAfterMs?: number | null;
 }): Promise<void> {
   if (!isRequestTimeoutAbort(input.error)) {
+    recordAppRuntimeRetryFailure({
+      telemetry: input.telemetry,
+      attempt: input.attempt,
+      failureKind: 'response_body_error',
+      attemptStartedAt: input.attemptStartedAt,
+      retryDelayMs: 0,
+      httpStatus: input.httpStatus,
+    });
+    emitAppRuntimeRetrySummary({
+      telemetry: input.telemetry,
+      action: input.action,
+      outcome: 'failed_after_retry',
+      attempts: input.attempt,
+      maxAttempts: input.maxAttempts,
+      finalAttemptStartedAt: input.attemptStartedAt,
+    });
     throw input.error;
   }
   if (
@@ -750,14 +884,37 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
       error: input.error,
     })
   ) {
-    await sleep(
-      Math.max(
-        applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
-        input.retryAfterMs ?? 0,
-      ),
+    const retryDelayMs = Math.max(
+      applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
+      input.retryAfterMs ?? 0,
     );
+    recordAppRuntimeRetryFailure({
+      telemetry: input.telemetry,
+      attempt: input.attempt,
+      failureKind: 'response_body_timeout',
+      attemptStartedAt: input.attemptStartedAt,
+      retryDelayMs,
+      httpStatus: input.httpStatus,
+    });
+    await sleep(retryDelayMs);
     return;
   }
+  recordAppRuntimeRetryFailure({
+    telemetry: input.telemetry,
+    attempt: input.attempt,
+    failureKind: 'response_body_timeout',
+    attemptStartedAt: input.attemptStartedAt,
+    retryDelayMs: 0,
+    httpStatus: input.httpStatus,
+  });
+  emitAppRuntimeRetrySummary({
+    telemetry: input.telemetry,
+    action: input.action,
+    outcome: 'failed_after_retry',
+    attempts: input.attempt,
+    maxAttempts: input.maxAttempts,
+    finalAttemptStartedAt: input.attemptStartedAt,
+  });
   throw new AppRuntimeApiTransportError({
     action: input.action,
     attempts: input.attempt,
@@ -811,6 +968,13 @@ async function postAppRuntimeApi<TResponse>(
   });
 
   const maxAttempts = appRuntimeMaxAttempts(body.action);
+  const retryTelemetry: AppRuntimeRetryTelemetry = {
+    enabled: isRuntimeStepReceiptAction(body.action),
+    startedAt: Date.now(),
+    retryCount: 0,
+    retrySleepMs: 0,
+    failures: [],
+  };
   const requestTimeoutMs = Math.max(
     1,
     Math.floor(
@@ -819,6 +983,7 @@ async function postAppRuntimeApi<TResponse>(
   );
 
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    const attemptStartedAt = Date.now();
     let response: Response;
     try {
       response = await runtimeFetch(resolveAppRuntimeApiUrl(context), {
@@ -843,11 +1008,34 @@ async function postAppRuntimeApi<TResponse>(
         attempt < maxAttempts &&
         isRetryableAppRuntimeFetchError({ action: body.action, error })
       ) {
-        await sleep(
-          applyRetryJitter(appRuntimeRetryDelayMs(body.action, attempt)),
+        const retryDelayMs = applyRetryJitter(
+          appRuntimeRetryDelayMs(body.action, attempt),
         );
+        recordAppRuntimeRetryFailure({
+          telemetry: retryTelemetry,
+          attempt,
+          failureKind: appRuntimeTransportFailureKind(error),
+          attemptStartedAt,
+          retryDelayMs,
+        });
+        await sleep(retryDelayMs);
         continue;
       }
+      recordAppRuntimeRetryFailure({
+        telemetry: retryTelemetry,
+        attempt,
+        failureKind: appRuntimeTransportFailureKind(error),
+        attemptStartedAt,
+        retryDelayMs: 0,
+      });
+      emitAppRuntimeRetrySummary({
+        telemetry: retryTelemetry,
+        action: body.action,
+        outcome: 'failed_after_retry',
+        attempts: attempt,
+        maxAttempts,
+        finalAttemptStartedAt: attemptStartedAt,
+      });
       throw new AppRuntimeApiTransportError({
         action: body.action,
         attempts: attempt,
@@ -856,13 +1044,25 @@ async function postAppRuntimeApi<TResponse>(
     }
     if (response.ok) {
       try {
-        return (await response.json()) as TResponse;
+        const parsed = (await response.json()) as TResponse;
+        emitAppRuntimeRetrySummary({
+          telemetry: retryTelemetry,
+          action: body.action,
+          outcome: 'recovered_after_retry',
+          attempts: attempt,
+          maxAttempts,
+          finalAttemptStartedAt: attemptStartedAt,
+        });
+        return parsed;
       } catch (error) {
         await retryAppRuntimeBodyTimeoutOrThrow({
           action: body.action,
           attempt,
           maxAttempts,
           error,
+          telemetry: retryTelemetry,
+          attemptStartedAt,
+          httpStatus: response.status,
         });
         continue;
       }
@@ -875,6 +1075,22 @@ async function postAppRuntimeApi<TResponse>(
         isRequestTimeoutAbort(error) &&
         !isRetryableAppRuntimeResponseStatus(response.status)
       ) {
+        recordAppRuntimeRetryFailure({
+          telemetry: retryTelemetry,
+          attempt,
+          failureKind: 'response_body_timeout',
+          attemptStartedAt,
+          retryDelayMs: 0,
+          httpStatus: response.status,
+        });
+        emitAppRuntimeRetrySummary({
+          telemetry: retryTelemetry,
+          action: body.action,
+          outcome: 'failed_after_retry',
+          attempts: attempt,
+          maxAttempts,
+          finalAttemptStartedAt: attemptStartedAt,
+        });
         throw new Error(
           `App runtime API ${body.action} failed with status ${response.status}: response body timed out`,
           { cause: error },
@@ -885,6 +1101,9 @@ async function postAppRuntimeApi<TResponse>(
         attempt,
         maxAttempts,
         error,
+        telemetry: retryTelemetry,
+        attemptStartedAt,
+        httpStatus: response.status,
         retryAfterMs: appRuntimeRetryAfterMs(response, ''),
       });
       continue;
@@ -897,14 +1116,43 @@ async function postAppRuntimeApi<TResponse>(
         body: responseText,
       })
     ) {
-      await sleep(
-        Math.max(
-          applyRetryJitter(appRuntimeRetryDelayMs(body.action, attempt)),
-          appRuntimeRetryAfterMs(response, responseText) ?? 0,
-        ),
+      const retryDelayMs = Math.max(
+        applyRetryJitter(appRuntimeRetryDelayMs(body.action, attempt)),
+        appRuntimeRetryAfterMs(response, responseText) ?? 0,
       );
+      recordAppRuntimeRetryFailure({
+        telemetry: retryTelemetry,
+        attempt,
+        failureKind: 'retryable_http',
+        attemptStartedAt,
+        retryDelayMs,
+        httpStatus: response.status,
+      });
+      await sleep(retryDelayMs);
       continue;
     }
+    recordAppRuntimeRetryFailure({
+      telemetry: retryTelemetry,
+      attempt,
+      failureKind: isRetryableAppRuntimeResponse({
+        action: body.action,
+        status: response.status,
+        body: responseText,
+      })
+        ? 'retryable_http'
+        : 'http_error',
+      attemptStartedAt,
+      retryDelayMs: 0,
+      httpStatus: response.status,
+    });
+    emitAppRuntimeRetrySummary({
+      telemetry: retryTelemetry,
+      action: body.action,
+      outcome: 'failed_after_retry',
+      attempts: attempt,
+      maxAttempts,
+      finalAttemptStartedAt: attemptStartedAt,
+    });
     const code = appRuntimeErrorCode(response, responseText);
     const requestId = response.headers.get('x-deepline-request-id')?.trim();
     throw new Error(
@@ -1239,7 +1487,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
   );
 }
 
-export async function appendRunEventsViaAppRuntime(
+async function appendRunEventsViaAppRuntimeRaw(
   context: WorkerRuntimeApiContext,
   input: {
     playId: string;
@@ -1253,6 +1501,57 @@ export async function appendRunEventsViaAppRuntime(
   });
 }
 
+/**
+ * Append Run Ledger events through the bounded transport used by every runtime
+ * producer. Scheduler terminal outbox events can carry a complete log replay,
+ * so this boundary must never forward an arbitrarily large event array or log
+ * event directly to Convex.
+ */
+export async function appendRunEventsViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: {
+    playId: string;
+    events: PlayRunLedgerEvent[];
+    idempotencyKey?: string;
+  },
+): Promise<void> {
+  const batches = partitionRunEventsForAppRuntime(input.events);
+  if (batches.length <= 1) {
+    await appendRunEventsViaAppRuntimeRaw(context, input);
+    return;
+  }
+
+  // Terminal idempotency keys record a completed delivery. Sending the same
+  // key on an earlier log-only batch would make Convex drop the terminal batch,
+  // so attach it to the batch that contains the final terminal event only.
+  let terminalBatchIndex = -1;
+  for (let index = batches.length - 1; index >= 0; index -= 1) {
+    if (
+      batches[index]!.some(
+        (event) =>
+          event.type === 'run.completed' ||
+          event.type === 'run.failed' ||
+          event.type === 'run.cancelled',
+      )
+    ) {
+      terminalBatchIndex = index;
+      break;
+    }
+  }
+  const idempotencyBatchIndex =
+    terminalBatchIndex >= 0 ? terminalBatchIndex : batches.length - 1;
+
+  for (let index = 0; index < batches.length; index += 1) {
+    await appendRunEventsViaAppRuntimeRaw(context, {
+      playId: input.playId,
+      events: batches[index]!,
+      ...(input.idempotencyKey && index === idempotencyBatchIndex
+        ? { idempotencyKey: input.idempotencyKey }
+        : {}),
+    });
+  }
+}
+
 export async function startRunViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: {

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -108,6 +108,10 @@ import {
   ToolExecuteAuthScopeChangedError,
 } from './tool-execute-retry-policy';
 import { ToolHttpError } from './tool-http-errors';
+import {
+  describeTransportError,
+  transportGatewayOriginForDiagnostic,
+} from './transport-error-diagnostics';
 import {
   buildDurableCtxCallCacheKey,
   buildDurableRunPlayInvocationScope,
@@ -8725,10 +8729,26 @@ export class PlayContextImpl {
           const retrySafeTransientHttp =
             retryPolicy?.retrySafeTransientHttp === true;
           let transportAttempt = 0;
-          const retryToolTransportFailure = async (
-            message: string,
-          ): Promise<void> => {
+          const retryToolTransportFailure = async (input: {
+            error: unknown;
+            elapsedMs: number;
+            requestId: string | null;
+            aborted: boolean;
+          }): Promise<void> => {
             transportAttempt += 1;
+            const diagnostic = describeTransportError(input.error);
+            this.log(
+              `[runtime.transport_failure] ${JSON.stringify({
+                tool_id: toolId,
+                gateway_origin: transportGatewayOriginForDiagnostic(url),
+                attempt: transportAttempt,
+                max_attempts: TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
+                elapsed_ms: input.elapsedMs,
+                request_id: input.requestId,
+                aborted: input.aborted,
+                error: diagnostic,
+              })}`,
+            );
             if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
               await this.governor.chargeBudget('retry');
               const retryAfterMs =
@@ -8738,14 +8758,14 @@ export class PlayContextImpl {
                 transportAttempt,
               );
               this.log(
-                `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
+                `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${diagnostic.message ?? 'unknown transport error'}`,
               );
               await options?.parkProviderCall?.();
               await this.sleepWithCheckpointHeartbeat(retryAfterMs);
               return;
             }
             throw new ToolHttpError(
-              `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
+              `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${diagnostic.message ?? 'unknown transport error'}`,
               null,
               0,
               'repairable',
@@ -8767,9 +8787,12 @@ export class PlayContextImpl {
                   : null;
             const providerIdempotencyKey =
               options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
+            // Every gateway request needs a correlation ID. When the call has
+            // an idempotency key, keep its stable ID across retries; otherwise
+            // give this individual attempt a fresh ID for gateway pairing.
             const deeplineRequestId = providerIdempotencyKey
               ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
-              : null;
+              : `ctx-tool-${crypto.randomUUID()}`;
             // Receipt ownership is a liveness contract, not a one-time check.
             // Keep it alive for the whole provider HTTP request. The cadence
             // comes from the store-issued expiry because a remote runner may
@@ -8837,10 +8860,10 @@ export class PlayContextImpl {
                       V2_EXECUTE_RESPONSE_CONTRACT,
                     [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
                     [EXECUTE_TOOL_METADATA_HEADER]: 'true',
-                    ...(deeplineRequestId
+                    'x-deepline-request-id': deeplineRequestId,
+                    ...(providerIdempotencyKey
                       ? {
-                          'x-deepline-request-id': deeplineRequestId,
-                          'x-deepline-idempotency-key': providerIdempotencyKey!,
+                          'x-deepline-idempotency-key': providerIdempotencyKey,
                         }
                       : {}),
                     ...(await this.vercelProtectionHeaders()),
@@ -8918,11 +8941,15 @@ export class PlayContextImpl {
                       `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
                     )
                 : error;
-              const message =
-                transportError instanceof Error
-                  ? transportError.message
-                  : String(transportError);
-              await retryToolTransportFailure(message);
+              await retryToolTransportFailure({
+                error: transportError,
+                elapsedMs:
+                  providerCallStartedAt === null
+                    ? 0
+                    : Date.now() - providerCallStartedAt,
+                requestId: deeplineRequestId,
+                aborted: abortController?.signal.aborted === true,
+              });
               continue;
             } finally {
               if (timeoutHandle) {
@@ -8941,9 +8968,14 @@ export class PlayContextImpl {
                   bodyText: text,
                 })
               ) {
-                await retryToolTransportFailure(
-                  'the Deepline development tunnel returned a branded 502 before reaching the app origin',
-                );
+                await retryToolTransportFailure({
+                  error: new Error(
+                    'the Deepline development tunnel returned a branded 502 before reaching the app origin',
+                  ),
+                  elapsedMs: providerCallElapsedMs ?? 0,
+                  requestId: deeplineRequestId,
+                  aborted: false,
+                });
                 continue;
               }
               const authScopeChanged = parseToolExecuteAuthScopeChangedError({

package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts

@@ -1,12 +1,6 @@
-export type PlayRunLiveStatus =
-  | 'queued'
-  | 'running'
-  | 'completed'
-  | 'failed'
-  | 'cancelled'
-  | 'terminated'
-  | 'timed_out'
-  | 'unknown';
+import type { PlayRunLifecycleStatus } from './run-lifecycle-policy';
+
+export type PlayRunLiveStatus = PlayRunLifecycleStatus;
 
 export type PlayVisualNodeStatus =
   | 'idle'

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -166,6 +166,12 @@ export type PlayRunLedgerEvent =
       playName?: string | null;
       runtimeBackend?: string | null;
     })
+  | (PlayRunLedgerBaseEvent & {
+      type: 'run.waiting';
+    })
+  | (PlayRunLedgerBaseEvent & {
+      type: 'run.resumed';
+    })
   | (PlayRunLedgerBaseEvent & {
       type: 'run.completed';
       result?: unknown;
@@ -1025,6 +1031,16 @@ export function reducePlayRunLedgerEvent(
   if (event.runId !== snapshot.runId) {
     return snapshot;
   }
+  // The scheduler's outbox can replay an older wait transition after its run
+  // reached a terminal state. That transition is no longer meaningful and
+  // must be byte-for-byte inert: even advancing `updatedAt` would re-publish
+  // a stale terminal snapshot to every observer.
+  if (
+    (event.type === 'run.waiting' || event.type === 'run.resumed') &&
+    shouldIgnoreNonTerminalAfterStickyTerminal(snapshot)
+  ) {
+    return snapshot;
+  }
   const occurredAt = Math.max(0, event.occurredAt);
   const base: PlayRunLedgerSnapshot = {
     ...snapshot,
@@ -1062,6 +1078,16 @@ export function reducePlayRunLedgerEvent(
           : 'running',
         startedAt: base.startedAt ?? occurredAt,
       });
+    case 'run.waiting':
+      return withTiming({
+        ...base,
+        status: 'waiting',
+      });
+    case 'run.resumed':
+      return withTiming({
+        ...base,
+        status: 'running',
+      });
     case 'run.completed':
       return (
         conflictingTerminalSnapshot(base, event.type, occurredAt, null, {

package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts

@@ -1,6 +1,7 @@
 export type PlayRunLifecycleStatus =
   | 'queued'
   | 'running'
+  | 'waiting'
   | 'completed'
   | 'failed'
   | 'cancelled'
@@ -35,8 +36,9 @@ export function normalizePlayRunLifecycleStatus(
       return 'running';
     case 'running':
     case 'started':
-    case 'waiting':
       return 'running';
+    case 'waiting':
+      return 'waiting';
     case 'completed':
     case 'complete':
     case 'succeeded':
@@ -62,7 +64,8 @@ export function isTerminalPlayRunLifecycleStatus(status: unknown): boolean {
 }
 
 export function isActivePlayRunLifecycleStatus(status: unknown): boolean {
-  return normalizePlayRunLifecycleStatus(status) === 'running';
+  const normalized = normalizePlayRunLifecycleStatus(status);
+  return normalized === 'running' || normalized === 'waiting';
 }
 
 export function isRecoverableDatasetRowStatus(status: unknown): boolean {