deepline 0.1.211 → 0.1.212

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -66,6 +66,22 @@ export type RuntimeStatusUpdate = {
    * positional cursor stay correct once the buffer has rotated.
    */
   liveLogTotalCount?: number;
+  /**
+   * Monotonic scheduler attempt that owns the positional log channel. A new
+   * attempt starts at offset zero without colliding with earlier attempts.
+   */
+  liveLogProducerAttempt?: number;
+  datasetLifecycleEvents?: Array<{
+    datasetId: string;
+    path: string;
+    tableNamespace: string;
+    phase: 'registered' | 'available' | 'failed';
+    persistedRows?: number;
+    succeededRows?: number;
+    failedRows?: number;
+    complete?: boolean;
+    at: number;
+  }>;
   /**
    * Explicit terminal-output replay for final runner logs when the caller must
    * keep `status` nonterminal. Direct-worker/Postgres progress finalizers do this so
@@ -75,6 +91,8 @@ export type RuntimeStatusUpdate = {
   terminalLogReplay?: {
     lines: string[];
     totalCount?: number;
+    dedupeMode?: 'semantic' | 'exact';
+    producerAttempt?: number;
   } | null;
   liveTimeline?: PlayRunTimelineEntry[];
   liveNodeProgress?: PlayVisualNodeProgressMap;
@@ -347,12 +365,84 @@ function applyRetryJitter(delayMs: number): number {
 const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
 const RUN_STATUS_LEDGER_SNAPSHOT_CACHE_LIMIT = 1_000;
 const runStatusLedgerSnapshots = new Map<string, PlayRunLedgerSnapshot>();
-// Positional log cursor per run: count of liveLogs lines already forwarded to
-// the ledger from this process. Process-local on purpose — a fresh process
-// re-sends from offset 0 and Run Log Stream ingestion skips the overlap
-// positionally (exactly-once on the channel, repeated identical lines kept).
+// Positional log cursor per run attempt: count of liveLogs lines already
+// forwarded from this process. A fresh process re-sends from offset 0; durable
+// ingestion skips overlap within that same attempt.
 const runLogChannelSentCounts = new Map<string, number>();
 const runStatusUpdateChains = new Map<string, Promise<void>>();
+const APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES = 128 * 1024;
+const APP_RUNTIME_EVENT_BATCH_MAX_JSON_BYTES = 512 * 1024;
+const appRuntimeTextEncoder = new TextEncoder();
+
+function splitRunEventForAppRuntime(
+  event: PlayRunLedgerEvent,
+): PlayRunLedgerEvent[] {
+  if (event.type !== 'log.appended' || event.lines.length === 0) return [event];
+  const split: PlayRunLedgerEvent[] = [];
+  let index = 0;
+  while (index < event.lines.length) {
+    const start = index;
+    let bytes = 2;
+    while (index < event.lines.length) {
+      const lineBytes = appRuntimeTextEncoder.encode(
+        JSON.stringify(event.lines[index]!),
+      ).length;
+      if (
+        index > start &&
+        bytes + lineBytes + 1 > APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES
+      ) {
+        break;
+      }
+      bytes += lineBytes + 1;
+      index += 1;
+    }
+    split.push({
+      ...event,
+      lines: event.lines.slice(start, index),
+      ...(typeof event.channelOffset === 'number'
+        ? { channelOffset: event.channelOffset + start }
+        : {}),
+    });
+  }
+  return split;
+}
+
+export function partitionRunEventsForAppRuntime(
+  events: readonly PlayRunLedgerEvent[],
+): PlayRunLedgerEvent[][] {
+  const batches: PlayRunLedgerEvent[][] = [];
+  let current: PlayRunLedgerEvent[] = [];
+  let currentBytes = 2;
+  for (const event of events.flatMap(splitRunEventForAppRuntime)) {
+    const eventBytes = appRuntimeTextEncoder.encode(
+      JSON.stringify(event),
+    ).length;
+    if (
+      current.length > 0 &&
+      currentBytes + eventBytes + 1 > APP_RUNTIME_EVENT_BATCH_MAX_JSON_BYTES
+    ) {
+      batches.push(current);
+      current = [];
+      currentBytes = 2;
+    }
+    current.push(event);
+    currentBytes += eventBytes + 1;
+  }
+  if (current.length > 0) batches.push(current);
+  return batches;
+}
+
+async function appendRunEventBatchesViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: { playId: string; events: readonly PlayRunLedgerEvent[] },
+): Promise<void> {
+  for (const events of partitionRunEventsForAppRuntime(input.events)) {
+    await appendRunEventsViaAppRuntime(context, {
+      playId: input.playId,
+      events,
+    });
+  }
+}
 
 function isTerminalRuntimeStatus(status: string): boolean {
   return (
@@ -364,6 +454,10 @@ function isTerminalRuntimeStatus(status: string): boolean {
   );
 }
 
+function runLogChannelKey(runId: string, producerAttempt?: number): string {
+  return `${runId}:${producerAttempt ?? 'legacy'}`;
+}
+
 function rememberRunStatusLedgerSnapshot(
   runId: string,
   snapshot: PlayRunLedgerSnapshot,
@@ -376,7 +470,11 @@ function rememberRunStatusLedgerSnapshot(
     const oldestRunId = runStatusLedgerSnapshots.keys().next().value;
     if (!oldestRunId) break;
     runStatusLedgerSnapshots.delete(oldestRunId);
-    runLogChannelSentCounts.delete(oldestRunId);
+    for (const key of runLogChannelSentCounts.keys()) {
+      if (key.startsWith(`${oldestRunId}:`)) {
+        runLogChannelSentCounts.delete(key);
+      }
+    }
     runStatusUpdateChains.delete(oldestRunId);
   }
 }
@@ -399,12 +497,16 @@ function isRetryableAppRuntimeResponse(input: {
   if (input.status === 500 && isTransientAppRuntimeFailureBody(input.body)) {
     return true;
   }
+  return isRetryableAppRuntimeResponseStatus(input.status);
+}
+
+function isRetryableAppRuntimeResponseStatus(status: number): boolean {
   return (
-    input.status === 429 ||
-    (input.status >= 520 && input.status <= 524) ||
-    input.status === 502 ||
-    input.status === 503 ||
-    input.status === 504
+    status === 429 ||
+    (status >= 520 && status <= 524) ||
+    status === 502 ||
+    status === 503 ||
+    status === 504
   );
 }
 
@@ -601,6 +703,38 @@ function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+async function retryAppRuntimeBodyTimeoutOrThrow(input: {
+  action: RuntimeApiRequest['action'];
+  attempt: number;
+  maxAttempts: number;
+  error: unknown;
+  retryAfterMs?: number | null;
+}): Promise<void> {
+  if (!isRequestTimeoutAbort(input.error)) {
+    throw input.error;
+  }
+  if (
+    input.attempt < input.maxAttempts &&
+    isRetryableAppRuntimeFetchError({
+      action: input.action,
+      error: input.error,
+    })
+  ) {
+    await sleep(
+      Math.max(
+        applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
+        input.retryAfterMs ?? 0,
+      ),
+    );
+    return;
+  }
+  throw new AppRuntimeApiTransportError({
+    action: input.action,
+    attempts: input.attempt,
+    cause: input.error,
+  });
+}
+
 export class AppRuntimeApiTransportError extends Error {
   readonly action: RuntimeApiRequest['action'];
   readonly attempts: number;
@@ -691,9 +825,40 @@ async function postAppRuntimeApi<TResponse>(
       });
     }
     if (response.ok) {
-      return (await response.json()) as TResponse;
+      try {
+        return (await response.json()) as TResponse;
+      } catch (error) {
+        await retryAppRuntimeBodyTimeoutOrThrow({
+          action: body.action,
+          attempt,
+          maxAttempts,
+          error,
+        });
+        continue;
+      }
+    }
+    let responseText: string;
+    try {
+      responseText = await response.text();
+    } catch (error) {
+      if (
+        isRequestTimeoutAbort(error) &&
+        !isRetryableAppRuntimeResponseStatus(response.status)
+      ) {
+        throw new Error(
+          `App runtime API ${body.action} failed with status ${response.status}: response body timed out`,
+          { cause: error },
+        );
+      }
+      await retryAppRuntimeBodyTimeoutOrThrow({
+        action: body.action,
+        attempt,
+        maxAttempts,
+        error,
+        retryAfterMs: appRuntimeRetryAfterMs(response, ''),
+      });
+      continue;
     }
-    const responseText = await response.text();
     if (
       attempt < maxAttempts &&
       isRetryableAppRuntimeResponse({
@@ -917,7 +1082,9 @@ async function updateRunStatusViaAppRuntimeUnlocked(
   // durable stream can recover any progress flush that was lost in transit.
   const sentCount = isTerminalRuntimeStatus(update.status)
     ? 0
-    : (runLogChannelSentCounts.get(update.playId) ?? 0);
+    : (runLogChannelSentCounts.get(
+        runLogChannelKey(update.playId, update.liveLogProducerAttempt),
+      ) ?? 0);
   const logSlice = Array.isArray(update.liveLogs)
     ? slicePositionalLogLines({
         bufferLines: update.liveLogs,
@@ -932,6 +1099,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
         ? {
             lines: update.liveLogs,
             totalCount: update.liveLogTotalCount,
+            producerAttempt: update.liveLogProducerAttempt,
           }
         : null;
   const statusPatchLogSlice = terminalLogReplay ? null : logSlice;
@@ -944,6 +1112,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
       lastCheckpointAt: update.lastCheckpointAt ?? null,
       liveLogs: statusPatchLogSlice?.lines ?? null,
       liveLogsChannelOffset: statusPatchLogSlice?.channelOffset ?? null,
+      liveLogsProducerAttempt: update.liveLogProducerAttempt ?? null,
       liveNodeProgress: update.liveNodeProgress ?? null,
       result: update.result,
     },
@@ -951,33 +1120,87 @@ async function updateRunStatusViaAppRuntimeUnlocked(
     now: update.lastCheckpointAt ?? Date.now(),
     source: 'worker',
   });
-  if (terminalLogReplay && terminalLogReplay.lines.length > 0) {
-    events.push(
-      ...buildTerminalLogReplayEvents({
-        runId: update.playId,
-        source: 'worker',
-        occurredAt: update.lastCheckpointAt ?? Date.now(),
-        lines: terminalLogReplay.lines,
-        liveLogTotalCount: terminalLogReplay.totalCount,
-      }),
-    );
+  for (const event of update.datasetLifecycleEvents ?? []) {
+    const current = previousSnapshot.datasetsById[event.datasetId];
+    const changesSnapshot =
+      !current ||
+      current.path !== event.path ||
+      current.tableNamespace !== event.tableNamespace ||
+      current.phase !== event.phase ||
+      (event.persistedRows ?? 0) > current.persistedRows ||
+      (event.succeededRows ?? 0) > current.succeededRows ||
+      (event.failedRows ?? 0) > current.failedRows ||
+      (event.complete === true && current.complete !== true);
+    if (!changesSnapshot) continue;
+    events.push({
+      type: 'dataset.lifecycle',
+      runId: update.playId,
+      source: 'worker',
+      occurredAt: event.at,
+      datasetId: event.datasetId,
+      path: event.path,
+      tableNamespace: event.tableNamespace,
+      phase: event.phase,
+      ...(event.persistedRows === undefined
+        ? {}
+        : { persistedRows: event.persistedRows }),
+      ...(event.succeededRows === undefined
+        ? {}
+        : { succeededRows: event.succeededRows }),
+      ...(event.failedRows === undefined
+        ? {}
+        : { failedRows: event.failedRows }),
+      ...(event.complete === undefined ? {} : { complete: event.complete }),
+    });
   }
-  if (events.length === 0) {
+  const terminalLogEvents =
+    terminalLogReplay && terminalLogReplay.lines.length > 0
+      ? buildTerminalLogReplayEvents({
+          runId: update.playId,
+          source: 'worker',
+          occurredAt: update.lastCheckpointAt ?? Date.now(),
+          lines: terminalLogReplay.lines,
+          liveLogTotalCount: terminalLogReplay.totalCount,
+          dedupeMode: terminalLogReplay.dedupeMode,
+          producerAttempt: terminalLogReplay.producerAttempt,
+        })
+      : [];
+  if (events.length === 0 && terminalLogEvents.length === 0) {
     return;
   }
-  await appendRunEventsViaAppRuntime(context, {
-    playId: update.playId,
-    events,
-  });
+  const combinedEvents = [...events, ...terminalLogEvents];
+  if (partitionRunEventsForAppRuntime(combinedEvents).length === 1) {
+    // Preserve the single-mutation fast path for ordinary bounded updates.
+    await appendRunEventsViaAppRuntime(context, {
+      playId: update.playId,
+      events: combinedEvents,
+    });
+  } else {
+    // Persist terminal lifecycle first so an oversized or interrupted
+    // best-effort log heal can never prevent settlement. Each bounded log
+    // batch is a separate mutation; retries are safe because ingestion owns
+    // positional reconciliation.
+    await appendRunEventBatchesViaAppRuntime(context, {
+      playId: update.playId,
+      events,
+    });
+    await appendRunEventBatchesViaAppRuntime(context, {
+      playId: update.playId,
+      events: terminalLogEvents,
+    });
+  }
   if (statusPatchLogSlice) {
     runLogChannelSentCounts.set(
-      update.playId,
+      runLogChannelKey(update.playId, update.liveLogProducerAttempt),
       statusPatchLogSlice.channelOffset + statusPatchLogSlice.lines.length,
     );
   }
   rememberRunStatusLedgerSnapshot(
     update.playId,
-    reducePlayRunLedgerEvents(previousSnapshot, events),
+    reducePlayRunLedgerEvents(previousSnapshot, [
+      ...events,
+      ...terminalLogEvents,
+    ]),
   );
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -106,6 +106,7 @@ import {
   parseToolExecuteAuthScopeChangedError,
   ToolExecuteAuthScopeChangedError,
 } from './tool-execute-retry-policy';
+import { ToolHttpError } from './tool-http-errors';
 import {
   buildDurableCtxCallCacheKey,
   buildDurableRunPlayInvocationScope,
@@ -3486,7 +3487,8 @@ export class PlayContextImpl {
       );
     }
     const finalWrapped =
-      completed?.status === 'completed' || completed?.status === 'skipped'
+      (completed?.status === 'completed' || completed?.status === 'skipped') &&
+      completed.output !== undefined
         ? await this.wrapToolExecutionResult({
             toolId,
             status:
@@ -6319,11 +6321,6 @@ export class PlayContextImpl {
             semanticKey: toolRequestIdentity,
             force: toolCachePolicy.force,
             staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
-            markSkipped: () => {
-              this.log(
-                `ctx.tools.execute(${toolId}): no-op due completed receipt ${toolRequestIdentity} (label: ${normalizedKey})`,
-              );
-            },
             onClaimedResult: (output, receiptKey) =>
               markToolExecuteResultExecutionOutcome(output, {
                 kind: 'live',
@@ -7277,11 +7274,6 @@ export class PlayContextImpl {
             egressSlot.release();
           }
         },
-        markSkipped: (output) => {
-          this.log(
-            `ctx.fetch(${output?.url ?? ''}): no-op due completed receipt`,
-          );
-        },
       },
     );
   }
@@ -7348,7 +7340,6 @@ export class PlayContextImpl {
           : options?.semanticKey,
         staleAfterSeconds: options?.staleAfterSeconds,
         markSkipped: (output) => {
-          this.log(`ctx.step(${normalizedKey}): no-op due completed receipt`);
           assertJsonSerializableStepOutput(normalizedKey, output);
         },
         execute: executeStep,
@@ -8500,6 +8491,8 @@ export class PlayContextImpl {
 
           while (true) {
             let response: Response;
+            let providerCallStartedAt: number | null = null;
+            let providerCallElapsedMs: number | null = null;
             const durableCallReceiptKey =
               options?.durableCallReceiptKey?.trim() || null;
             const executionAuthScopeDigest =
@@ -8533,7 +8526,6 @@ export class PlayContextImpl {
                 }, timeoutMs)
               : null;
             try {
-              const providerCallStartedAt = Date.now();
               const ownershipStartedAt = Date.now();
               await options?.beforeProviderCall?.();
               if (runtimeReceiptReadTraceEnabled) {
@@ -8541,6 +8533,7 @@ export class PlayContextImpl {
                   `[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`,
                 );
               }
+              providerCallStartedAt = Date.now();
               const heartbeatIntervalMs = hasReceiptHeartbeat
                 ? runtimeLeaseHeartbeatIntervalFromExpiry({
                     leaseExpiresAt: options?.receiptLeaseExpiresAt,
@@ -8640,10 +8633,11 @@ export class PlayContextImpl {
                     `[perf] tool call id=${toolId} phase=integration_fetch_headers elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
                   );
                 }
+                providerCallElapsedMs = Date.now() - providerCallStartedAt;
                 this.resourceGovernor.observe({
                   toolId,
                   providerResourceKey: `tool:${toolId}`,
-                  providerLatencyMs: Date.now() - providerCallStartedAt,
+                  providerLatencyMs: providerCallElapsedMs,
                   providerSuccess: response.ok,
                   provider429: response.status === 429,
                 });
@@ -8654,16 +8648,18 @@ export class PlayContextImpl {
               if (heartbeatFailure) {
                 throw heartbeatFailure;
               }
-              if (abortController?.signal.aborted) {
-                throw abortController.signal.reason instanceof Error
+              const transportError = abortController?.signal.aborted
+                ? abortController.signal.reason instanceof Error
                   ? abortController.signal.reason
                   : new Error(
                       `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
-                    );
-              }
+                    )
+                : error;
               transportAttempt += 1;
               const message =
-                error instanceof Error ? error.message : String(error);
+                transportError instanceof Error
+                  ? transportError.message
+                  : String(transportError);
               if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
                 await this.governor.chargeBudget('retry');
                 const retryAfterMs =
@@ -8679,8 +8675,11 @@ export class PlayContextImpl {
                 await this.sleepWithCheckpointHeartbeat(retryAfterMs);
                 continue;
               }
-              throw new Error(
+              throw new ToolHttpError(
                 `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
+                null,
+                0,
+                'repairable',
               );
             } finally {
               if (timeoutHandle) {
@@ -8711,6 +8710,9 @@ export class PlayContextImpl {
                 bodyText: text,
                 retryAfterHeader: response.headers.get('retry-after'),
                 transientHttpRetrySafe: retrySafeTransientHttp,
+                ...(providerCallElapsedMs !== null
+                  ? { providerLatencyMs: providerCallElapsedMs }
+                  : {}),
               });
               if (failure.backpressureDelayMs !== null) {
                 // Feed the server-observed Retry-After back into the shared

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -380,6 +380,18 @@ export interface MapExecutionScope {
 }
 
 export type PlayExecutionEvent =
+  | {
+      type: 'dataset.lifecycle';
+      datasetId: string;
+      path: string;
+      tableNamespace: string;
+      phase: 'registered' | 'available' | 'failed';
+      persistedRows?: number;
+      succeededRows?: number;
+      failedRows?: number;
+      complete?: boolean;
+      at: number;
+    }
   | {
       type: 'map.preparing';
       mapInvocationId: string;

package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts

@@ -1,10 +1,16 @@
-import { createHash } from 'node:crypto';
+import {
+  normalizePlayNameForSheet,
+  normalizeTableNamespace,
+  sha256Hex,
+} from '../plays/row-identity';
 
 export function createRuntimeDatasetId(
   playName: string,
   tableNamespace: string,
 ): string {
-  return createHash('sha1')
-    .update(`${playName}:${tableNamespace}`)
-    .digest('hex');
+  const normalizedPlayName = normalizePlayNameForSheet(playName);
+  const normalizedTableNamespace = normalizeTableNamespace(tableNamespace);
+  return sha256Hex(
+    `${normalizedPlayName}:${normalizedTableNamespace}`,
+  ).slice(0, 40);
 }

package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts

@@ -56,6 +56,7 @@ export type CreateDbSessionRequest = {
 
 export type CreateDbSessionResponse = {
   sessionId: string;
+  executionRole?: string;
   postgresUrl?: string;
   encryptedPostgresUrl?: EncryptedPostgresUrl;
   postgres?: {
@@ -398,6 +399,14 @@ export const createDbSessionResponseSchema =
     }
     return {
       sessionId: parseString(value.sessionId, 'sessionId'),
+      ...(value.executionRole !== undefined
+        ? {
+            executionRole: parseOptionalString(
+              value.executionRole,
+              'executionRole',
+            ),
+          }
+        : {}),
       ...(value.postgresUrl !== undefined
         ? { postgresUrl: parseOptionalString(value.postgresUrl, 'postgresUrl') }
         : {}),

package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts

@@ -412,9 +412,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     receipt: RuntimeStepReceipt,
     source: DurableReceiptRecoverySource = 'cache',
   ): Promise<T> => {
-    input.log(
-      `ctx.${input.operation}(${input.id}): recovered result from receipt`,
-    );
+    input.log(`ctx.${input.operation}(${input.id}): reused completed work`);
     if (receipt.output === undefined) {
       return receipt.output as T;
     }
@@ -447,7 +445,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     }
     if (reclaimed?.status === 'failed') {
       throw new Error(
-        `ctx.${input.operation}(${input.id}): receipt is failed and could not be reclaimed: ${reclaimed.error ?? 'unknown error'}.`,
+        `ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused: ${reclaimed.error ?? 'unknown error'}.`,
       );
     }
     if (
@@ -509,7 +507,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     receipt: RuntimeStepReceipt,
   ): Promise<never> => {
     throw new Error(
-      `ctx.${input.operation}(${input.id}): receipt is failed and could not be claimed: ${receipt.error ?? 'unknown error'}.`,
+      `ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused: ${receipt.error ?? 'unknown error'}.`,
     );
   };
 

package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts

@@ -1,7 +1,10 @@
 import { createSecretRedactionContext } from './secret-redaction';
-import { RUNTIME_RECEIPT_OUTPUT_MAX_BYTES } from './output-size-limits';
+import {
+  assertRuntimeReceiptOutputWithinLimit,
+  TERMINAL_RUN_RESULT_MAX_BYTES,
+} from './output-size-limits';
 
-export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
+export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = TERMINAL_RUN_RESULT_MAX_BYTES;
 export const MAX_LEDGER_LOG_LINES_PER_EVENT = 500;
 export const MAX_LEDGER_LOG_LINE_LENGTH = 2_000;
 
@@ -21,6 +24,15 @@ export function sanitizeLedgerPayload(value: unknown): unknown {
   return ledgerIngressRedactor.redact(value);
 }
 
+/** Secret-redact a receipt payload without coupling its 10 MiB cell limit to the ledger. */
+export function sanitizeRuntimeReceiptPayload(value: unknown): unknown {
+  assertRuntimeReceiptOutputWithinLimit({
+    output: value,
+    path: 'runtime receipt output',
+  });
+  return ledgerIngressRedactor.redact(value);
+}
+
 export function sanitizeLedgerLogLines(value: unknown): string[] {
   if (!Array.isArray(value)) return [];
   return value

package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts

@@ -3,8 +3,10 @@ const encoder = typeof TextEncoder === 'undefined' ? null : new TextEncoder();
 export const TERMINAL_RUN_RESULT_MAX_BYTES = 2_500_000;
 export const CUSTOMER_OUTPUT_VALUE_MAX_BYTES = 1 * 1024 * 1024;
 export const CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 2 * 1024 * 1024;
-export const RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 2 * 1024 * 1024;
-export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 16 * 1024 * 1024;
+/** Inline Postgres receipt contract: one serialized output is at most 10 MiB. */
+export const RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
+/** One completion request may contain multiple receipts up to 32 MiB total. */
+export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
 
 export class OutputTooLargeError extends Error {
   readonly code = 'OUTPUT_TOO_LARGE';

package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts

@@ -1,6 +1,26 @@
 import { RECEIPT_STATUS_CODE } from './receipt-status';
 import { workReceiptFailureKindCode } from './work-receipts';
 
+export function workReceiptRepairableFailurePredicateSql(
+  receiptTable: string,
+): string {
+  return `(
+    ${receiptTable}.failure_kind = ${workReceiptFailureKindCode('repairable')}::smallint
+    OR (
+      ${receiptTable}.failure_kind = ${workReceiptFailureKindCode('terminal')}::smallint
+      AND ${receiptTable}.error IS NOT NULL
+      AND lower(${receiptTable}.error) NOT LIKE '%billing cap%'
+      AND lower(${receiptTable}.error) NOT LIKE '%insufficient credits%'
+      AND lower(${receiptTable}.error) NOT LIKE '%monthly billing limit%'
+      AND (
+        ${receiptTable}.error ~* '(^|[^0-9])5[0-9]{2}([^0-9]|$)'
+        OR lower(${receiptTable}.error) LIKE '%transport failed calling%'
+        OR lower(${receiptTable}.error) LIKE '%runtime api call timed out%'
+      )
+    )
+  )`;
+}
+
 export function workReceiptClaimableStatusCodes(input: {
   forceRefresh?: boolean;
   forceFailedRefresh?: boolean;
@@ -47,7 +67,7 @@ export function workReceiptClaimConflictPredicateSql(input: {
               )
               OR ${table}.status <> ${RECEIPT_STATUS_CODE.failed}::smallint
               OR (
-                ${table}.failure_kind = ${workReceiptFailureKindCode('repairable')}::smallint
+                ${workReceiptRepairableFailurePredicateSql(table)}
                 AND (
                   ${table}.run_id IS DISTINCT FROM ${input.claimantRunIdSql}
                   OR COALESCE(${table}.lease_owner_attempt, 0) < ${claimantRunAttemptSql}::integer