deepline 0.1.211 → 0.1.213

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -24,6 +24,7 @@ import type { PreviousCell } from './cell-staleness';
 import type { MapRowOutcome } from './durability-store';
 import type { WorkReceiptFailureKind } from './work-receipts';
 import type { RunExecutionScope } from './run-execution-scope';
+import type { PlayCallExecution } from './play-call-execution';
 
 export interface RowState {
   results: Map<string, unknown>;
@@ -69,12 +70,26 @@ export interface RuntimeStepReceipt {
   claimState?: 'claimed' | 'existing';
 }
 
+export type AcquireRuntimeReceiptExecutionLockInput = {
+  key: string;
+  runId: string;
+  ownerExecutionId: string;
+  ttlMs: number;
+};
+
+export type ReleaseRuntimeReceiptExecutionLockInput = {
+  key: string;
+  runId: string;
+  ownerExecutionId: string;
+};
+
 export interface GetRuntimeStepReceiptInput {
   key: string;
 }
 
 export interface ClaimRuntimeStepReceiptInput {
   key: string;
+  leaseId?: string;
   runId: string;
   runAttempt?: number | null;
   leaseAware?: boolean;
@@ -128,6 +143,7 @@ export interface GetRuntimeStepReceiptsInput {
 
 export interface ClaimRuntimeStepReceiptsInput {
   keys: string[];
+  leaseIds?: string[];
   runId: string;
   runAttempt?: number | null;
   leaseAware?: boolean;
@@ -281,6 +297,7 @@ export type CustomerDbQueryHandler = (
 
 export interface PlayCallOptions {
   description?: string;
+  execution?: PlayCallExecution;
 }
 
 export interface StepOptions {
@@ -380,6 +397,18 @@ export interface MapExecutionScope {
 }
 
 export type PlayExecutionEvent =
+  | {
+      type: 'dataset.lifecycle';
+      datasetId: string;
+      path: string;
+      tableNamespace: string;
+      phase: 'registered' | 'available' | 'failed';
+      persistedRows?: number;
+      succeededRows?: number;
+      failedRows?: number;
+      complete?: boolean;
+      at: number;
+    }
   | {
       type: 'map.preparing';
       mapInvocationId: string;
@@ -585,7 +614,10 @@ export interface ContextOptions {
     rows: MapRowOutcome[];
     outputFields: string[];
     staticPipeline?: PlayStaticPipeline | null;
-  }) => Promise<void>;
+  }) => Promise<void | {
+    updated: number;
+    staleDroppedKeys?: string[];
+  }>;
   playId?: string;
   runId?: string;
   /** Physical executor run that owns leases for an inline child context. */
@@ -622,6 +654,7 @@ export interface ContextOptions {
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolRetryPolicy?: (toolId: string) => Promise<{
     retrySafeTransientHttp?: boolean;
+    requiresExecutionFence?: boolean;
   } | null>;
   getToolActionCacheVersion?: (toolId: string) => Promise<string> | string;
   getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
@@ -660,6 +693,12 @@ export interface ContextOptions {
   getRuntimeStepReceipt?: (
     input: GetRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  acquireRuntimeReceiptExecutionLock?: (
+    input: AcquireRuntimeReceiptExecutionLockInput,
+  ) => Promise<{ ownerExecutionId: string; expiresAt: string } | null>;
+  releaseRuntimeReceiptExecutionLock?: (
+    input: ReleaseRuntimeReceiptExecutionLockInput,
+  ) => Promise<boolean>;
   getRuntimeStepReceipts?: (
     input: GetRuntimeStepReceiptsInput,
   ) =>
@@ -920,6 +959,7 @@ export type PlayStep =
   | {
       type: 'play_call';
       playId: string;
+      execution?: PlayCallExecution;
       results?: PlayStepRowResult[];
       nestedSteps: PlayStep[];
       description?: string;
@@ -959,6 +999,7 @@ export type PlayDatasetSubstep =
   | {
       type: 'play_call';
       playId: string;
+      execution?: PlayCallExecution;
       results?: PlayStepRowResult[];
       nestedSteps: PlayStep[];
       description?: string;

package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts

@@ -1,10 +1,16 @@
-import { createHash } from 'node:crypto';
+import {
+  normalizePlayNameForSheet,
+  normalizeTableNamespace,
+  sha256Hex,
+} from '../plays/row-identity';
 
 export function createRuntimeDatasetId(
   playName: string,
   tableNamespace: string,
 ): string {
-  return createHash('sha1')
-    .update(`${playName}:${tableNamespace}`)
-    .digest('hex');
+  const normalizedPlayName = normalizePlayNameForSheet(playName);
+  const normalizedTableNamespace = normalizeTableNamespace(tableNamespace);
+  return sha256Hex(
+    `${normalizedPlayName}:${normalizedTableNamespace}`,
+  ).slice(0, 40);
 }

package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts

@@ -56,6 +56,7 @@ export type CreateDbSessionRequest = {
 
 export type CreateDbSessionResponse = {
   sessionId: string;
+  executionRole?: string;
   postgresUrl?: string;
   encryptedPostgresUrl?: EncryptedPostgresUrl;
   postgres?: {
@@ -398,6 +399,14 @@ export const createDbSessionResponseSchema =
     }
     return {
       sessionId: parseString(value.sessionId, 'sessionId'),
+      ...(value.executionRole !== undefined
+        ? {
+            executionRole: parseOptionalString(
+              value.executionRole,
+              'executionRole',
+            ),
+          }
+        : {}),
       ...(value.postgresUrl !== undefined
         ? { postgresUrl: parseOptionalString(value.postgresUrl, 'postgresUrl') }
         : {}),

package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts

@@ -25,6 +25,9 @@ const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
 const TOOL_RECEIPT_DEFAULT_WAIT_MS = 300_000;
 const TOOL_RECEIPT_COMPLETION_BUFFER_MS = 30_000;
 const TOOL_RECEIPT_MAX_WAIT_MS = 30 * 60_000;
+const DANGEROUS_SIDE_EFFECT_LOCK_WAIT_MS = 15_000;
+export const COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID =
+  '__deepline_completed_cache_insert__';
 
 export class RuntimeReceiptWaitTimeoutError extends Error {
   constructor(key: string) {
@@ -179,6 +182,15 @@ export type DurableReceiptExecutionStore = {
   ): Promise<RuntimeStepReceipt | null>;
   canPersistFailure: boolean;
   canPersistCompletion: boolean;
+  acquireExecutionLock?: (input: {
+    receiptKey: string;
+    ownerExecutionId: string;
+    ttlMs: number;
+  }) => Promise<{ ownerExecutionId: string; expiresAt: string } | null>;
+  releaseExecutionLock?: (input: {
+    receiptKey: string;
+    ownerExecutionId: string;
+  }) => Promise<boolean>;
 };
 
 async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
@@ -243,7 +255,7 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
       rejectLeaseLost(error as RuntimeReceiptLeaseLostError),
   });
 
-  // markRunning is the authoritative pre-dispatch ownership fence. Renewal is
+  // The claim is the authoritative pre-dispatch ownership fence. Renewal is
   // only needed when the lease approaches expiry; an immediate read/renewal
   // adds a round trip to every short tool call without extending ownership.
   // Completion remains conditional on this lease, and long work still renews.
@@ -390,6 +402,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
   shouldPersistFailure?: (error: unknown) => boolean;
   heartbeatIntervalMs?: number;
   markRunningBeforeExecute?: boolean;
+  /** New receipt-cache path: read completed facts, execute without creating an
+   * in-flight receipt, then atomically insert the completed fact. */
+  completedCacheOnly?: boolean;
+  requiresExecutionLock?: boolean;
+  executionLockTtlMs?: number;
   formatError: (error: unknown) => string;
   log: (message: string) => void;
   execute: (context: { leaseId: string | null }) => Promise<T>;
@@ -412,9 +429,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     receipt: RuntimeStepReceipt,
     source: DurableReceiptRecoverySource = 'cache',
   ): Promise<T> => {
-    input.log(
-      `ctx.${input.operation}(${input.id}): recovered result from receipt`,
-    );
+    input.log(`ctx.${input.operation}(${input.id}): reused completed work`);
     if (receipt.output === undefined) {
       return receipt.output as T;
     }
@@ -428,6 +443,143 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     return recovered;
   };
 
+  if (input.completedCacheOnly === true) {
+    if (input.force !== true) {
+      const cached = await input.store.get(input.receiptKey);
+      if (cached?.status === 'completed' || cached?.status === 'skipped') {
+        return await recoverCompletedReceipt(cached);
+      }
+    }
+    const needsLock = input.requiresExecutionLock === true;
+    const ownerExecutionId = `${input.runId}:${crypto.randomUUID()}`;
+    let ownsLock = false;
+    let lockSupervisor: ReturnType<
+      typeof createRuntimeReceiptHeartbeatSupervisor
+    > | null = null;
+    if (needsLock) {
+      if (
+        !input.store.acquireExecutionLock ||
+        !input.store.releaseExecutionLock
+      ) {
+        throw new Error(
+          `ctx.${input.operation}(${input.id}): non-idempotent provider requires the receipt execution-lock backend.`,
+        );
+      }
+      const deadline = Date.now() + DANGEROUS_SIDE_EFFECT_LOCK_WAIT_MS;
+      while (!ownsLock) {
+        const lock = await input.store.acquireExecutionLock({
+          receiptKey: input.receiptKey,
+          ownerExecutionId,
+          ttlMs: input.executionLockTtlMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS,
+        });
+        ownsLock = lock?.ownerExecutionId === ownerExecutionId;
+        if (ownsLock) break;
+        const winner = await input.store.get(input.receiptKey);
+        if (
+          input.force !== true &&
+          (winner?.status === 'completed' || winner?.status === 'skipped')
+        ) {
+          return await recoverCompletedReceipt(winner, 'in_flight');
+        }
+        if (Date.now() >= deadline) {
+          throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
+        }
+        await sleepReceiptWait(DURABLE_RECEIPT_WAIT_DELAY_MS);
+      }
+      const afterLock = await input.store.get(input.receiptKey);
+      if (
+        input.force !== true &&
+        (afterLock?.status === 'completed' ||
+          afterLock?.status === 'skipped')
+      ) {
+        await input.store.releaseExecutionLock({
+          receiptKey: input.receiptKey,
+          ownerExecutionId,
+        });
+        return await recoverCompletedReceipt(afterLock, 'in_flight');
+      }
+      const lockTtlMs =
+        input.executionLockTtlMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS;
+      lockSupervisor = createRuntimeReceiptHeartbeatSupervisor({
+        intervalMs: Math.max(1_000, Math.floor(lockTtlMs / 3)),
+        heartbeat: async () => {
+          const renewed = await input.store.acquireExecutionLock!({
+            receiptKey: input.receiptKey,
+            ownerExecutionId,
+            ttlMs: lockTtlMs,
+          });
+          if (renewed?.ownerExecutionId !== ownerExecutionId) {
+            throw new RuntimeReceiptLeaseLostError({
+              receiptKey: input.receiptKey,
+              runId: input.runId,
+              leaseId: ownerExecutionId,
+            });
+          }
+          return 'active';
+        },
+        isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
+        onLeaseLost: (error) => {
+          input.log(
+            `ctx.${input.operation}(${input.id}): dangerous side-effect execution fence lost: ${error instanceof Error ? error.message : String(error)}`,
+          );
+        },
+      });
+      lockSupervisor.start();
+    }
+    const releaseOwnLock = async (): Promise<void> => {
+      lockSupervisor?.stop();
+      lockSupervisor = null;
+      if (ownsLock) {
+        ownsLock = false;
+        await input.store.releaseExecutionLock!({
+          receiptKey: input.receiptKey,
+          ownerExecutionId,
+        });
+      }
+    };
+    let executed: T;
+    try {
+      executed = await input.execute({ leaseId: null });
+    } catch (error) {
+      await releaseOwnLock();
+      throw error;
+    }
+    try {
+      const result = input.onClaimedResult
+        ? input.onClaimedResult(executed, input.receiptKey)
+        : executed;
+      assertNoSecretTaint(result, `ctx.${input.operation} result`);
+      // Force bypasses the immutable cache. It does not grant permission to
+      // replace the completed fact already published under this semantic key.
+      if (input.force === true) {
+        return result;
+      }
+      const completed = await input.store.complete(
+        input.receiptKey,
+        input.runId,
+        isToolExecuteResult(result)
+          ? serializeToolExecuteResult(result)
+          : result,
+        COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+      );
+      if (
+        completed?.status === 'completed' ||
+        completed?.status === 'skipped'
+      ) {
+        return await recoverCompletedReceipt(completed, 'owner');
+      }
+      const winner = await input.store.get(input.receiptKey);
+      if (winner?.status === 'completed' || winner?.status === 'skipped') {
+        return await recoverCompletedReceipt(winner, 'owner');
+      }
+      throw new Error(
+        `ctx.${input.operation}(${input.id}): completed receipt cache insert failed.`,
+      );
+    } finally {
+      await releaseOwnLock();
+    }
+  }
+
   let ownedLeaseId: string | null = null;
   let ownedLeaseExpiresAt: string | null = null;
 
@@ -447,7 +599,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     }
     if (reclaimed?.status === 'failed') {
       throw new Error(
-        `ctx.${input.operation}(${input.id}): receipt is failed and could not be reclaimed: ${reclaimed.error ?? 'unknown error'}.`,
+        `ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused: ${reclaimed.error ?? 'unknown error'}.`,
       );
     }
     if (
@@ -509,7 +661,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     receipt: RuntimeStepReceipt,
   ): Promise<never> => {
     throw new Error(
-      `ctx.${input.operation}(${input.id}): receipt is failed and could not be claimed: ${receipt.error ?? 'unknown error'}.`,
+      `ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused: ${receipt.error ?? 'unknown error'}.`,
     );
   };
 
@@ -556,7 +708,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
 
   let result: T;
   try {
-    if (input.markRunningBeforeExecute !== false && input.store.markRunning) {
+    // New runners treat a claimed, nonterminal receipt as active. Keep the
+    // mark-running transition as an explicit compatibility option for older
+    // stores during their migration window, but do not pay the extra durable
+    // mutation on the normal claim -> terminal path.
+    if (input.markRunningBeforeExecute === true && input.store.markRunning) {
       const markRunningStartedAt = Date.now();
       const running = await input.store.markRunning(
         input.receiptKey,

package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts

@@ -0,0 +1,93 @@
+import { createHash } from 'node:crypto';
+
+import { createSingleFlight } from './single-flight';
+
+export type GatewayAuthSession<TAuth, TLaunch> = {
+  auth: TAuth;
+  launch: TLaunch;
+  source: 'hit' | 'miss' | 'coalesced';
+};
+
+type StoredSession<TAuth, TLaunch> = {
+  auth: TAuth;
+  launch: TLaunch;
+  expiresAt: number;
+  runId: string;
+  runAttempt: number | null;
+};
+
+/** Briefly reuse an authenticated immutable launch, bounded by token identity. */
+export function createGatewayAuthSessionCache<TAuth, TLaunch>(options: {
+  ttlMs: number;
+  maxEntries: number;
+  identity: (auth: TAuth) => {
+    runId: string | null;
+    runAttempt: number | null;
+  };
+  now?: () => number;
+}) {
+  const sessions = new Map<string, StoredSession<TAuth, TLaunch>>();
+  const flights = createSingleFlight<string, StoredSession<TAuth, TLaunch>>();
+  const now = options.now ?? Date.now;
+  const fingerprint = (token: string) =>
+    createHash('sha256').update(token).digest('base64url');
+
+  const prune = (at: number) => {
+    for (const [key, session] of sessions) {
+      if (session.expiresAt <= at) sessions.delete(key);
+    }
+    while (sessions.size >= options.maxEntries) {
+      const oldest = sessions.keys().next().value;
+      if (!oldest) break;
+      sessions.delete(oldest);
+    }
+  };
+
+  return {
+    async getOrEstablish(input: {
+      token: string;
+      tokenExpiresAt: number;
+      establish: () => Promise<{ auth: TAuth; launch: TLaunch }>;
+    }): Promise<GatewayAuthSession<TAuth, TLaunch>> {
+      const key = fingerprint(input.token);
+      const at = now();
+      const cached = sessions.get(key);
+      if (cached && cached.expiresAt > at) {
+        const identity = options.identity(cached.auth);
+        if (
+          identity.runId === cached.runId &&
+          identity.runAttempt === cached.runAttempt
+        ) {
+          return { ...cached, source: 'hit' };
+        }
+      }
+      if (cached) sessions.delete(key);
+
+      const coalesced = flights.has(key);
+      const session = await flights.run(key, async () => {
+        const established = await input.establish();
+        const identity = options.identity(established.auth);
+        if (!identity.runId) {
+          throw new Error('Gateway authentication produced no run identity.');
+        }
+        const establishedAt = now();
+        const stored: StoredSession<TAuth, TLaunch> = {
+          ...established,
+          expiresAt: Math.min(
+            establishedAt + options.ttlMs,
+            input.tokenExpiresAt,
+          ),
+          runId: identity.runId,
+          runAttempt: identity.runAttempt,
+        };
+        if (stored.expiresAt > establishedAt) {
+          prune(establishedAt);
+          sessions.set(key, stored);
+        }
+        return stored;
+      });
+      return { ...session, source: coalesced ? 'coalesced' : 'miss' };
+    },
+    size: () => sessions.size,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts

@@ -1,7 +1,10 @@
 import { createSecretRedactionContext } from './secret-redaction';
-import { RUNTIME_RECEIPT_OUTPUT_MAX_BYTES } from './output-size-limits';
+import {
+  assertRuntimeReceiptOutputWithinLimit,
+  TERMINAL_RUN_RESULT_MAX_BYTES,
+} from './output-size-limits';
 
-export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
+export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = TERMINAL_RUN_RESULT_MAX_BYTES;
 export const MAX_LEDGER_LOG_LINES_PER_EVENT = 500;
 export const MAX_LEDGER_LOG_LINE_LENGTH = 2_000;
 
@@ -21,6 +24,15 @@ export function sanitizeLedgerPayload(value: unknown): unknown {
   return ledgerIngressRedactor.redact(value);
 }
 
+/** Secret-redact a receipt payload without coupling its 10 MiB cell limit to the ledger. */
+export function sanitizeRuntimeReceiptPayload(value: unknown): unknown {
+  assertRuntimeReceiptOutputWithinLimit({
+    output: value,
+    path: 'runtime receipt output',
+  });
+  return ledgerIngressRedactor.redact(value);
+}
+
 export function sanitizeLedgerLogLines(value: unknown): string[] {
   if (!Array.isArray(value)) return [];
   return value

package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts

@@ -3,8 +3,10 @@ const encoder = typeof TextEncoder === 'undefined' ? null : new TextEncoder();
 export const TERMINAL_RUN_RESULT_MAX_BYTES = 2_500_000;
 export const CUSTOMER_OUTPUT_VALUE_MAX_BYTES = 1 * 1024 * 1024;
 export const CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 2 * 1024 * 1024;
-export const RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 2 * 1024 * 1024;
-export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 16 * 1024 * 1024;
+/** Inline Postgres receipt contract: one serialized output is at most 10 MiB. */
+export const RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
+/** One completion request may contain multiple receipts up to 32 MiB total. */
+export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
 
 export class OutputTooLargeError extends Error {
   readonly code = 'OUTPUT_TOO_LARGE';

package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts

@@ -0,0 +1 @@
+export type PlayCallExecution = 'inline' | 'child-workflow';

package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts

@@ -7,6 +7,7 @@ export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_SIZE = 200;
 export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_FLUSH_MS = 10;
 export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BUFFERED_BYTES =
   RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES;
+export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_BYTES = 1_000_000;
 export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_INPUT_BYTES =
   RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
 
@@ -22,6 +23,8 @@ export interface RuntimeReceiptCompletionSinkOptions<TInput, TOutput> {
   maxBatchSize?: number;
   maxFlushMs?: number;
   maxBufferedBytes?: number;
+  /** Flush trigger and maximum ordinary transport batch size. */
+  maxBatchBytes?: number;
   maxInputBytes?: number;
   estimateBytes?: (input: TInput) => number;
   completeMany: (inputs: TInput[]) => Promise<TOutput[]>;
@@ -83,12 +86,14 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
   readonly maxBatchSize: number;
   readonly maxFlushMs: number;
   readonly maxBufferedBytes: number;
+  readonly maxBatchBytes: number;
   readonly maxInputBytes: number;
 
   #pending: Array<PendingReceiptCompletion<TInput, TOutput>> = [];
   #pendingHead = 0;
   /** Pending plus in-flight bytes. Released only after transport settles. */
   #residentBytes = 0;
+  #pendingBytes = 0;
   #capacityWaiters: CapacityWaiter[] = [];
   #flushTimer: ReceiptCompletionSinkTimer | null = null;
   #flushPromise: Promise<void> | null = null;
@@ -116,6 +121,10 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
       options.maxBufferedBytes,
       DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BUFFERED_BYTES,
     );
+    this.maxBatchBytes = normalizePositiveInteger(
+      options.maxBatchBytes,
+      DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_BYTES,
+    );
     this.maxInputBytes = normalizePositiveInteger(
       options.maxInputBytes,
       DEFAULT_RECEIPT_COMPLETION_SINK_MAX_INPUT_BYTES,
@@ -194,12 +203,16 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
     }
     this.#throwIfUnavailable();
     this.#residentBytes += bytes;
+    this.#pendingBytes += bytes;
     const completion = new Promise<TOutput>((resolve, reject) => {
       this.#pending.push({ input, bytes, resolve, reject });
       if (this.#pendingCount() >= this.maxBatchSize) {
         void this.flush().catch(() => undefined);
       }
     });
+    if (this.#pendingBytes >= this.maxBatchBytes) {
+      void this.flush().catch(() => undefined);
+    }
     this.#scheduleFlush();
     return await completion;
   }
@@ -237,12 +250,13 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
     let batchBytes = 0;
     while (this.#pendingCount() > 0 && batch.length < this.maxBatchSize) {
       const next = this.#pending[this.#pendingHead]!;
-      if (batch.length > 0 && batchBytes + next.bytes > this.maxBufferedBytes) {
+      if (batch.length > 0 && batchBytes + next.bytes > this.maxBatchBytes) {
         break;
       }
       this.#pendingHead += 1;
       batch.push(next);
       batchBytes += next.bytes;
+      this.#pendingBytes = Math.max(0, this.#pendingBytes - next.bytes);
     }
     if (
       this.#pendingHead > 1_024 &&
@@ -321,6 +335,7 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
     while (this.#pendingCount() > 0) {
       const pending = this.#pending[this.#pendingHead++]!;
       this.#residentBytes = Math.max(0, this.#residentBytes - pending.bytes);
+      this.#pendingBytes = Math.max(0, this.#pendingBytes - pending.bytes);
       pending.reject(this.#failure);
     }
     this.#wakeCapacityWaiters();

package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts

@@ -1,6 +1,26 @@
 import { RECEIPT_STATUS_CODE } from './receipt-status';
 import { workReceiptFailureKindCode } from './work-receipts';
 
+export function workReceiptRepairableFailurePredicateSql(
+  receiptTable: string,
+): string {
+  return `(
+    ${receiptTable}.failure_kind = ${workReceiptFailureKindCode('repairable')}::smallint
+    OR (
+      ${receiptTable}.failure_kind = ${workReceiptFailureKindCode('terminal')}::smallint
+      AND ${receiptTable}.error IS NOT NULL
+      AND lower(${receiptTable}.error) NOT LIKE '%billing cap%'
+      AND lower(${receiptTable}.error) NOT LIKE '%insufficient credits%'
+      AND lower(${receiptTable}.error) NOT LIKE '%monthly billing limit%'
+      AND (
+        ${receiptTable}.error ~* '(^|[^0-9])5[0-9]{2}([^0-9]|$)'
+        OR lower(${receiptTable}.error) LIKE '%transport failed calling%'
+        OR lower(${receiptTable}.error) LIKE '%runtime api call timed out%'
+      )
+    )
+  )`;
+}
+
 export function workReceiptClaimableStatusCodes(input: {
   forceRefresh?: boolean;
   forceFailedRefresh?: boolean;
@@ -47,7 +67,7 @@ export function workReceiptClaimConflictPredicateSql(input: {
               )
               OR ${table}.status <> ${RECEIPT_STATUS_CODE.failed}::smallint
               OR (
-                ${table}.failure_kind = ${workReceiptFailureKindCode('repairable')}::smallint
+                ${workReceiptRepairableFailurePredicateSql(table)}
                 AND (
                   ${table}.run_id IS DISTINCT FROM ${input.claimantRunIdSql}
                   OR COALESCE(${table}.lease_owner_attempt, 0) < ${claimantRunAttemptSql}::integer

package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts

@@ -1,8 +1,14 @@
 import { RuntimeReceiptCompletionSink } from './receipt-completion-sink';
 
+/** Claims/state transitions are small; flush once a 512 KB SQL payload forms. */
+export const DEFAULT_RECEIPT_STATE_SINK_MAX_BATCH_BYTES = 512_000;
+
 export interface RuntimeReceiptStateSinkOptions<TInput, TOutput> {
   maxBatchSize?: number;
   maxFlushMs?: number;
+  maxBatchBytes?: number;
+  maxBufferedBytes?: number;
+  maxInputBytes?: number;
   estimateBytes?: (input: TInput) => number;
   applyMany: (inputs: TInput[]) => Promise<TOutput[]>;
   onFlushStart?: (event: { count: number; bytes: number }) => void;
@@ -33,6 +39,10 @@ export class RuntimeReceiptStateSink<TInput, TOutput> {
     this.#sink = new RuntimeReceiptCompletionSink<TInput, TOutput>({
       maxBatchSize: options.maxBatchSize,
       maxFlushMs: options.maxFlushMs,
+      maxBatchBytes:
+        options.maxBatchBytes ?? DEFAULT_RECEIPT_STATE_SINK_MAX_BATCH_BYTES,
+      maxBufferedBytes: options.maxBufferedBytes,
+      maxInputBytes: options.maxInputBytes,
       estimateBytes: options.estimateBytes,
       completeMany: options.applyMany,
       onFlushStart: options.onFlushStart,