deepline 0.3.200 → 0.3.202

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -84,6 +84,7 @@ import {
   type PlayRunLedgerIngressEvent,
 } from '../run-ledger';
 import { stableStringify } from '../../plays/row-identity';
+import { runtimeWorkCountersFromResources } from '../runtime-work-counters';
 import {
   isCtxFetchHttpFailureDiagnostic,
   parseCtxFetchHttpFailureDiagnostic,
@@ -8774,13 +8775,29 @@ export async function resolvePostgresSchedulerAmbiguousModalCreateIntentFromProv
 ): Promise<boolean> {
   const resolved = await client.query<{ run_id: string }>(
     `
-      UPDATE ${tableName('ambiguous_modal_creates', options)}
-      SET status = 'resolved', resolved_at = clock_timestamp()
-      WHERE run_id = $1
-        AND attempt = $2
-        AND modal_app_id = $3
-        AND status = 'pending'
-      RETURNING run_id
+      WITH resolved AS (
+        UPDATE ${tableName('ambiguous_modal_creates', options)}
+        SET status = 'resolved', resolved_at = clock_timestamp()
+        WHERE run_id = $1
+          AND attempt = $2
+          AND modal_app_id = $3
+          AND status = 'pending'
+        RETURNING run_id, sandbox_capacity_lease_id
+      ), abandoned AS (
+        UPDATE ${tableName('runtime_sandbox_operations', options)} AS operation
+        SET state = 'abandoned', updated_at = clock_timestamp()
+        FROM resolved
+        WHERE operation.operation_key = resolved.sandbox_capacity_lease_id
+          AND operation.state = 'reserved'
+        RETURNING operation.operation_key
+      ), released AS (
+        UPDATE ${tableName('runtime_sandbox_capacity_leases', options)} AS lease
+        SET released_at = clock_timestamp()
+        FROM abandoned
+        WHERE lease.lease_id = abandoned.operation_key
+          AND lease.released_at IS NULL
+      )
+      SELECT run_id FROM resolved
     `,
     [input.runId, input.attempt, input.modalAppId],
   );
@@ -10587,6 +10604,131 @@ export async function enqueuePostgresSchedulerSandboxCleanupForAttempt(
   return enqueued.rows.length;
 }
 
+/** Queue every known sandbox before a requested cancellation can terminalize. */
+export async function enqueuePostgresSchedulerSandboxCleanupForRun(
+  client: PostgresSchedulerQueryClient,
+  input: { runId: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<void> {
+  await client.query(
+    `
+      INSERT INTO ${tableName('sandbox_cleanup_jobs_v2', options)} (
+        provider, sandbox_id, run_id, attempt, environment, routing_domain
+      )
+      SELECT DISTINCT
+        CASE resource->>'kind'
+          WHEN 'modal_sandbox' THEN 'modal'
+          ELSE 'daytona'
+        END,
+        resource->>'sandboxId',
+        attempt.run_id,
+        attempt.attempt,
+        COALESCE(
+          NULLIF(resource->>'runtimeEnvironment', ''),
+          NULLIF(resource->>'daytonaEnvironment', ''),
+          'production'
+        ),
+        CASE resource->>'kind'
+          WHEN 'modal_sandbox' THEN NULLIF(resource->>'modalAppId', '')
+          ELSE NULLIF(resource->>'daytonaOrganizationId', '')
+        END
+      FROM ${tableName('attempts', options)} AS attempt
+      CROSS JOIN LATERAL jsonb_array_elements(
+        CASE
+          WHEN jsonb_typeof(attempt.resource_json) = 'array'
+            THEN attempt.resource_json
+          ELSE jsonb_build_array(attempt.resource_json)
+        END
+      ) AS resource
+      WHERE attempt.run_id = $1
+        AND attempt.resource_json IS NOT NULL
+        AND resource->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
+        AND NULLIF(resource->>'sandboxId', '') IS NOT NULL
+      UNION ALL
+      SELECT operation.provider, operation.sandbox_id, operation.run_id,
+             operation.attempt, operation.environment, operation.routing_domain
+      FROM ${tableName('runtime_sandbox_operations', options)} AS operation
+      WHERE operation.run_id = $1
+        AND operation.state IN ('live', 'deleting')
+        AND operation.sandbox_id IS NOT NULL
+      ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
+    `,
+    [input.runId],
+  );
+}
+
+/** Modal cannot create without a durable intent; an old reservation with no
+ * pending selector or recorded resource is safe to release after recovery time. */
+export async function abandonUnstartedModalCancellationReservations(
+  client: PostgresSchedulerQueryClient,
+  input: { runId: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<void> {
+  await client.query(
+    `WITH abandoned AS (
+       UPDATE ${tableName('runtime_sandbox_operations', options)} AS operation
+       SET state = 'abandoned', updated_at = clock_timestamp()
+       WHERE operation.run_id = $1
+         AND operation.provider = 'modal'
+         AND operation.state = 'reserved'
+         AND operation.created_at <= clock_timestamp()
+           - ($2::double precision * interval '1 millisecond')
+         AND NOT EXISTS (
+           SELECT 1 FROM ${tableName('ambiguous_modal_creates', options)} AS intent
+           WHERE intent.sandbox_capacity_lease_id = operation.operation_key
+             AND intent.status = 'pending'
+         )
+         AND NOT EXISTS (
+           SELECT 1 FROM ${tableName('attempts', options)} AS attempt
+           CROSS JOIN LATERAL jsonb_array_elements(
+             CASE
+               WHEN jsonb_typeof(attempt.resource_json) = 'array'
+                 THEN attempt.resource_json
+               WHEN attempt.resource_json IS NULL THEN '[]'::jsonb
+               ELSE jsonb_build_array(attempt.resource_json)
+             END
+           ) AS resource
+           WHERE attempt.run_id = operation.run_id
+             AND resource->>'sandboxCapacityLeaseId' = operation.operation_key
+         )
+       RETURNING operation.operation_key
+     )
+     UPDATE ${tableName('runtime_sandbox_capacity_leases', options)} AS lease
+     SET released_at = clock_timestamp()
+     FROM abandoned
+     WHERE lease.lease_id = abandoned.operation_key
+       AND lease.released_at IS NULL`,
+    [input.runId, RUNTIME_RELIABILITY_POLICY.recovery.horizonMs],
+  );
+}
+
+/** A reserved create or unconfirmed deletion keeps cancellation pending. */
+export async function hasPendingPostgresSchedulerSandboxCleanupForRun(
+  client: PostgresSchedulerQueryClient,
+  input: { runId: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<boolean> {
+  const pending = await client.query<{ pending: boolean }>(
+    `
+      SELECT EXISTS (
+        SELECT 1
+        FROM ${tableName('sandbox_cleanup_jobs_v2', options)}
+        WHERE run_id = $1 AND status <> 'deleted'
+      ) OR EXISTS (
+        SELECT 1
+        FROM ${tableName('runtime_sandbox_operations', options)}
+        WHERE run_id = $1 AND state = 'reserved'
+      ) OR EXISTS (
+        SELECT 1
+        FROM ${tableName('ambiguous_modal_creates', options)}
+        WHERE run_id = $1 AND status = 'pending'
+      ) AS pending
+    `,
+    [input.runId],
+  );
+  return pending.rows[0]?.pending ?? true;
+}
+
 export async function claimPostgresSchedulerSandboxCleanupJobs(
   client: PostgresSchedulerQueryClient,
   input: {
@@ -12278,7 +12420,26 @@ export async function cancelPostgresSchedulerRun(
                     WHEN elem->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
                      AND (elem->'billingEndedAt' IS NULL OR elem->'billingEndedAt' = 'null'::jsonb)
                     THEN elem || jsonb_build_object(
-                      'billingEndedAt', extract(epoch FROM clock_timestamp()) * 1000,
+                      'billingEndedAt', COALESCE(
+                        (
+                          SELECT extract(epoch FROM cleanup.deleted_at) * 1000
+                          FROM ${tableName('sandbox_cleanup_jobs_v2', options)} AS cleanup
+                          WHERE cleanup.provider = CASE elem->>'kind'
+                                  WHEN 'modal_sandbox' THEN 'modal'
+                                  ELSE 'daytona'
+                                END
+                            AND cleanup.sandbox_id = elem->>'sandboxId'
+                            AND cleanup.routing_domain_key = COALESCE(
+                                  CASE elem->>'kind'
+                                    WHEN 'modal_sandbox' THEN NULLIF(elem->>'modalAppId', '')
+                                    ELSE NULLIF(elem->>'daytonaOrganizationId', '')
+                                  END,
+                                  ''
+                                )
+                            AND cleanup.status = 'deleted'
+                        ),
+                        extract(epoch FROM clock_timestamp()) * 1000
+                      ),
                       'terminalAt', extract(epoch FROM clock_timestamp()) * 1000,
                       'terminalReason', 'cancelled'
                     )
@@ -12885,7 +13046,7 @@ function terminalLogLinesForOutbox(
     .slice(-LOG_TAIL_LIMIT);
 }
 
-function attachRunnerWorkProgressMetadata(
+export function attachRunnerWorkProgressMetadata(
   output: unknown,
   runnerResult: unknown,
   options: {
@@ -12937,6 +13098,13 @@ function attachRunnerWorkProgressMetadata(
   const runtimeResources = metadata?.runtimeResources
     ? null
     : boundedRuntimeResourcesMetadata(runnerResult);
+  // Keep the two scalar counts when the detailed runtimeResources record
+  // exceeds its metadata budget. Compact status must not scan a call log to
+  // reconstruct physical work.
+  const runtimeCounters = runtimeWorkCountersFromResources(
+    (runnerResult as { stats?: { runtimeResources?: unknown } }).stats
+      ?.runtimeResources,
+  );
   const outputWarnings = Array.isArray(
     (runnerResult as { outputWarnings?: unknown }).outputWarnings,
   )
@@ -12979,6 +13147,8 @@ function attachRunnerWorkProgressMetadata(
     (total === null || skipped === null || metadata?.workProgress) &&
     !runLogTail &&
     !runtimeResources &&
+    !runtimeCounters &&
+    !metadata?.runtimeCounters &&
     !rowOutcomes &&
     mergedOutputWarnings.length === 0
   ) {
@@ -13010,6 +13180,9 @@ function attachRunnerWorkProgressMetadata(
     ...record,
     _metadata: {
       ...(metadata ?? {}),
+      // A previous attempt's scalar evidence cannot describe this terminal
+      // attempt. Undefined removes it from persisted JSON when unavailable.
+      runtimeCounters: runtimeCounters ?? undefined,
       ...(workProgress ? { workProgress } : {}),
       ...(rowOutcomes ? { rowOutcomes } : {}),
       ...(runLogTail ? { runLogTail } : {}),
@@ -15507,8 +15680,63 @@ export async function readPostgresSchedulerResult(
 export async function readPostgresSchedulerRunStatus(
   client: PostgresSchedulerQueryClient,
   runId: string,
-  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+  options?: Pick<PostgresSchedulerOptions, 'schema'> & {
+    includeDetails?: boolean;
+  },
 ): Promise<PostgresSchedulerRunStatusSnapshot | null> {
+  if (options?.includeDetails === false) {
+    // An admission projection gap needs lifecycle truth only. Launch, result,
+    // suspension and error JSON may contain row/call-sized detail: never even
+    // select them for status polling. Missing progress remains unconfirmed.
+    const result = await client.query<
+      Pick<
+        RunStatusRow,
+        | 'run_id'
+        | 'org_id'
+        | 'status'
+        | 'play_name'
+        | 'graph_hash'
+        | 'runtime_profile'
+        | 'created_at'
+        | 'updated_at'
+        | 'started_at'
+        | 'finished_at'
+      >
+    >(
+      `
+      SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
+        created_at, updated_at, started_at, finished_at
+      FROM ${tableName('runs', options)}
+      WHERE run_id = $1
+    `,
+      [runId],
+    );
+    const row = result.rows[0];
+    if (!row) return null;
+    return {
+      runId: row.run_id,
+      orgId: row.org_id ?? '',
+      status: row.status,
+      playName: row.play_name,
+      graphHash: row.graph_hash,
+      runtimeBackend: null,
+      schedulerBackend: 'absurd',
+      executionProfile: row.runtime_profile,
+      createdAt: row.created_at?.getTime() ?? null,
+      updatedAt: row.updated_at?.getTime() ?? null,
+      startedAt: row.started_at?.getTime() ?? null,
+      finishedAt: row.finished_at?.getTime() ?? null,
+      durationMs:
+        row.finished_at && row.started_at
+          ? Math.max(0, row.finished_at.getTime() - row.started_at.getTime())
+          : null,
+      runtimeTiming: null,
+      result: null,
+      error: null,
+      suspension: null,
+      staticPipeline: null,
+    };
+  }
   const result = await client.query<RunStatusRow>(
     `
       SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
@@ -15723,6 +15951,10 @@ export async function assertPostgresRunLedgerDelivery(
     runId: string;
     idempotencyKey: string;
     leaseOwner?: string;
+    projectedEvents?: readonly AcceptedPlayRunLedgerEvent[];
+    normalizeProjectedEvents?: (
+      events: readonly PlayRunLedgerIngressEvent[],
+    ) => readonly AcceptedPlayRunLedgerEvent[];
   },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<void> {
@@ -15733,6 +15965,8 @@ export async function assertPostgresRunLedgerDelivery(
     lease_live: boolean;
     due: boolean;
     blocked: boolean;
+    payload_json: unknown;
+    created_at: Date;
   }>(
     `
     SELECT outbox.delivery_key,
@@ -15740,7 +15974,9 @@ export async function assertPostgresRunLedgerDelivery(
            outbox.lease_owner,
            outbox.lease_expires_at > now() AS lease_live,
            coalesce(outbox.next_attempt_at, outbox.created_at) <= now() AS due,
-           outbox.failed_at IS NOT NULL AS blocked
+           outbox.failed_at IS NOT NULL AS blocked,
+           outbox.payload_json,
+           outbox.created_at
     FROM ${tableName('outbox', options)} AS outbox
     JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
     WHERE outbox.run_id = $1 AND runs.org_id = $2
@@ -15755,12 +15991,9 @@ export async function assertPostgresRunLedgerDelivery(
     throw new RunLedgerProjectionLeaseLostError();
   }
 
-  // A normal projector leases only the batch it is about to send. Cancellation
-  // intentionally transfers the entire pending causal prefix to one stable
-  // owner before synchronously draining it, while run.events are still claimed
-  // one row at a time. Validate the submitted key against a *contiguous due
-  // prefix*, not every row carrying that owner; otherwise a valid cancellation
-  // head is rejected whenever later facts share the cancellation lease.
+  // Cancellation may own several pending facts at once. Validate the exact
+  // contiguous, due prefix named by the receipt instead of all rows sharing
+  // that cancellation lease.
   const prefix: typeof rows = [];
   for (const row of rows) {
     if (
@@ -15780,6 +16013,9 @@ export async function assertPostgresRunLedgerDelivery(
         candidate.event_type,
       ),
     );
+    const prefixDigest = createHash('sha256')
+      .update(prefix.map((candidate) => candidate.delivery_key).join('\n'))
+      .digest('hex');
     let expectedKey: string;
     if (terminal) {
       expectedKey = buildRunLedgerTerminalIdempotencyKey({
@@ -15795,29 +16031,58 @@ export async function assertPostgresRunLedgerDelivery(
         deliveryKey: first.delivery_key,
       });
     } else {
-      const digest = createHash('sha256')
-        .update(prefix.map((candidate) => candidate.delivery_key).join('\n'))
-        .digest('hex');
       expectedKey = buildRunLedgerEventsIdempotencyKey({
         runId: input.runId,
-        deliveryDigest: digest,
+        deliveryDigest: prefixDigest,
       });
     }
-    if (input.idempotencyKey === expectedKey) return;
 
-    // The pre-batch projector used run-terminal:<delivery-key> for a single
-    // nonterminal fact. Retain that wire-compatibility key for the current
-    // singleton head, but never let it authorize a later row in the prefix.
-    if (
+    // Older batch dispatchers may use the digest receipt even when the batch
+    // contains a terminal fact. Keep accepting it for this exact prefix.
+    const batchKey = buildRunLedgerEventsIdempotencyKey({
+      runId: input.runId,
+      deliveryDigest: prefixDigest,
+    });
+
+    // Preserve the old singleton key for non-start, nonterminal facts.
+    const legacySingletonKey =
       prefix.length === 1 &&
       !['run.created', 'run.started'].includes(first.event_type) &&
       input.idempotencyKey ===
         buildRunLedgerTerminalIdempotencyKey({
           runId: input.runId,
           deliveryKey: first.delivery_key,
-        })
+        });
+    if (
+      input.idempotencyKey === expectedKey ||
+      input.idempotencyKey === batchKey ||
+      legacySingletonKey
     ) {
-      return;
+      if (input.projectedEvents === undefined) return;
+      const prefixEvents: PlayRunLedgerIngressEvent[] = [];
+      for (const prefixRow of prefix) {
+        prefixEvents.push(
+          ...buildRunLedgerEventsFromPostgresSchedulerOutbox({
+            eventId: prefixRow.delivery_key,
+            runId: input.runId,
+            eventType: prefixRow.event_type,
+            payload: prefixRow.payload_json,
+            createdAt: new Date(prefixRow.created_at),
+            leaseOwner: prefixRow.lease_owner ?? undefined,
+            deliveryKey: prefixRow.delivery_key,
+          }),
+        );
+      }
+      const comparablePrefixEvents = input.normalizeProjectedEvents
+        ? input.normalizeProjectedEvents(prefixEvents)
+        : prefixEvents;
+      if (
+        stableStringify(input.projectedEvents) ===
+        stableStringify(comparablePrefixEvents)
+      ) {
+        return;
+      }
+      throw new RunLedgerProjectionLeaseLostError();
     }
 
     if (
@@ -15997,7 +16262,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
                 )
               )
           )
-        ORDER BY outbox.created_at, outbox.event_id
+        ORDER BY ${runLedgerOrderSql('outbox')}
         FOR UPDATE OF outbox
         LIMIT $1
       ), terminal_batch AS (
@@ -16033,7 +16298,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
       SELECT event_id, run_id, event_type, payload_json, created_at, lease_owner,
              delivery_key
       FROM claimed
-      ORDER BY created_at, event_id
+      ORDER BY ${runLedgerOrderSql('claimed')}
     `,
       [limit, runId, leaseOwner, leaseSeconds],
     );
@@ -16127,7 +16392,9 @@ export async function readPostgresSchedulerOutboxNextDueAt(
 export async function readPostgresSchedulerOutboxDeliveryStatus(
   client: PostgresSchedulerQueryClient,
   input: { runId: string; eventId: string },
-  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+  options?: Pick<PostgresSchedulerOptions, 'schema'> & {
+    includeBlocked?: boolean;
+  },
 ): Promise<PostgresSchedulerOutboxDeliveryStatus> {
   const runId = input.runId.trim();
   const eventId = input.eventId.trim();
@@ -16138,13 +16405,17 @@ export async function readPostgresSchedulerOutboxDeliveryStatus(
   }
   const result = await client.query<{ delivered: boolean; blocked: boolean }>(
     `SELECT target.delivered_at IS NOT NULL AS delivered,
-            target.failed_at IS NOT NULL OR EXISTS (
+            ${
+              options?.includeBlocked === false
+                ? 'false'
+                : `target.failed_at IS NOT NULL OR EXISTS (
               SELECT 1
               FROM ${tableName('outbox', options)} predecessor
               WHERE predecessor.run_id = target.run_id
                 AND predecessor.failed_at IS NOT NULL
                 AND ${runLedgerPrecedesSql('predecessor', 'target', true)}
-            ) AS blocked
+            )`
+            } AS blocked
      FROM ${tableName('outbox', options)} target
      WHERE target.run_id = $1 AND target.event_id = $2`,
     [runId, eventId],

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts

@@ -66,6 +66,7 @@ import {
   createRuntimeReceiptValuePersister,
   createRuntimeReceiptValueReader,
   RUNTIME_RECEIPT_VALUE_INLINE_MAX_BYTES,
+  RuntimeValueStoreUnavailableError,
   type RuntimeReceiptValueBlobStore,
   type RuntimeReceiptValuePersister,
   type RuntimeReceiptValueReader,
@@ -250,6 +251,26 @@ export function receiptGatewayFailureResponse(
       },
     );
   }
+  if (error instanceof RuntimeValueStoreUnavailableError) {
+    return receiptGatewayJsonResponse(
+      {
+        error: `${operationLabel} object store is temporarily unavailable.`,
+        code: 'runtime_value_store_unavailable',
+        retryable: true,
+        retry_after_ms: 1_000,
+        detail: error.message,
+        ...(context.action ? { action: context.action } : {}),
+      },
+      {
+        status: 503,
+        headers: {
+          'retry-after': '1',
+          'x-deepline-error-code': 'runtime_value_store_unavailable',
+          'x-deepline-error-detail': error.message,
+        },
+      },
+    );
+  }
   if (isRuntimeSchedulerCapacityError(error)) {
     const retryAfterMs = runtimeSchedulerCapacityRetryAfterMs(error);
     return receiptGatewayJsonResponse(

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts

@@ -25,6 +25,40 @@ export type RuntimeValueBlobStore = {
 /** Backward-compatible receipt-specific name for the shared object store. */
 export type RuntimeReceiptValueBlobStore = RuntimeValueBlobStore;
 
+export class RuntimeValueStoreUnavailableError extends Error {
+  constructor(
+    readonly operation: 'put' | 'head',
+    readonly status: number,
+    cause: unknown,
+  ) {
+    super(`Runtime value object store ${operation} returned HTTP ${status}.`, {
+      cause,
+    });
+    this.name = 'RuntimeValueStoreUnavailableError';
+  }
+}
+
+function transientStoreStatus(error: unknown): number | null {
+  if (!error || typeof error !== 'object') return null;
+  const metadata = (error as { $metadata?: { httpStatusCode?: unknown } })
+    .$metadata;
+  const status = metadata?.httpStatusCode;
+  return typeof status === 'number' &&
+    (status === 408 || status === 429 || status >= 500)
+    ? status
+    : null;
+}
+
+function transientStoreError(
+  operation: 'put' | 'head',
+  error: unknown,
+): RuntimeValueStoreUnavailableError | null {
+  const status = transientStoreStatus(error);
+  return status === null
+    ? null
+    : new RuntimeValueStoreUnavailableError(operation, status, error);
+}
+
 /**
  * Foundation owns value references and bounded I/O admission; the deployment
  * injects its object-store adapter. This keeps R2 credentials and transport
@@ -185,7 +219,13 @@ export async function persistRuntimeReceiptValue(
     // An immutable deterministic object that is observable after a failed PUT
     // is the same attempted publication. Treat it as a recovered write, never
     // as permission to call the provider again.
-    if (!(await store.exists(outputRef.key))) throw error;
+    let exists: boolean;
+    try {
+      exists = await store.exists(outputRef.key);
+    } catch (probeError) {
+      throw transientStoreError('head', probeError) ?? probeError;
+    }
+    if (!exists) throw transientStoreError('put', error) ?? error;
   }
   return { kind: 'reference', outputJson: null, outputRef };
 }
@@ -209,11 +249,10 @@ export function createRuntimeReceiptValuePersister(
     serialized: string;
     inlineFields?: RuntimeReceiptValueInlineFields;
   };
-  const writes = new BoundedBatchFunction<
-    Write,
-    PersistedRuntimeReceiptValue,
-    'r2'
-  >({
+  type WriteOutcome =
+    | { ok: true; value: PersistedRuntimeReceiptValue }
+    | { ok: false; error: unknown };
+  const writes = new BoundedBatchFunction<Write, WriteOutcome, 'r2'>({
     keyId: () => 'r2',
     estimateBytes: (value) => Buffer.byteLength(value.serialized, 'utf8'),
     maxBatchItems: 4,
@@ -224,16 +263,24 @@ export function createRuntimeReceiptValuePersister(
     maxRetainedUnits: 32,
     maxInFlightBatches: 2,
     flushMs: 0,
+    // R2 has no multi-PUT. One failed object must not reject the other
+    // independent receipts admitted alongside it.
     execute: async ({ values }) =>
       await Promise.all(
-        values.map(
-          async (value) =>
-            await persistRuntimeReceiptValue({
-              ...value,
-              inlineMaxBytes,
-              ...(input.store ? { store: input.store } : {}),
-            }),
-        ),
+        values.map(async (value): Promise<WriteOutcome> => {
+          try {
+            return {
+              ok: true,
+              value: await persistRuntimeReceiptValue({
+                ...value,
+                inlineMaxBytes,
+                ...(input.store ? { store: input.store } : {}),
+              }),
+            };
+          } catch (error) {
+            return { ok: false, error };
+          }
+        }),
       ),
   });
   return async (value) => {
@@ -244,7 +291,9 @@ export function createRuntimeReceiptValuePersister(
         ...(input.store ? { store: input.store } : {}),
       });
     }
-    return await writes.submit({ key: 'r2', value });
+    const outcome = await writes.submit({ key: 'r2', value });
+    if (!outcome.ok) throw outcome.error;
+    return outcome.value;
   };
 }