deepline 0.1.182 → 0.1.183

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -0,0 +1,245 @@
+import {
+  acquireRateStateViaAppRuntime,
+  penalizeRateStateViaAppRuntime,
+  releaseRateStateViaAppRuntime,
+  type WorkerRuntimeApiContext,
+} from '../app-runtime-api';
+import {
+  noopPacingPermit,
+  type PacingPermit,
+  type PacingRule,
+  type RateStateBackend,
+} from './rate-state-backend';
+
+/**
+ * Permits leased per app-runtime round trip for the Hatchet/Node substrate.
+ *
+ * Eight keeps DB/API round trips amortized for batch-heavy maps while limiting
+ * stale-block over-issue to 8 x live runner processes per bucket/window.
+ */
+export const APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE = 8;
+const LEASE_BLOCK_TTL_MS = 250;
+const MAX_ACQUIRE_SLEEP_MS = 5_000;
+const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
+
+interface LeasedBlock {
+  remaining: number;
+  expiresAt: number;
+  rulesKey: string;
+  rules: PacingRule[];
+}
+
+interface Options {
+  schedulerSchema?: string | null;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+  onStoreFailure?: (info: { bucketId: string; error: string }) => void;
+}
+
+export class AppRuntimeRateStateBackend implements RateStateBackend {
+  private readonly context: WorkerRuntimeApiContext;
+  private readonly schedulerSchema: string | null;
+  private readonly now: () => number;
+  private readonly sleep: (ms: number) => Promise<void>;
+  private readonly onStoreFailure: (info: {
+    bucketId: string;
+    error: string;
+  }) => void;
+  private readonly blocks = new Map<string, LeasedBlock>();
+  private pendingReleaseFailure: Error | null = null;
+
+  constructor(context: WorkerRuntimeApiContext, options: Options = {}) {
+    this.context = context;
+    this.schedulerSchema = options.schedulerSchema?.trim() || null;
+    this.now = options.now ?? (() => Date.now());
+    this.sleep =
+      options.sleep ??
+      ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
+    this.onStoreFailure =
+      options.onStoreFailure ??
+      ((info) => {
+        console.error('[app-runtime-rate-state] store failure', info);
+      });
+  }
+
+  async acquire(input: {
+    bucketId: string;
+    rules: readonly PacingRule[];
+    signal?: AbortSignal;
+  }): Promise<PacingPermit> {
+    const { bucketId, rules, signal } = input;
+    if (rules.length === 0) return noopPacingPermit();
+    this.throwPendingReleaseFailure();
+
+    const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
+    const rulesKey = rulesSignature(ordered);
+    const localPermit = this.drawFromBlock(bucketId, rulesKey);
+    if (localPermit) return localPermit;
+
+    let consecutiveFailures = 0;
+    while (true) {
+      if (signal?.aborted) {
+        throw signal.reason instanceof Error
+          ? signal.reason
+          : new Error('Rate-state acquire aborted.');
+      }
+      try {
+        const response = await acquireRateStateViaAppRuntime(this.context, {
+          bucketId,
+          rules: ordered,
+          requested: APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE,
+          schedulerSchema: this.schedulerSchema,
+        });
+        consecutiveFailures = 0;
+        if (response.granted > 0) {
+          const remaining = response.granted - 1;
+          if (remaining > 0)
+            this.mergeBlock(bucketId, remaining, rulesKey, ordered);
+          return this.permitFor(bucketId, ordered);
+        }
+        await this.sleep(
+          Math.max(1, Math.min(response.waitMs, MAX_ACQUIRE_SLEEP_MS)),
+        );
+      } catch (error) {
+        // Fail closed. A degraded rate-state store must not grant unlimited
+        // provider calls; after this bounded retry ladder the run fails loudly.
+        consecutiveFailures += 1;
+        const normalized = normalizeError(error);
+        this.onStoreFailure({ bucketId, error: normalized.message });
+        if (consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length) {
+          throw new Error(
+            `Rate-state store unavailable for ${bucketId}; failing closed after ${consecutiveFailures} acquire attempts: ${normalized.message}`,
+          );
+        }
+        await this.sleep(
+          STORE_FAILURE_RETRY_DELAYS_MS[consecutiveFailures - 1] ?? 250,
+        );
+      }
+    }
+  }
+
+  penalize(input: { bucketId: string; cooldownMs: number }): void {
+    if (input.cooldownMs <= 0) return;
+    this.drainBlock(input.bucketId);
+    void penalizeRateStateViaAppRuntime(this.context, {
+      bucketId: input.bucketId,
+      cooldownMs: input.cooldownMs,
+      schedulerSchema: this.schedulerSchema,
+    }).catch((error) => {
+      const normalized = normalizeError(error);
+      this.pendingReleaseFailure = new Error(
+        `Rate-state penalize failed for ${input.bucketId}: ${normalized.message}`,
+      );
+      this.onStoreFailure({
+        bucketId: input.bucketId,
+        error: normalized.message,
+      });
+    });
+  }
+
+  private permitFor(bucketId: string, rules: PacingRule[]): PacingPermit {
+    if (!rules.some((rule) => rule.maxConcurrency != null)) {
+      return noopPacingPermit();
+    }
+    let released = false;
+    return {
+      release: () => {
+        if (released) return;
+        released = true;
+        this.releaseReserved(bucketId, rules, 1);
+      },
+    };
+  }
+
+  private mergeBlock(
+    bucketId: string,
+    remaining: number,
+    rulesKey: string,
+    rules: PacingRule[],
+  ): void {
+    const freshExpiresAt = this.now() + LEASE_BLOCK_TTL_MS;
+    const existing = this.blocks.get(bucketId);
+    if (
+      existing &&
+      existing.rulesKey === rulesKey &&
+      existing.expiresAt > this.now()
+    ) {
+      existing.remaining += remaining;
+      existing.expiresAt = Math.min(existing.expiresAt, freshExpiresAt);
+      return;
+    }
+    if (existing)
+      this.releaseReserved(bucketId, existing.rules, existing.remaining);
+    this.blocks.set(bucketId, {
+      remaining,
+      expiresAt: freshExpiresAt,
+      rulesKey,
+      rules,
+    });
+  }
+
+  private drawFromBlock(
+    bucketId: string,
+    rulesKey: string,
+  ): PacingPermit | null {
+    const block = this.blocks.get(bucketId);
+    if (!block) return null;
+    if (block.rulesKey !== rulesKey || block.expiresAt <= this.now()) {
+      this.blocks.delete(bucketId);
+      this.releaseReserved(bucketId, block.rules, block.remaining);
+      return null;
+    }
+    block.remaining -= 1;
+    if (block.remaining <= 0) this.blocks.delete(bucketId);
+    return this.permitFor(bucketId, block.rules);
+  }
+
+  private drainBlock(bucketId: string): void {
+    const block = this.blocks.get(bucketId);
+    if (!block) return;
+    this.blocks.delete(bucketId);
+    this.releaseReserved(bucketId, block.rules, block.remaining);
+  }
+
+  private releaseReserved(
+    bucketId: string,
+    rules: readonly PacingRule[],
+    count: number,
+  ): void {
+    if (count <= 0 || !rules.some((rule) => rule.maxConcurrency != null))
+      return;
+    void releaseRateStateViaAppRuntime(this.context, {
+      bucketId,
+      rules: [...rules],
+      count,
+      schedulerSchema: this.schedulerSchema,
+    }).catch((error) => {
+      const normalized = normalizeError(error);
+      this.pendingReleaseFailure = new Error(
+        `Rate-state release failed for ${bucketId}: ${normalized.message}`,
+      );
+      this.onStoreFailure({ bucketId, error: normalized.message });
+    });
+  }
+
+  private throwPendingReleaseFailure(): void {
+    if (!this.pendingReleaseFailure) return;
+    const error = this.pendingReleaseFailure;
+    this.pendingReleaseFailure = null;
+    throw error;
+  }
+}
+
+function rulesSignature(rules: readonly PacingRule[]): string {
+  return [...rules]
+    .map(
+      (rule) =>
+        `${rule.ruleId}:${rule.requestsPerWindow}:${rule.windowMs}:${rule.maxConcurrency ?? ''}`,
+    )
+    .sort()
+    .join('|');
+}
+
+function normalizeError(error: unknown): Error {
+  return error instanceof Error ? error : new Error(String(error));
+}

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -8,7 +8,7 @@
  * "may I, and how many at once" policy. See ADR 0007 + CONTEXT.md.
  *
  * Surface (small, by design):
- *   - acquireRowSlot / acquireChildPlaySlot / acquireToolSlot → blocking leases
+ *   - acquireRowSlot / acquireChildSubmitSlot / acquireToolSlot → blocking leases
  *   - chargeBudget                                            → throws on breach
  *   - forkChild                                               → child lineage snapshot
  *   - resolveRowConcurrency / reportProviderBackpressure / snapshot
@@ -80,8 +80,8 @@ export interface PlayExecutionGovernor {
 
   /** Block until a map-row slot is free. */
   acquireRowSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
-  /** Block until a child-play slot is free. */
-  acquireChildPlaySlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
+  /** Block until a child-play submit slot is free. Released after submit, not terminal. */
+  acquireChildSubmitSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
   /**
    * Block until a global tool-concurrency slot AND the per-(org,provider) pacer
    * permit are free, then charge the tool-call budget and return a lease. Order:
@@ -111,7 +111,7 @@ export interface PlayExecutionGovernor {
    *
    * Unlike {@link acquireToolSlot} (which charges last so an aborted acquire
    * never consumes budget), child-lineage counters are charged here at fork
-   * time, BEFORE the caller acquires a child-play slot. A slot acquire that then
+   * time, BEFORE the caller acquires a child-submit slot. A slot acquire that then
    * fails (e.g. abort) does NOT refund these counters. This is intentional and
    * safe given the 100k child caps: the charge reserves lineage-global capacity
    * for a launch the caller has committed to, and forkChild must return the
@@ -264,7 +264,7 @@ export function createPlayExecutionGovernor(
     policy,
 
     acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
-    acquireChildPlaySlot: (opts) => childPlaySlots.acquire(opts?.signal),
+    acquireChildSubmitSlot: (opts) => childPlaySlots.acquire(opts?.signal),
 
     async acquireToolSlot(toolId, opts) {
       // 1. global tool-concurrency slot.

package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts

@@ -72,6 +72,10 @@ export interface ExecutionPacingPolicy {
   readonly defaultProviderRequestsPerSecond: number;
   /** Parallelism the scheduler suggests before a provider's own hints tighten it. */
   readonly suggestedMaxParallelism: number;
+  /** Elapsed isolate work window before the worker dispatcher yields. */
+  readonly workerYieldElapsedMs: number;
+  /** Fallback batch parallelism when a tool declares no provider hints. */
+  readonly workerToolBatchDefaultParallelism: number;
 }
 
 export interface ResolvedExecutionPolicy {
@@ -118,6 +122,16 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
     // Undeclared providers; declared providers (rate-limit-definitions.ts) win.
     defaultProviderRequestsPerSecond: 10,
     suggestedMaxParallelism: 50,
+    // Worker isolate pacing knobs. The budget module owns platform accounting;
+    // the Governor owns the policy values that decide when work is chunked.
+    workerYieldElapsedMs: 45_000,
+    // Fallback wave width for batched tool chunks that carry no provider pacing
+    // hint. This is only a floor for the undeclared case: it stays bounded by
+    // suggestedMaxParallelism (50) above, and declared providers still get
+    // tightened by their per-provider pacing (rate-limit-definitions.ts). Raised
+    // 4 -> 16 so full batches fan out wide enough to close the map-throughput gap
+    // without over-running providers that actually declare limits.
+    workerToolBatchDefaultParallelism: 16,
   },
 };
 

package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts

@@ -0,0 +1,52 @@
+/**
+ * Lease timing policy for runtime receipts and Runtime Sheet attempts.
+ *
+ * Expiry is a liveness signal, not a logical ordering token. B2 moves write
+ * ordering to coordinator run-attempt epochs; these values only decide how
+ * long an owner may hold a lease without renewal.
+ */
+
+const DEFAULT_PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS = 10 * 60_000;
+const DEFAULT_PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS = 10 * 60_000;
+
+function readRuntimeLeaseEnv(name: string): string | undefined {
+  if (typeof process === 'undefined') return undefined;
+  return process.env?.[name];
+}
+
+function resolveRuntimeLeaseTtlMs(
+  envName: string,
+  defaultValueMs: number,
+): number {
+  const raw = readRuntimeLeaseEnv(envName);
+  if (raw === undefined || raw.trim() === '') return defaultValueMs;
+  if (!/^\d+$/.test(raw.trim())) {
+    throw new Error(
+      `${envName} must be a positive integer number of milliseconds.`,
+    );
+  }
+  const value = Number(raw.trim());
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(
+      `${envName} must be a positive integer number of milliseconds.`,
+    );
+  }
+  return value;
+}
+
+export const PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS = resolveRuntimeLeaseTtlMs(
+  'DEEPLINE_WORK_RECEIPT_LEASE_TTL_MS',
+  DEFAULT_PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
+);
+export const PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS = resolveRuntimeLeaseTtlMs(
+  'DEEPLINE_SHEET_ATTEMPT_LEASE_TTL_MS',
+  DEFAULT_PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
+);
+export const PLAY_RUNTIME_LEASE_HEARTBEAT_DIVISOR = 3;
+
+export function runtimeLeaseHeartbeatIntervalMs(ttlMs: number): number {
+  if (!Number.isFinite(ttlMs) || ttlMs <= 0) {
+    throw new Error('Runtime lease heartbeat interval needs a positive TTL.');
+  }
+  return Math.max(1, Math.floor(ttlMs / PLAY_RUNTIME_LEASE_HEARTBEAT_DIVISOR));
+}

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -9,6 +9,42 @@ import type { PlayExecutionSuspension } from './suspension';
 import type { PlayStaticPipeline } from '../plays/static-pipeline';
 import type { PlayLiveEventSource } from './live-events';
 import type { PreloadedRuntimeDbSession } from './db-session';
+import type { PacingRule } from './governor/rate-state-backend';
+import type { GovernanceSnapshot } from './governor/governor';
+
+export type PlayRunnerRateStateBackendConfig =
+  | {
+      kind: 'in_memory';
+    }
+  | {
+      kind: 'app_runtime_postgres';
+      schedulerSchema?: string | null;
+    };
+
+export type PlayRunnerRateStateAcquireInput = {
+  bucketId: string;
+  rules: PacingRule[];
+  requested: number;
+  schedulerSchema?: string | null;
+};
+
+export type PlayRunnerRateStateAcquireResult = {
+  granted: number;
+  waitMs: number;
+};
+
+export type PlayRunnerRateStateReleaseInput = {
+  bucketId: string;
+  rules: PacingRule[];
+  count: number;
+  schedulerSchema?: string | null;
+};
+
+export type PlayRunnerRateStatePenalizeInput = {
+  bucketId: string;
+  cooldownMs: number;
+  schedulerSchema?: string | null;
+};
 
 export interface PlayRunnerContextConfig {
   executorToken?: string;
@@ -19,10 +55,12 @@ export interface PlayRunnerContextConfig {
   workflowId?: string;
   playId?: string;
   runId?: string;
+  runAttempt?: number | null;
   playName?: string;
   userEmail?: string;
   convexUrl?: string;
   staticPipeline?: PlayStaticPipeline | null;
+  force?: boolean | null;
   forceToolRefresh?: boolean | null;
   /**
    * Controls how scoped Postgres sessions enter the runner. `preloaded` means
@@ -38,6 +76,9 @@ export interface PlayRunnerContextConfig {
    * exact-enough match for the requested play/table/operation scope.
    */
   preloadedDbSessions?: PreloadedRuntimeDbSession[];
+  rateStateBackend?: PlayRunnerRateStateBackendConfig;
+  runtimeSchedulerSchema?: string | null;
+  governance?: GovernanceSnapshot | null;
 }
 
 export interface PlayRunnerExecutionConfig {

package/dist/bundling-sources/shared_libs/play-runtime/providers.ts

@@ -44,9 +44,10 @@ export const PLAY_RUNTIME_PROVIDERS: Record<
     id: PLAY_RUNTIME_PROVIDER_IDS.hatchet,
     scheduler: PLAY_SCHEDULER_BACKENDS.hatchet,
     runner: PLAY_RUNTIME_BACKENDS.daytona,
-    dedup: PLAY_DEDUP_BACKENDS.durableObject,
+    dedup: PLAY_DEDUP_BACKENDS.inMemory,
     artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
-    label: 'Hatchet scheduler + one-shot Daytona runner + DO dedup',
+    label:
+      'Hatchet scheduler + one-shot Daytona runner + receipt-backed in-process dedup',
   },
   local: {
     id: PLAY_RUNTIME_PROVIDER_IDS.local,

package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts

@@ -0,0 +1,124 @@
+import { RECEIPT_STATUS_CODE } from './receipt-status';
+import { workReceiptFailureKindCode } from './work-receipts';
+
+export function workReceiptClaimableStatusCodes(input: {
+  forceRefresh?: boolean;
+  forceFailedRefresh?: boolean;
+  reclaimRunning?: boolean;
+}): number[] {
+  if (input.forceRefresh === true) {
+    return [
+      RECEIPT_STATUS_CODE.pending,
+      RECEIPT_STATUS_CODE.failed,
+      ...(input.reclaimRunning === true ? [RECEIPT_STATUS_CODE.running] : []),
+      RECEIPT_STATUS_CODE.completed,
+      RECEIPT_STATUS_CODE.skipped,
+    ];
+  }
+  if (input.reclaimRunning === true) {
+    return [
+      RECEIPT_STATUS_CODE.pending,
+      RECEIPT_STATUS_CODE.running,
+      RECEIPT_STATUS_CODE.failed,
+    ];
+  }
+  return [RECEIPT_STATUS_CODE.pending, RECEIPT_STATUS_CODE.failed];
+}
+
+export function workReceiptClaimConflictPredicateSql(input: {
+  receiptTable: string;
+  claimableStatusesSql: string;
+  claimantRunIdSql: string;
+  claimantRunAttemptSql?: string;
+  forceRefreshSql: string;
+  forceFailedRefreshSql?: string;
+}): string {
+  const table = input.receiptTable;
+  const claimantRunAttemptSql = input.claimantRunAttemptSql ?? '0';
+  const forceFailedRefreshSql = input.forceFailedRefreshSql ?? 'false';
+  return `${table}.status = ANY(${input.claimableStatusesSql}::smallint[])
+            AND (
+              ${input.forceRefreshSql}::boolean
+              OR (
+                ${forceFailedRefreshSql}::boolean
+                AND ${table}.status = ${RECEIPT_STATUS_CODE.failed}::smallint
+              )
+              OR ${table}.status <> ${RECEIPT_STATUS_CODE.failed}::smallint
+              OR (
+                ${table}.failure_kind = ${workReceiptFailureKindCode('repairable')}::smallint
+                AND (
+                  ${table}.run_id IS DISTINCT FROM ${input.claimantRunIdSql}
+                  OR COALESCE(${table}.lease_owner_attempt, 0) < ${claimantRunAttemptSql}::integer
+                )
+              )
+            )
+            AND (
+              ${table}.status = ${RECEIPT_STATUS_CODE.failed}::smallint
+              OR (${table}.lease_id IS NULL AND ${table}.lease_expires_at IS NULL)
+              OR ${table}.lease_expires_at <= now()
+              OR (
+                /* owner: runtime; remove lease-owner fallback compat after ledger cutover M1 */
+                ${table}.lease_id IS NOT NULL
+                AND ${table}.lease_expires_at IS NOT NULL
+                AND
+                COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.claimantRunIdSql}
+                AND COALESCE(${table}.lease_owner_attempt, 0) < ${claimantRunAttemptSql}::integer
+              )
+            )`;
+}
+
+export function workReceiptRunningOwnerPredicateSql(input: {
+  receiptTable: string;
+  ownerRunIdSql: string;
+  ownerRunAttemptSql?: string;
+  leaseIdSql: string;
+}): string {
+  const table = input.receiptTable;
+  const ownerRunAttemptSql = input.ownerRunAttemptSql ?? '0';
+  return `${table}.status = ${RECEIPT_STATUS_CODE.running}::smallint
+            /* owner: runtime; remove lease-owner fallback compat after ledger cutover M1 */
+            AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.ownerRunIdSql}
+            AND COALESCE(${table}.lease_owner_attempt, 0) = ${ownerRunAttemptSql}::integer
+            AND (
+              ${table}.lease_id IS NULL
+              OR (
+                ${input.leaseIdSql} IS NOT NULL
+                AND ${table}.lease_id = ${input.leaseIdSql}
+              )
+            )`;
+}
+
+/**
+ * Fence for a controlled work-receipt lease RELEASE on run-fatal teardown.
+ * Matches running receipts still leased to the releasing run+attempt, regardless
+ * of the specific lease id (teardown does not thread per-receipt lease ids). It
+ * never matches a receipt owned by a different run+attempt, so releasing back to
+ * `pending` can never steal a live competitor's claim.
+ */
+export function workReceiptReleaseOwnerPredicateSql(input: {
+  receiptTable: string;
+  ownerRunIdSql: string;
+  ownerRunAttemptSql?: string;
+}): string {
+  const table = input.receiptTable;
+  const ownerRunAttemptSql = input.ownerRunAttemptSql ?? '0';
+  return `${table}.status = ${RECEIPT_STATUS_CODE.running}::smallint
+            /* owner: runtime; remove lease-owner fallback compat after ledger cutover M1 */
+            AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.ownerRunIdSql}
+            AND COALESCE(${table}.lease_owner_attempt, 0) = ${ownerRunAttemptSql}::integer`;
+}
+
+export function workReceiptHeartbeatPredicateSql(input: {
+  receiptTable: string;
+  ownerRunIdSql: string;
+  ownerRunAttemptSql?: string;
+  leaseIdSql: string;
+}): string {
+  const table = input.receiptTable;
+  const ownerRunAttemptSql = input.ownerRunAttemptSql ?? '0';
+  return `${table}.status = ${RECEIPT_STATUS_CODE.running}::smallint
+            /* owner: runtime; remove lease-owner fallback compat after ledger cutover M1 */
+            AND COALESCE(${table}.lease_owner_run_id, ${table}.run_id) = ${input.ownerRunIdSql}
+            AND COALESCE(${table}.lease_owner_attempt, 0) = ${ownerRunAttemptSql}::integer
+            AND ${table}.lease_id = ${input.leaseIdSql}`;
+}

package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts

@@ -12,7 +12,10 @@ export type ReceiptStatusCode =
   (typeof RECEIPT_STATUS_CODE)[keyof typeof RECEIPT_STATUS_CODE];
 
 export function receiptStatusFromCode(value: unknown): WorkReceiptStatus {
-  switch (Number(value ?? RECEIPT_STATUS_CODE.pending)) {
+  if (value == null) {
+    throw new Error(`Unknown runtime work receipt status code: ${value}`);
+  }
+  switch (Number(value)) {
     case RECEIPT_STATUS_CODE.running:
       return 'running';
     case RECEIPT_STATUS_CODE.completed:
@@ -22,8 +25,9 @@ export function receiptStatusFromCode(value: unknown): WorkReceiptStatus {
     case RECEIPT_STATUS_CODE.skipped:
       return 'skipped';
     case RECEIPT_STATUS_CODE.pending:
-    default:
       return 'pending';
+    default:
+      throw new Error(`Unknown runtime work receipt status code: ${value}`);
   }
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts

@@ -1,4 +1,5 @@
 import { isHardBillingToolHttpError } from './tool-http-errors';
+import { isRuntimePersistenceCircuitOpenError } from './persistence-latch';
 
 /**
  * Thrown by runner Adapters when a Play Run is externally cancelled. Row
@@ -27,13 +28,58 @@ export function isAbortLikeError(error: unknown): boolean {
  * Errors that must stay run-fatal even under default map row failure isolation.
  *
  * Provider/tool HTTP failures, including exhausted 429/5xx retries, are row
- * outcomes after the tool-call Adapter spends its local retry budget. Abort,
- * Governor budget exhaustion, and hard billing failures escape row isolation.
+ * outcomes after the tool-call Adapter spends its local retry budget. Runtime
+ * persistence failures are not row data failures: once receipts or sheet writes
+ * are unreliable, completing the run would hide paid/side-effecting work that
+ * no durable ledger can prove.
  */
 export function isRowIsolationExemptError(error: unknown): boolean {
   if (isAbortLikeError(error)) return true;
   if (error instanceof Error && error.name === 'GovernorBudgetError') {
     return true;
   }
+  if (isRuntimeReceiptPersistenceError(error)) return true;
+  if (isRuntimeStoragePersistenceError(error)) return true;
+  if (isRuntimePersistenceCircuitOpenError(error)) return true;
   return isHardBillingToolHttpError(error);
 }
+
+function isRuntimeReceiptPersistenceError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  if (
+    error instanceof Error &&
+    error.name === 'RuntimeReceiptPersistenceError'
+  ) {
+    return true;
+  }
+  const nestedErrors = (error as { errors?: unknown }).errors;
+  return (
+    Array.isArray(nestedErrors) &&
+    nestedErrors.some(isRuntimeReceiptPersistenceError)
+  );
+}
+
+function isRuntimeStoragePersistenceError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const nestedErrors = (error as { errors?: unknown }).errors;
+  if (
+    Array.isArray(nestedErrors) &&
+    nestedErrors.some(isRuntimeStoragePersistenceError)
+  ) {
+    return true;
+  }
+  const message =
+    error instanceof Error ? error.message : String((error as object) ?? '');
+  return (
+    /Runtime Postgres connection timed out/i.test(message) ||
+    /\bNeonDbError\b/i.test(message) ||
+    /timeout exceeded when trying to connect/i.test(message) ||
+    /connection terminated unexpectedly/i.test(message) ||
+    /permission denied (?:for|to grant) (?:schema|table|relation|database|sequence|role)\b/i.test(
+      message,
+    ) ||
+    /must be owner of (?:table|relation|sequence|schema|database|function|procedure|routine|type)\b/i.test(
+      message,
+    )
+  );
+}