deepline 0.1.179 → 0.1.181

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -78,6 +78,13 @@ import {
   type DurableReceiptExecutionStore,
 } from './durable-receipt-execution';
 import { isRowIsolationExemptError } from './row-isolation';
+import {
+  createRuntimePersistenceLatch,
+  PROVIDER_DISPATCH_WAVE_SIZE,
+  RuntimePersistenceCircuitOpenError,
+  tripRuntimePersistenceLatch,
+  type RuntimePersistenceLatch,
+} from './persistence-latch';
 import { createRuntimeDatasetId } from './dataset-id';
 import { dedupeExplicitMapKeyRows } from './map-row-identity';
 import {
@@ -712,6 +719,16 @@ export class PlayContextImpl {
     createSecretRedactionContext();
   private mapInvocationIndex = 0;
   private readonly stepCallIndexByKey = new Map<string, number>();
+  /**
+   * Runtime persistence-failure circuit breaker (postgres_fast parity with the
+   * Workers runtime). The first persistence failure — a receipt-completion
+   * failure after a successful tool call, a receipt-failure write failure, or a
+   * map-row sheet flush failure — trips this latch; the dispatch loops then stop
+   * dispatching NEW provider calls so a dead tenant DB cannot keep billing calls
+   * that have nowhere durable to land.
+   */
+  private readonly persistenceLatch: RuntimePersistenceLatch =
+    createRuntimePersistenceLatch();
   /**
    * The single source of every concurrency, budget, and pacing decision for this
    * run-attempt. Tool/play/row slots are blocking semaphores; budgets are charged
@@ -1722,8 +1739,10 @@ export class PlayContextImpl {
         receiptKey,
       }),
     });
-    const completed = receiptKey
-      ? (
+    let completed: RuntimeStepReceipt | null | undefined = null;
+    if (receiptKey) {
+      try {
+        completed = (
           await this.completeRuntimeStepReceipts([
             {
               key: receiptKey,
@@ -1731,8 +1750,14 @@ export class PlayContextImpl {
               output: serializeToolExecuteResult(wrapped),
             },
           ])
-        ).get(receiptKey)
-      : null;
+        ).get(receiptKey);
+      } catch (receiptError) {
+        // Completion failed AFTER a successful (billed) tool call: trip the
+        // breaker so remaining rows stop dispatching new provider calls.
+        tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
+        throw receiptError;
+      }
+    }
     return await this.finalizeResolvedToolCall(
       toolId,
       request,
@@ -1781,10 +1806,19 @@ export class PlayContextImpl {
           ]
         : [];
     });
-    const completedByKey =
-      receiptInputs.length > 0
-        ? await this.completeRuntimeStepReceipts(receiptInputs)
-        : new Map<string, RuntimeStepReceipt>();
+    let completedByKey: Map<string, RuntimeStepReceipt>;
+    if (receiptInputs.length > 0) {
+      try {
+        completedByKey = await this.completeRuntimeStepReceipts(receiptInputs);
+      } catch (receiptError) {
+        // Completion failed AFTER successful (billed) tool calls: trip the
+        // breaker so remaining rows stop dispatching new provider calls.
+        tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
+        throw receiptError;
+      }
+    } else {
+      completedByKey = new Map<string, RuntimeStepReceipt>();
+    }
     const results: unknown[] = [];
     for (const entry of wrappedEntries) {
       const receiptKey = entry.request.receiptKey?.trim() || null;
@@ -1875,6 +1909,7 @@ export class PlayContextImpl {
           },
         ]);
       } catch (receiptError) {
+        tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
         rejectionError = new AggregateError(
           [error, receiptError],
           'Tool call failed and durable receipt could not be marked failed',
@@ -2285,17 +2320,24 @@ export class PlayContextImpl {
       let chunkBytes = 0;
       const flushChunk = async () => {
         if (chunk.length === 0) return;
-        await this.#options.onMapRowsCompleted!({
-          playName: this.#options.playName,
-          playId: this.#options.playId,
-          runId: this.#options.runId,
-          tableNamespace: resolvedTableNamespace,
-          rows: chunk,
-          outputFields: datasetColumnNames.filter((field) =>
-            shouldPersistMapCellField(field),
-          ),
-          staticPipeline: this.#options.staticPipeline ?? null,
-        });
+        try {
+          await this.#options.onMapRowsCompleted!({
+            playName: this.#options.playName,
+            playId: this.#options.playId,
+            runId: this.#options.runId,
+            tableNamespace: resolvedTableNamespace,
+            rows: chunk,
+            outputFields: datasetColumnNames.filter((field) =>
+              shouldPersistMapCellField(field),
+            ),
+            staticPipeline: this.#options.staticPipeline ?? null,
+          });
+        } catch (error) {
+          // Output-sheet flush failed: trip the breaker so the dispatch loops
+          // stop dispatching new provider calls for the rest of this map.
+          tripRuntimePersistenceLatch(this.persistenceLatch, error);
+          throw error;
+        }
         chunk = [];
         chunkBytes = 0;
       };
@@ -4742,8 +4784,23 @@ export class PlayContextImpl {
             await executeChunkedRequests({
               requests: compiledBatches,
               batchSize,
-              execute: async (batch) =>
-                await this.callToolAPI(
+              // Bounded dispatch wave (postgres_fast parity with workers_edge):
+              // cap the provider calls (counted by batch member size) a single
+              // chunk dispatches before the latch is read again, so a
+              // persistence failure prevents the rest of a large batch group.
+              maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
+              weightOf: (batch) => batch.memberRequests.length,
+              execute: async (batch) => {
+                // Circuit breaker: skip dispatching this batch's provider call
+                // once a persistence failure has occurred in this run.
+                if (this.persistenceLatch.tripped) {
+                  this.persistenceLatch.preventedCallCount +=
+                    batch.memberRequests.length;
+                  throw new RuntimePersistenceCircuitOpenError(
+                    this.persistenceLatch,
+                  );
+                }
+                return await this.callToolAPI(
                   batch.batchOperation,
                   batch.batchPayload,
                   {
@@ -4751,7 +4808,8 @@ export class PlayContextImpl {
                       batch.memberRequests,
                     ),
                   },
-                ),
+                );
+              },
               onChunkComplete: async (chunkResults) => {
                 for (const entry of chunkResults) {
                   if (entry.error !== undefined) {
@@ -4790,9 +4848,13 @@ export class PlayContextImpl {
               },
             });
           } else {
-            const batchSize = await this.governor.suggestedParallelism(
-              toolId,
-              50,
+            // Bounded dispatch wave (postgres_fast parity with workers_edge):
+            // an unbatched tool group can carry a whole chunk's rows, so cap the
+            // provider calls dispatched before the latch is read again to the
+            // wave size. Respect a lower provider-pacing suggestion when present.
+            const batchSize = Math.min(
+              await this.governor.suggestedParallelism(toolId, 50),
+              PROVIDER_DISPATCH_WAVE_SIZE,
             );
             for (
               let start = 0;
@@ -4802,6 +4864,18 @@ export class PlayContextImpl {
               const chunk = pendingRequests.slice(start, start + batchSize);
               await Promise.allSettled(
                 chunk.map(async (request) => {
+                  // Circuit breaker: do not dispatch this provider call once a
+                  // persistence failure has occurred in this run.
+                  if (this.persistenceLatch.tripped) {
+                    this.persistenceLatch.preventedCallCount += 1;
+                    await rejectWithLiveFollowers(
+                      request,
+                      new RuntimePersistenceCircuitOpenError(
+                        this.persistenceLatch,
+                      ),
+                    );
+                    return;
+                  }
                   try {
                     const execution = await this.callToolExecutionAPI(
                       toolId,

package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts

@@ -3,8 +3,13 @@ import type {
   PostgresUrlEncryptionRequest,
 } from './db-session-crypto';
 
-export const DB_SESSION_DEFAULT_TTL_SECONDS = 10 * 60;
-export const DB_SESSION_MAX_TTL_SECONDS = 30 * 60;
+// Workflow DB sessions must not expire before the workflow run/retry state they
+// pair with. Run/retry state lives 60 min (WORKFLOW_RUN_STATE_TTL_MS in the
+// coordinator), so these are aligned to 60 min. Raising the default WITHOUT
+// raising the max would leave the coordinator DO clamp / request validator
+// silently capping the session at 30 min, so both move together.
+export const DB_SESSION_DEFAULT_TTL_SECONDS = 60 * 60;
+export const DB_SESSION_MAX_TTL_SECONDS = 60 * 60;
 
 export const DB_SESSION_OPERATIONS = [
   'rows.read',

package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts

@@ -0,0 +1,167 @@
+/**
+ * Runtime persistence-failure circuit breaker.
+ *
+ * When a runtime persistence write fails mid-map (a work-receipt claim/complete
+ * or an output-sheet flush against the tenant Postgres), continuing to dispatch
+ * NEW provider calls only widens the "billed but not durable" gap: every extra
+ * call bills server-side yet has nowhere to land. The latch trips on the first
+ * such failure and is checked by the worker dispatch loops before they dispatch
+ * any further provider call, converting "all billed, half persisted" into
+ * "billed ≈ persisted + small in-flight window".
+ *
+ * The latch is a plain mutable object shared across all row workers of a map
+ * (single isolate, cooperative scheduling — no locking needed).
+ */
+export type RuntimePersistenceLatch = {
+  /** True once the first runtime persistence failure has been observed. */
+  tripped: boolean;
+  /** Formatted first-failure message (first trip wins). */
+  cause: string | null;
+  /** Provider calls NOT dispatched because the latch was already tripped. */
+  preventedCallCount: number;
+  /**
+   * True once a full completeReceipt retry ladder has been exhausted in this
+   * map. Bookkeeping only: the shorten decision now keys on `tripped` (see
+   * `shouldShortenCompleteReceiptLadder`) so fresh entrants collapse as soon as
+   * a persistence failure is confirmed rather than only after the first ladder
+   * fully drains.
+   */
+  latchRetryBudgetExhausted: boolean;
+};
+
+export function createRuntimePersistenceLatch(): RuntimePersistenceLatch {
+  return {
+    tripped: false,
+    cause: null,
+    preventedCallCount: 0,
+    latchRetryBudgetExhausted: false,
+  };
+}
+
+/**
+ * Bounded provider-dispatch wave size.
+ *
+ * A single map chunk can hold hundreds of rows. The persistence latch only
+ * trips AFTER a receipt completes (i.e. after the provider call already billed),
+ * so if every row in a big chunk dispatches its provider call in one synchronous
+ * wave, the latch cannot gate anything within that chunk — all calls bill before
+ * the first failure is even observed. That is the 250/250 incident (run
+ * play/x-lima-shard-03/run/20260703t231850-473-44f3d408: a customer billed
+ * 250/250 with preventedCallCount=0, because enrich batches compile to one
+ * ~245-row chunk with effectively unbounded row concurrency).
+ *
+ * Dispatching provider calls in waves of this size, with a single latch read
+ * between waves, bounds billed exposure for ANY chunk shape to ~one wave plus
+ * the calls already in flight when the latch trips. Constant on purpose (house
+ * preference: searchable, no env var).
+ */
+export const PROVIDER_DISPATCH_WAVE_SIZE = 25;
+
+/**
+ * Partition dispatch units into bounded waves. Each unit contributes `sizeOf`
+ * toward the wave budget: 1 for a single provider call, the batch member count
+ * for a batched group. A batch is atomic and is never split across waves — an
+ * over-budget batch forms its own wave and still counts its full member size
+ * toward the budget. Pure + synchronous so the wave arithmetic is unit-testable
+ * apart from the async dispatch loops that consume it.
+ */
+export function partitionDispatchWaves<T>(
+  units: readonly T[],
+  sizeOf: (unit: T) => number,
+  waveSize: number = PROVIDER_DISPATCH_WAVE_SIZE,
+): T[][] {
+  const budget = Math.max(1, Math.floor(waveSize));
+  const waves: T[][] = [];
+  let current: T[] = [];
+  let weight = 0;
+  for (const unit of units) {
+    if (current.length > 0 && weight >= budget) {
+      waves.push(current);
+      current = [];
+      weight = 0;
+    }
+    current.push(unit);
+    weight += Math.max(1, Math.floor(sizeOf(unit)));
+  }
+  if (current.length > 0) waves.push(current);
+  return waves;
+}
+
+function formatLatchCause(error: unknown): string {
+  const raw =
+    error instanceof Error
+      ? error.message
+      : typeof error === 'string'
+        ? error
+        : (() => {
+            try {
+              return JSON.stringify(error);
+            } catch {
+              return String(error);
+            }
+          })();
+  const message = (raw ?? '').replace(/\s+/g, ' ').trim();
+  if (!message) return 'Runtime persistence failed.';
+  return message.length > 1_000 ? `${message.slice(0, 1_000)}…` : message;
+}
+
+/**
+ * Trip the latch on the first persistence failure. Idempotent: later trips are
+ * no-ops so `cause` keeps the first, root failure.
+ */
+export function tripRuntimePersistenceLatch(
+  latch: RuntimePersistenceLatch,
+  error: unknown,
+): void {
+  if (latch.tripped) return;
+  latch.tripped = true;
+  latch.cause = formatLatchCause(error);
+}
+
+/**
+ * Thrown in place of dispatching a provider call once the latch is tripped. The
+ * message deliberately contains the substring `persistence failure` so that a
+ * `failed` receipt carrying this error is reclaimable on resume (see
+ * `canReclaimFailedWorkerToolReceipt`).
+ */
+export class RuntimePersistenceCircuitOpenError extends Error {
+  constructor(latch: RuntimePersistenceLatch) {
+    super(
+      `Runtime persistence failure circuit breaker: skipped dispatching this provider call ` +
+        `because a runtime persistence failure already occurred in this map. ` +
+        `First persistence error: ${latch.cause}`,
+    );
+    this.name = 'RuntimePersistenceCircuitOpenError';
+  }
+}
+
+/**
+ * Decide whether a completeReceipt retry ladder entrant should collapse to a
+ * single attempt instead of running the full backoff ladder.
+ *
+ * Keyed on `latch.tripped` (NOT `latchRetryBudgetExhausted`): the latch only
+ * trips once a persistence failure is CONFIRMED — either a full completeReceipt
+ * ladder already gave up, or an output-sheet flush failed. A transient blip that
+ * a ladder rides out never trips the latch, so the first ladder always keeps its
+ * full budget and blip tolerance is preserved. Once the latch is tripped the
+ * endpoint is known-dead, so every FRESH entrant collapses to one attempt rather
+ * than piling another full ladder of nested connect attempts onto a saturated
+ * database. A ladder that already committed to retrying (`committedToFullLadder`)
+ * is protected: it keeps its full budget so an in-progress leader is never
+ * truncated mid-flight by a peer that tripped the latch a moment later.
+ */
+export function shouldShortenCompleteReceiptLadder(
+  latch: RuntimePersistenceLatch,
+  committedToFullLadder: boolean,
+): boolean {
+  return latch.tripped && !committedToFullLadder;
+}
+
+export function isRuntimePersistenceCircuitOpenError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  if (error instanceof RuntimePersistenceCircuitOpenError) return true;
+  return (
+    error instanceof Error &&
+    error.name === 'RuntimePersistenceCircuitOpenError'
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/receipt-salvage.ts

@@ -0,0 +1,158 @@
+/**
+ * Salvage of orphaned billed tool results.
+ *
+ * When a provider call has billed server-side but its durable receipt cannot be
+ * marked completed (tenant Postgres unwritable) even after the retry ladder,
+ * the serialized tool output would otherwise live only in isolate memory and be
+ * lost when the run fails. This module collects those outputs and builds a
+ * size-capped salvage record that rides on the run.failed ledger event's
+ * `result` field — a different durable system (Convex run ledger) than the dead
+ * tenant Postgres.
+ *
+ * The salvage is a LOUD marker on a FAILED run, never a fallback that lets the
+ * run succeed. Entries carry exactly the bytes a receipt would have stored
+ * (customer-owned tool output); no provider-spend / cost metadata is added.
+ */
+
+/** Total salvage payload cap. Well under Convex's ~1MB document limit once the
+ * rest of the failure result + run snapshot are accounted for. */
+export const SALVAGE_MAX_TOTAL_BYTES = 262_144;
+/** Per-entry output cap: an oversized single output is dropped (metadata kept). */
+export const SALVAGE_MAX_ENTRY_BYTES = 32_768;
+
+export type ReceiptSalvageEntry = {
+  receiptKey: string;
+  toolId: string;
+  cacheKey?: string;
+  /** Serialized tool output (what completeReceipt would have persisted).
+   * Omitted when the output was dropped for size. */
+  output?: unknown;
+  /** UTF-8 byte length of the serialized output (0 when there was none). */
+  outputBytes: number;
+  /** True when this entry's output was dropped for size. */
+  truncated: boolean;
+  /** completeReceipt attempts made before giving up. */
+  completeAttempts: number;
+  /** epoch ms of the first completion failure for this entry. */
+  firstErrorAt: number;
+};
+
+export type ReceiptSalvage = {
+  entries: ReceiptSalvageEntry[];
+  /** Total orphaned results collected, including any dropped entirely. */
+  totalEntries: number;
+  /** Entries dropped entirely (not even metadata retained) for total-size cap. */
+  droppedEntries: number;
+  /** True when any output was dropped or any entry was dropped entirely. */
+  truncated: boolean;
+  /** Loud human-readable marker (see note strings below). */
+  note: string;
+};
+
+export type ReceiptSalvageInput = {
+  receiptKey: string;
+  toolId: string;
+  cacheKey?: string;
+  /** Serialized durable output (already JSON-safe, as receipts store it). */
+  output: unknown;
+  completeAttempts: number;
+  firstErrorAt: number;
+};
+
+function jsonByteLength(value: unknown): number {
+  let json: string | undefined;
+  try {
+    json = JSON.stringify(value);
+  } catch {
+    return 0;
+  }
+  if (!json) return 0;
+  return new TextEncoder().encode(json).length;
+}
+
+/**
+ * Cap-enforcing, pure builder. Keeps metadata-only entries until even metadata
+ * would exceed the total cap, then drops remaining entries entirely and counts
+ * them in `droppedEntries`.
+ */
+export function buildReceiptSalvage(
+  inputs: readonly ReceiptSalvageInput[],
+): ReceiptSalvage | null {
+  if (inputs.length === 0) return null;
+  const totalEntries = inputs.length;
+  const entries: ReceiptSalvageEntry[] = [];
+  // Reserve the enclosing `[` and `]`. Every appended entry also reserves one
+  // byte for the joining comma, so `usedBytes` stays a conservative upper bound
+  // on the serialized `entries` array length.
+  let usedBytes = 2;
+  let droppedEntries = 0;
+  let truncatedOutputs = 0;
+  for (const input of inputs) {
+    const outputBytes = jsonByteLength(input.output);
+    const metaOnly: ReceiptSalvageEntry = {
+      receiptKey: input.receiptKey,
+      toolId: input.toolId,
+      ...(input.cacheKey ? { cacheKey: input.cacheKey } : {}),
+      outputBytes,
+      truncated: outputBytes > 0,
+      completeAttempts: input.completeAttempts,
+      firstErrorAt: input.firstErrorAt,
+    };
+    const metaBytes = jsonByteLength(metaOnly) + 1;
+    // Even the metadata-only footprint would blow the total cap: drop entirely.
+    if (usedBytes + metaBytes > SALVAGE_MAX_TOTAL_BYTES) {
+      droppedEntries += 1;
+      continue;
+    }
+    const withOutput: ReceiptSalvageEntry = {
+      ...metaOnly,
+      truncated: false,
+      output: input.output,
+    };
+    const fullBytes = jsonByteLength(withOutput) + 1;
+    const canIncludeOutput =
+      outputBytes > 0 &&
+      outputBytes <= SALVAGE_MAX_ENTRY_BYTES &&
+      usedBytes + fullBytes <= SALVAGE_MAX_TOTAL_BYTES;
+    if (canIncludeOutput) {
+      entries.push(withOutput);
+      usedBytes += fullBytes;
+    } else {
+      entries.push(metaOnly);
+      if (outputBytes > 0) truncatedOutputs += 1;
+      usedBytes += metaBytes;
+    }
+  }
+  const truncated = droppedEntries > 0 || truncatedOutputs > 0;
+  const note = truncated
+    ? `SALVAGE TRUNCATED: ${truncatedOutputs + droppedEntries} of ${totalEntries} billed-but-unpersisted tool outputs exceeded the salvage size cap and were dropped. All listed receiptKeys were billed; re-running will re-execute unpersisted calls.`
+    : `${totalEntries} billed tool result(s) could not be durably persisted; serialized outputs are attached for manual export.`;
+  return { entries, totalEntries, droppedEntries, truncated, note };
+}
+
+/** Loud one-line suffix appended to the run.failed error message. */
+export function receiptSalvageFailureSuffix(salvage: ReceiptSalvage): string {
+  return `${salvage.totalEntries} provider result(s) were billed but could not be persisted; salvage attached to the run record.`;
+}
+
+/**
+ * Run-scoped collection buffer. Thin wrapper over an array so a single
+ * reference can be threaded to every scheduler in the run; `build()` runs the
+ * pure cap-enforcing builder at terminal time.
+ */
+export type ReceiptSalvageBuffer = {
+  push(input: ReceiptSalvageInput): void;
+  size(): number;
+  build(): ReceiptSalvage | null;
+};
+
+export function createReceiptSalvageBuffer(): ReceiptSalvageBuffer {
+  const inputs: ReceiptSalvageInput[] = [];
+  return {
+    push: (input) => {
+      inputs.push(input);
+    },
+    size: () => inputs.length,
+    build: () => buildReceiptSalvage(inputs),
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -7,6 +7,37 @@ export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
 export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
   'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
 
+export const WORKSPACE_STORAGE_NOT_READY_CODE = 'WORKSPACE_STORAGE_NOT_READY';
+
+// User-facing message. Contains NO SQL, role names, database names, or hosts by
+// construction, so it is safe past redactRuntimeInternalError (which only strips
+// postgres-URI passwords). The raw pg error goes in the Error `cause` for server
+// logs only, never into this message or the normalized public failure.
+export const WORKSPACE_STORAGE_NOT_READY_MESSAGE =
+  'Workspace storage for this organization is not ready, so the run was stopped before any provider calls were made (nothing was billed). ' +
+  'Ask an organization admin to run `deepline db repair` (POST /api/v2/ingestion/repair), then re-run. ' +
+  'If this keeps happening, contact Deepline support with the run ID.';
+
+export class WorkspaceStorageNotReadyError extends Error {
+  readonly code = WORKSPACE_STORAGE_NOT_READY_CODE;
+  constructor(options?: { cause?: unknown }) {
+    // Prefix the code token so `String(error)` and any stored error string
+    // carry it. The run doc persists only the message string, and the read
+    // path re-normalizes that string (extractPublicRunErrors), so the token
+    // is how `errorCode` survives the serialization boundary back to the CLI.
+    super(
+      `${WORKSPACE_STORAGE_NOT_READY_CODE}: ${WORKSPACE_STORAGE_NOT_READY_MESSAGE}`,
+    );
+    this.name = 'WorkspaceStorageNotReadyError';
+    // Attach `cause` manually instead of via the 2-arg Error constructor:
+    // ErrorOptions is ES2022, but the Convex tsconfig typechecks this file
+    // transitively under `lib: ES2021`, where the 2-arg form is a type error.
+    if (options?.cause !== undefined) {
+      (this as { cause?: unknown }).cause = options.cause;
+    }
+  }
+}
+
 export type PlayRunFailureDetails = {
   code: string;
   phase: string;
@@ -45,6 +76,19 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
       cause,
     };
   }
+  if (
+    error instanceof WorkspaceStorageNotReadyError ||
+    /\bWORKSPACE_STORAGE_NOT_READY\b/.test(cause)
+  ) {
+    return {
+      code: WORKSPACE_STORAGE_NOT_READY_CODE,
+      phase: 'storage',
+      // Deliberately return the clean constant, NOT `cause`: the raw string may
+      // carry the code token prefix, and we never surface tenant DB internals.
+      message: WORKSPACE_STORAGE_NOT_READY_MESSAGE,
+      retryable: false,
+    };
+  }
   const lowerCause = cause.toLowerCase();
   if (
     lowerCause.includes('neondberror') ||

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -112,6 +112,12 @@ const runtimeSheetEnsureCache = new Map<
 const DIRECT_POSTGRES_BATCH_SIZE = 10_000;
 const RUNTIME_DB_SESSION_ROW_LIMIT_FLOOR = DIRECT_POSTGRES_BATCH_SIZE;
 const APPEND_KEY_SUFFIX_LENGTH = 12;
+// On-demand (create_db_session) mint TTL. Deliberately NOT aligned with
+// DB_SESSION_DEFAULT_TTL_SECONDS (60 min): on-demand sessions are isolate-
+// cached and re-minted through sessionHasRenewalWindow, never stored in the
+// dedup DO, so they are not subject to the 410 expired-retention race the
+// 60-min alignment fixed. A shorter TTL only causes periodic re-mints (cheap;
+// the tenant runtime role is long-lived), not run failures.
 const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
 const RUNTIME_SHEET_ENSURE_CACHE_TTL_MS = 10 * 60_000;
 const RUNTIME_SHEET_READY_AFTER_ENSURE_RETRY_DELAYS_MS = [