deepline 0.3.407 → 0.3.409

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -18598,8 +18598,8 @@ function runLedgerOutboxPrefixClaimSql(
   const lockFence = fence
     ? `current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(fence.runId)}`
     : undefined;
-  const pending = (alias: string) =>
-    `${alias}.delivered_at IS NULL${
+  const pending = (alias: string, primaryKeyLookup = false) =>
+    `${primaryKeyLookup ? `coalesce(${alias}.delivered_at IS NULL, false)` : `${alias}.delivered_at IS NULL`}${
       acknowledgement ? ` AND NOT (${alias}.event_id = ANY($5::text[]))` : ''
     }`;
   // The dispatcher already chose this run. Seek its oldest pending fact
@@ -18628,11 +18628,20 @@ function runLedgerOutboxPrefixClaimSql(
         ? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})`
         : 'false';
   const findSearchBlockersOnce = seekRunHead && input.searchOnly === true;
-  const searchFactColumnsSql = (alias: string) =>
+  // Only ordinary scoped lifecycle enumeration changes. Its existing limit
+  // bounds candidate keys before the original eligibility/fence predicates;
+  // a frozen terminal in that window retains complete receipt membership.
+  // Carry the locked event type instead of fetching it again by primary key.
+  // Recheck pending/run identity without inviting stale partial-index scans.
+  // Predecessor checks share snapshot metadata; bounded keys own live row access.
+  const inspectLifecycleCandidates = seekRunHead && input.searchOnly !== true;
+  const claimFactColumnsSql = (alias: string) =>
     `${alias}.event_id, ${alias}.run_id, ${alias}.event_type,
      ${alias}.created_at, ${alias}.failed_at, ${alias}.next_attempt_at,
      ${alias}.lease_owner, ${alias}.lease_expires_at,
      ${supportsTerminalBatchMarker ? `${alias}.delivery_batch_started_at` : 'NULL::timestamptz AS delivery_batch_started_at'}`;
+  const lifecycleFactColumnsSql = (alias: string) =>
+    `${claimFactColumnsSql(alias)}, ${alias}.delivered_at`;
   const blockingPredecessorSql = (alias: string) => `(
       (${alias}.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
        AND NOT (${ingressPrefixSql}))
@@ -18646,6 +18655,27 @@ function runLedgerOutboxPrefixClaimSql(
         AND ${alias}.lease_owner IS DISTINCT FROM $3::text
       )
     )`;
+  // A modern frozen head cannot cross an unmarked predecessor. A witness in
+  // this snapshot's window therefore blocks every later key, even if a live
+  // candidate is rejected after locking. Keep full membership without that
+  // witness: an all-marked window can refill after a concurrent ACK.
+  const lifecycleRequiresFullMembershipSql = inspectLifecycleCandidates
+    ? `EXISTS (
+          SELECT 1 FROM lifecycle_window AS terminal
+          WHERE terminal.delivery_batch_started_at IS NOT NULL
+            AND terminal.event_type IN (${RUN_EVENT_KIND_SQL_LITERAL.runCompleted}, ${RUN_EVENT_KIND_SQL_LITERAL.runFailed}, ${RUN_EVENT_KIND_SQL_LITERAL.runCancelled})${
+              supportsTerminalBatchMarker && input.allowIngressPrefix === true
+                ? `
+            AND NOT EXISTS (
+              SELECT 1 FROM head AS frozen_head
+              CROSS JOIN lifecycle_window AS unmarked
+              WHERE frozen_head.delivery_batch_started_at IS NOT NULL
+                AND unmarked.delivery_batch_started_at IS NULL
+            )`
+                : ''
+            }
+        )`
+    : 'false';
   return `
       WITH ${
         acknowledgement
@@ -18681,9 +18711,8 @@ function runLedgerOutboxPrefixClaimSql(
                ${findSearchBlockersOnce ? 'candidate.created_at,' : ''}
                ${supportsTerminalBatchMarker ? 'candidate.delivery_batch_started_at' : 'NULL::timestamptz'} AS delivery_batch_started_at
         FROM ${tableName('outbox', options)} AS candidate
-        ${seekRunHead ? 'JOIN pending_head ON pending_head.event_id = candidate.event_id' : ''}
-        ${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
-        WHERE ${pending('candidate')}
+        ${seekRunHead ? '' : '\n        '}${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
+        WHERE ${seekRunHead ? `candidate.event_id = (SELECT event_id FROM pending_head) AND ${pending('candidate')}` : pending('candidate')}
           ${lockFence ? `AND (${lockFence})` : ''}
           ${fence?.expectedHeadEventId ? `AND candidate.event_id = ${escapeLiteral(fence.expectedHeadEventId)}` : ''}
           AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
@@ -18766,14 +18795,16 @@ function runLedgerOutboxPrefixClaimSql(
       ), ${
         findSearchBlockersOnce
           ? `search_prefix AS MATERIALIZED (
-        SELECT ${searchFactColumnsSql('pending')}
+        SELECT prefix.* FROM head
+        CROSS JOIN LATERAL (
+        SELECT ${claimFactColumnsSql('pending')}
         FROM ${tableName('outbox', options)} AS pending
-        JOIN head ON head.run_id = pending.run_id
-        WHERE pending.run_id = $2::text
+        WHERE pending.run_id = head.run_id
           AND ${pending('pending')}
           AND ${searchLaneSql('pending')} = true
         ORDER BY ${runLedgerOrderSql('pending')}
         LIMIT $1
+              ) AS prefix
       ), frozen_search_prefix AS MATERIALIZED (
         SELECT EXISTS (
           SELECT 1 FROM search_prefix AS fact
@@ -18791,7 +18822,7 @@ function runLedgerOutboxPrefixClaimSql(
         -- A frozen terminal receipt can contain marked facts beyond newer
         -- unmarked rows. Retain that full membership scan only for this case;
         -- limiting its raw window would strand an already attempted receipt.
-        SELECT ${searchFactColumnsSql('pending')}
+        SELECT ${claimFactColumnsSql('pending')}
         FROM ${tableName('outbox', options)} AS pending
         JOIN head ON head.run_id = pending.run_id
         WHERE (SELECT has_terminal FROM frozen_search_prefix)
@@ -18826,13 +18857,49 @@ function runLedgerOutboxPrefixClaimSql(
         LIMIT 1
       ),`
           : ''
+      }${
+        inspectLifecycleCandidates
+          ? `lifecycle_window AS MATERIALIZED (
+        SELECT prefix.* FROM head
+        CROSS JOIN LATERAL (
+          SELECT ${lifecycleFactColumnsSql('pending')}
+          FROM ${tableName('outbox', options)} AS pending
+          WHERE pending.run_id = head.run_id
+            AND pending.run_id = $2::text
+            AND ${pending('pending')}
+            AND ${searchLaneSql('pending')} = false
+          ORDER BY ${runLedgerOrderSql('pending')}
+          LIMIT $1
+        ) AS prefix
+      ), lifecycle_facts AS MATERIALIZED (
+        SELECT prefix.* FROM lifecycle_window AS prefix
+        WHERE NOT ${lifecycleRequiresFullMembershipSql}
+        UNION ALL
+        SELECT ${lifecycleFactColumnsSql('pending')}
+        FROM ${tableName('outbox', options)} AS pending
+        JOIN head ON head.run_id = pending.run_id
+        WHERE ${lifecycleRequiresFullMembershipSql}
+          AND pending.run_id = $2::text
+          AND ${pending('pending')}
+          AND ${searchLaneSql('pending')} = false
+      ), `
+          : ''
       }candidates AS MATERIALIZED (
-        SELECT outbox.event_id
+        ${
+          runId !== null
+            ? `SELECT candidate.event_id${inspectLifecycleCandidates ? ', candidate.event_type' : ''} FROM head
+        CROSS JOIN LATERAL (
+        SELECT outbox.event_id${inspectLifecycleCandidates ? ', outbox.event_type' : ''}
+        FROM ${tableName('outbox', options)} AS outbox
+        WHERE ${pending('outbox', inspectLifecycleCandidates)}
+          AND ${inspectLifecycleCandidates ? 'coalesce(outbox.run_id = head.run_id, false)' : 'outbox.run_id = head.run_id'}`
+            : `SELECT outbox.event_id
         FROM ${tableName('outbox', options)} AS outbox
         JOIN head ON head.run_id = outbox.run_id
         WHERE ${pending('outbox')}
-          ${runId !== null ? 'AND outbox.run_id = $2::text' : ''}
-          ${findSearchBlockersOnce ? 'AND outbox.event_id = ANY(ARRAY(SELECT event_id FROM search_facts))' : ''}
+          `
+        }
+          ${findSearchBlockersOnce ? 'AND outbox.event_id = ANY(ARRAY(SELECT event_id FROM search_facts))' : ''}${inspectLifecycleCandidates ? '\n          AND outbox.event_id = ANY(ARRAY(SELECT event_id FROM lifecycle_facts))' : ''}
           AND ${searchLaneSql('outbox')} = ${input.searchOnly === true ? 'true' : 'false'}
           AND ${outboxNotFailedSql('outbox')}
           -- Admission is causally first and must be projected alone. A later
@@ -18881,8 +18948,8 @@ function runLedgerOutboxPrefixClaimSql(
           )`
               : `          AND NOT EXISTS (
             SELECT 1
-            FROM ${tableName('outbox', options)} AS terminal_predecessor
-            WHERE terminal_predecessor.run_id = outbox.run_id
+            FROM ${inspectLifecycleCandidates ? 'lifecycle_facts' : tableName('outbox', options)} AS terminal_predecessor
+            WHERE terminal_predecessor.run_id = outbox.run_id${inspectLifecycleCandidates ? '\n              AND terminal_predecessor.run_id = $2::text' : ''}
               AND ${pending('terminal_predecessor')}
               AND ${
                 supportsTerminalBatchMarker
@@ -18897,16 +18964,16 @@ function runLedgerOutboxPrefixClaimSql(
               AND terminal_predecessor.event_type IN (
                 ${RUN_EVENT_KIND_SQL_LITERAL.runCompleted}, ${RUN_EVENT_KIND_SQL_LITERAL.runFailed}, ${RUN_EVENT_KIND_SQL_LITERAL.runCancelled}
               )
-              AND ${runLedgerPrecedesSql('terminal_predecessor', 'outbox', false, input.searchOnly === true)}
+              AND ${inspectLifecycleCandidates ? `${runLedgerPositionSql('terminal_predecessor')} < ${runLedgerPositionSql('outbox')}` : runLedgerPrecedesSql('terminal_predecessor', 'outbox', false, input.searchOnly === true)}
           )
           -- Claim only the contiguous due prefix. A delayed, leased, or failed
           -- predecessor retains ownership of everything that follows it.
           AND NOT EXISTS (
             SELECT 1
-            FROM ${tableName('outbox', options)} AS predecessor
-            WHERE predecessor.run_id = outbox.run_id
+            FROM ${inspectLifecycleCandidates ? 'lifecycle_facts' : tableName('outbox', options)} AS predecessor
+            WHERE predecessor.run_id = outbox.run_id${inspectLifecycleCandidates ? '\n              AND predecessor.run_id = $2::text' : ''}
               AND ${pending('predecessor')}
-              AND ${runLedgerPrecedesSql('predecessor', 'outbox', false, input.searchOnly === true)}
+              AND ${inspectLifecycleCandidates ? `${runLedgerPositionSql('predecessor')} < ${runLedgerPositionSql('outbox')}` : runLedgerPrecedesSql('predecessor', 'outbox', false, input.searchOnly === true)}
               AND ${blockingPredecessorSql('predecessor')}
           )
 `
@@ -18914,14 +18981,18 @@ function runLedgerOutboxPrefixClaimSql(
         ORDER BY ${runLedgerOrderSql('outbox')}
         FOR UPDATE OF outbox
         LIMIT $1
-      ), terminal_batch AS (
+      ${runId !== null ? '        ) AS candidate\n      ' : ''}), ${
+        inspectLifecycleCandidates
+          ? ''
+          : `terminal_batch AS (
         SELECT EXISTS (
           SELECT 1
           FROM ${tableName('outbox', options)} AS candidate
           JOIN candidates ON candidates.event_id = candidate.event_id
           WHERE candidate.event_type IN (${RUN_EVENT_KIND_SQL_LITERAL.runCompleted}, ${RUN_EVENT_KIND_SQL_LITERAL.runFailed}, ${RUN_EVENT_KIND_SQL_LITERAL.runCancelled})
         ) AS has_terminal
-      ), claimed AS (
+      ), `
+      }claimed AS (
         UPDATE ${tableName('outbox', options)} outbox
         SET lease_owner = $3,
             lease_expires_at = CASE
@@ -18931,14 +19002,14 @@ function runLedgerOutboxPrefixClaimSql(
             ${
               supportsTerminalBatchMarker
                 ? `, delivery_batch_started_at = CASE
-              WHEN terminal_batch.has_terminal ${input.allowIngressPrefix === true ? `OR ${urgentLifecyclePendingSql('outbox')} OR EXISTS (SELECT 1 FROM head WHERE head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})` : ''}
+              WHEN ${inspectLifecycleCandidates ? `EXISTS (SELECT 1 FROM candidates AS terminal_candidate WHERE terminal_candidate.event_type IN (${RUN_EVENT_KIND_SQL_LITERAL.runCompleted}, ${RUN_EVENT_KIND_SQL_LITERAL.runFailed}, ${RUN_EVENT_KIND_SQL_LITERAL.runCancelled}))` : 'terminal_batch.has_terminal'} ${input.allowIngressPrefix === true ? `OR ${urgentLifecyclePendingSql('outbox')} OR EXISTS (SELECT 1 FROM head WHERE head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})` : ''}
                 THEN COALESCE(outbox.delivery_batch_started_at, statement_timestamp())
               ELSE outbox.delivery_batch_started_at
             END`
                 : ''
             }
         FROM candidates
-        CROSS JOIN terminal_batch
+        ${inspectLifecycleCandidates ? '' : 'CROSS JOIN terminal_batch'}
         WHERE outbox.event_id = candidates.event_id
         RETURNING outbox.event_id, outbox.run_id, outbox.event_type,
                   outbox.payload_json, outbox.created_at, outbox.lease_owner,
@@ -19002,9 +19073,34 @@ function runLedgerOutboxPrefixClaimBatchSql(input: {
   lockStrength?: 'no_key_update';
   transportHeadSql?: string;
 }): string {
-  const encoded = escapeLiteral(JSON.stringify(input.params));
+  // Ordinary claims have four bounded primitive arguments. Give PostgreSQL
+  // constants instead of a scalar InitPlan for every repeated argument. Keep
+  // the JSON renderer for ACK arrays, transport/extension claims and values
+  // whose existing JSON conversion or cast failure must remain unchanged.
+  const directScalars =
+    input.params.length === 4 &&
+    !input.transportHeadSql &&
+    input.params.every((value, index) =>
+      index === 0 || index === 3
+        ? typeof value === 'number' &&
+          Number.isInteger(value) &&
+          value >= -2_147_483_648 &&
+          value <= 2_147_483_647
+        : value === null ||
+          (typeof value === 'string' &&
+            /^(?:[\u0001-\uD7FF\uE000-\uFFFF]|[\uD800-\uDBFF][\uDC00-\uDFFF])*$/.test(
+              value,
+            )),
+    );
+  const encoded = directScalars
+    ? undefined
+    : escapeLiteral(JSON.stringify(input.params));
   const parameter = (index: number) => {
     const value = input.params[index - 1];
+    if (directScalars) {
+      const type = typeof value === 'number' ? 'integer' : 'text';
+      return `(${value === null ? 'NULL' : escapeLiteral(String(value))}::${type})`;
+    }
     if (Array.isArray(value)) {
       return `(SELECT ARRAY(SELECT jsonb_array_elements_text(values_json->${index - 1})) FROM run_ledger_parameters)`;
     }
@@ -19022,9 +19118,11 @@ function runLedgerOutboxPrefixClaimBatchSql(input: {
       (match, index: string | undefined) =>
         index ? parameter(Number(index)) : match,
     )
-    .replace(
-      /^\s*WITH\s/,
-      `WITH run_ledger_parameters AS MATERIALIZED (SELECT ${encoded}::jsonb AS values_json), `,
+    // Use a callback so replacement metacharacters inside generated IDs stay literal.
+    .replace(/^\s*WITH\s/, (match) =>
+      directScalars
+        ? match
+        : `WITH run_ledger_parameters AS MATERIALIZED (SELECT ${encoded}::jsonb AS values_json), `,
     );
   return `BEGIN;
     SELECT set_config('lock_timeout', '${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms', true),
@@ -20025,8 +20123,9 @@ export async function markPostgresSchedulerOutboxDelivered(
 
 /**
  * Shared lease-fenced ACK, including the canonical retained-payload cleanup.
- * Known-run callers keep that scope visible to the planner: a materialized ID
- * array alone can make stale pending-index statistics favor a cross-run scan.
+ * Known-run callers recheck scope and pending state without offering a second
+ * index access path beside their exact IDs. Stale statistics otherwise favor a
+ * full run bitmap or pending partial-index scan before filtering those IDs.
  */
 function runLedgerAcknowledgeSql(
   options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
@@ -20047,9 +20146,9 @@ function runLedgerAcknowledgeSql(
                  AND run.run_event_store = 'planet_scale'
             ) THEN '{}'::jsonb ELSE outbox.payload_json END
         WHERE outbox.event_id = ANY(${eventIdsParameter}::text[])
-          ${runIdParameter ? `AND outbox.run_id = ${runIdParameter}::text` : ''}
+          ${runIdParameter ? `AND coalesce(outbox.run_id = ${runIdParameter}::text, false)` : ''}
           ${lockFence ? `AND (${lockFence})` : ''}
-          AND outbox.delivered_at IS NULL
+          AND ${runIdParameter ? 'coalesce(outbox.delivered_at IS NULL, false)' : 'outbox.delivered_at IS NULL'}
           AND (${leaseOwnerParameter}::text IS NULL OR outbox.lease_owner = ${leaseOwnerParameter})
         RETURNING outbox.event_id,
                   outbox.event_type,
@@ -21452,6 +21551,14 @@ async function dispatchRunLedgerOutboxBatchWithScope(
     }
   }
   let events = claimed.events;
+  const searchOnlyTerminalLogCarrier =
+    input.searchOnly === true &&
+    events.some(
+      (event) =>
+        event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs &&
+        (event.eventId.startsWith('runner-terminal-logs:') ||
+          event.eventId.startsWith('terminal-log-chunk:')),
+    );
   const unconfirmedTransportError = claimed.transportOwnershipUnconfirmed
     ? new RunLedgerProjectionLeaseLostError('delivery_lease_not_current')
     : null;
@@ -21521,6 +21628,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
   const appendAttempt: { transportAttemptId: string | null } = {
     transportAttemptId: null,
   };
+  // Fixed-shape diagnostic timings for the search-only terminal-log carrier.
+  // These are process-local observations and never enter durable run state.
+  let terminalLogMaterializeTransactionCommitMs: number | null = null;
+  let terminalLogAcknowledgeMs: number | null = null;
   const callerOnRequestAttempt = input.runtimeApi?.onRequestAttempt;
   const materializeDeferredObservability = async () => {
     if (
@@ -21579,6 +21690,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
             },
           ),
         );
+        const materializeStartedAtMs = performance.now();
         const checkoutStartedAtMs = performance.now();
         let materializeEnteredAtMs: number | null = null;
         try {
@@ -21639,6 +21751,14 @@ async function dispatchRunLedgerOutboxBatchWithScope(
             (searchPhaseTimingsMs.materializeAcquireAndRetry ?? 0) +
             (materializeEnteredAtMs ?? performance.now()) -
             checkoutStartedAtMs;
+          if (searchOnlyTerminalLogCarrier) {
+            const elapsedMs = Math.max(
+              0,
+              performance.now() - materializeStartedAtMs,
+            );
+            terminalLogMaterializeTransactionCommitMs =
+              (terminalLogMaterializeTransactionCommitMs ?? 0) + elapsedMs;
+          }
         }
       };
       try {
@@ -21701,20 +21821,31 @@ async function dispatchRunLedgerOutboxBatchWithScope(
         )
       )
         throw new Error('Search lease includes a lifecycle fact.');
-      const delivered = await observeSearchPhase('acknowledge', () =>
-        withClient(
-          (client) =>
-            markPostgresSchedulerOutboxDeliveredRows(
-              client,
-              events.map((event) => event.eventId),
-              { ...options, leaseOwner },
-            ),
-          {
-            runId: events[0]!.runId,
-            phase: 'run_ledger_outbox_search_acknowledge',
-          },
-        ),
-      );
+      const acknowledgeStartedAtMs = performance.now();
+      let delivered: RunLedgerOutboxDeliveredRow[];
+      try {
+        delivered = await observeSearchPhase('acknowledge', () =>
+          withClient(
+            (client) =>
+              markPostgresSchedulerOutboxDeliveredRows(
+                client,
+                events.map((event) => event.eventId),
+                { ...options, leaseOwner },
+              ),
+            {
+              runId: events[0]!.runId,
+              phase: 'run_ledger_outbox_search_acknowledge',
+            },
+          ),
+        );
+      } finally {
+        if (searchOnlyTerminalLogCarrier) {
+          terminalLogAcknowledgeMs = Math.max(
+            0,
+            performance.now() - acknowledgeStartedAtMs,
+          );
+        }
+      }
       if (delivered.length !== events.length)
         throw new RunLedgerProjectionLeaseLostError(
           'delivery_lease_not_current',
@@ -22441,6 +22572,18 @@ async function dispatchRunLedgerOutboxBatchWithScope(
         continuationMs:
           acknowledgedAtMs === null ? null : settledAtMs - acknowledgedAtMs,
         durationMs: settledAtMs - dispatchStartedAtMs,
+        ...(searchOnlyTerminalLogCarrier
+          ? {
+              // Keep this fixed-shape scalar packet on the existing diagnostic;
+              // no per-event payload, rows, or durable fields are added.
+              searchOnlyTerminalLogPhaseTimingsMs: {
+                claim: Math.max(0, claimElapsedMs),
+                materializeTransactionCommit:
+                  terminalLogMaterializeTransactionCommitMs,
+                acknowledge: terminalLogAcknowledgeMs,
+              },
+            }
+          : {}),
         ...(input.searchOnly ? { searchPhaseTimingsMs } : {}),
         // Which platform requests served the two app hops, so a slow
         // authority or append stage can be found in the app's request log.

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -119,6 +119,21 @@ export type RunLedgerOutboxRecoveryDeps = {
   logger?: RunLedgerOutboxRecoveryLogger;
 };
 
+/**
+ * Production runs two projector replicas at eight deliveries each. An
+ * isolated Preview estate has one replica, so use the existing bounded
+ * 32-delivery ceiling there to keep fixture admission and its catalog
+ * projection inside the same latency budget. This does not change Production
+ * capacity or the controller's hard maximum.
+ */
+export function runLedgerOutboxRecoveryParallelism(
+  environment: 'preview' | 'production',
+): number {
+  return environment === 'preview'
+    ? RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT
+    : RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_IN_FLIGHT;
+}
+
 export type RunLedgerOutboxRecoveryController = {
   /**
    * Ask the singleton projector to re-read durable outbox work. This is
@@ -1011,6 +1026,7 @@ export async function deliverRunCreatedOutboxEventInline(input: {
 export function startRunLedgerOutboxRecoveryController(input: {
   poolOptions: RuntimeSchedulerPoolOptions;
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  environment?: 'preview' | 'production';
   /** Direct LISTEN is a latency hint and may be disabled for isolated schemas. */
   useDirectNotify?: boolean;
   /** Observation hook for acknowledged rows (lag metrics). */
@@ -1071,10 +1087,12 @@ export function startRunLedgerOutboxRecoveryController(input: {
     dispatchObservabilityOnce: (runId) => dispatch('oldest_ready', true, runId),
     observabilityParallelism: RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT,
     // `dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope` releases each
-    // short scheduler checkout before it calls the Vercel gateway. The two
-    // replicas therefore drain up to sixteen independent runs concurrently
-    // without turning one outbox notification into sixty-four head scans.
-    parallelism: RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_IN_FLIGHT,
+    // short scheduler checkout before it calls the Vercel gateway. Production
+    // retains two replicas at eight slots each; one isolated Preview replica
+    // uses the same controller's existing hard ceiling of 32.
+    parallelism: runLedgerOutboxRecoveryParallelism(
+      input.environment ?? 'production',
+    ),
     notifyUnavailableSafetyWakeMs: runLedgerOutboxNotifySafetyWakeMs(
       schedulerOptions.schema,
     ),

package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts

@@ -335,7 +335,13 @@ function formatInsufficientCreditsMessage(input: {
     getStringField(input.billing, 'operation_id') ??
     getStringField(input.billing, 'operation') ??
     input.toolId;
-  const balance = formatCreditAmount(input.billing.balance_credits);
+  // The explicit field is authoritative, including null. The older
+  // balance_credits field remains a fallback for already persisted errors.
+  const spendableBalance =
+    input.billing.spendable_balance_credits !== undefined
+      ? input.billing.spendable_balance_credits
+      : input.billing.balance_credits;
+  const balance = formatCreditAmount(spendableBalance);
   const required = formatCreditAmount(input.billing.required_credits);
   const recommended = formatCreditAmount(
     input.billing.recommended_add_credits ?? input.billing.needed_credits,
@@ -357,7 +363,7 @@ function formatInsufficientCreditsMessage(input: {
     typeof minimum === 'number' &&
     input.billing.required_credits === minimum
   ) {
-    return `Workspace balance ${balance} is below the ${formatCreditAmount(minimum)}-credit workspace minimum for starting a billable call (${operation}); this is a workspace-wide minimum, separate from the tool's own price.${estimatedPrice}${addSuffix}`;
+    return `Spendable workspace balance ${balance} credits is below the ${formatCreditAmount(minimum)}-credit workspace minimum for starting a billable call (${operation}); this is a workspace-wide minimum, separate from the tool's own price.${estimatedPrice} Spendable credits exclude pending spending and can be lower than the posted balance shown by billing balance.${addSuffix}`;
   }
   // A required balance above the call's estimate is headroom, not a price.
   if (
@@ -365,9 +371,9 @@ function formatInsufficientCreditsMessage(input: {
     typeof input.billing.required_credits === 'number' &&
     input.billing.required_credits > estimate
   ) {
-    return `Workspace balance ${balance} is below the ${required}-credit balance required to start ${operation}; this is a required balance, not the price.${estimatedPrice}${addSuffix}`;
+    return `Spendable workspace balance ${balance} credits is below the ${required}-credit balance required to start ${operation}; this is a required balance, not the price.${estimatedPrice} Spendable credits exclude pending spending and can be lower than the posted balance shown by billing balance.${addSuffix}`;
   }
-  return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
+  return `Spendable workspace balance ${balance} credits < required ${required} for ${operation}. Spendable credits exclude pending spending and can be lower than the posted balance shown by billing balance.${addSuffix}`;
 }
 
 /** An HTML document, as an edge proxy (Cloudflare, Vercel) serves for a 5xx. */

package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts

@@ -561,6 +561,16 @@ function parseExecuteToolMetadata(
     extractors: readExtractors(record.extractors),
     targetGetters: readGetters(record.targetGetters),
     listExtractorPaths,
+    ...(isRecord(record.listExtractorAliases)
+      ? {
+          listExtractorAliases: Object.fromEntries(
+            Object.entries(record.listExtractorAliases).filter(
+              (entry): entry is [string, string] =>
+                typeof entry[1] === 'string' && entry[1].length > 0,
+            ),
+          ),
+        }
+      : {}),
     listIdentityGetters: readGetters(record.listIdentityGetters),
   };
 }
@@ -725,6 +735,9 @@ function candidateResultPaths(path: string): string[] {
     providerPath,
     normalizeResultPath(path),
     toV2RawToolOutputPathPreservingProviderData(path),
+    ...(path.startsWith('result.data.')
+      ? [`toolResponse.raw.${path.slice('result.data.'.length)}`]
+      : []),
   ];
   return candidates.filter(
     (candidate, index, all) =>
@@ -1142,6 +1155,7 @@ function coerceToEnum(
 function resolveListRows(
   result: unknown,
   listExtractorPaths: readonly string[] | undefined,
+  aliases?: Readonly<Record<string, string>>,
 ): Record<string, { path: string; rows: Record<string, unknown>[] }> {
   const lists: Record<
     string,
@@ -1178,6 +1192,9 @@ function resolveListRows(
     }
     lists[listName] = resolved;
   }
+  for (const [alias, source] of Object.entries(aliases ?? {})) {
+    if (!lists[alias] && lists[source]) lists[alias] = lists[source];
+  }
   return lists;
 }
 
@@ -1546,6 +1563,7 @@ export function createToolExecuteResult<TResult = unknown>(input: {
   const resolvedLists = resolveListRows(
     resultRoot,
     input.metadata.listExtractorPaths,
+    input.metadata.listExtractorAliases,
   );
   const lists = buildLists(resolvedLists, input.metadata);
   const metadata = {
@@ -1553,6 +1571,9 @@ export function createToolExecuteResult<TResult = unknown>(input: {
     execution: input.execution,
     targets,
     listExtractorPaths: [...(input.metadata.listExtractorPaths ?? [])],
+    ...(input.metadata.listExtractorAliases
+      ? { listExtractorAliases: { ...input.metadata.listExtractorAliases } }
+      : {}),
     ...(input.metadata.extractors
       ? { extractors: input.metadata.extractors }
       : {}),
@@ -1849,6 +1870,7 @@ function serializedListRowsFromRaw(input: {
   const resolved = resolveListRows(
     { toolResponse: { raw: input.raw } },
     input.metadata.listExtractorPaths,
+    input.metadata.listExtractorAliases,
   );
   const entries = Object.entries(resolved).map(([name, list]) => [
     name,
@@ -2134,6 +2156,9 @@ function metadataInputFromToolExecuteResult(
     listExtractorPaths:
       value._metadata.listExtractorPaths ??
       Object.values(value._metadata.lists).map((list) => list.path),
+    ...(value._metadata.listExtractorAliases
+      ? { listExtractorAliases: { ...value._metadata.listExtractorAliases } }
+      : {}),
     listIdentityGetters: Object.fromEntries(
       Object.values(value._metadata.lists)
         .flatMap((list) => Object.entries(list.keys))
@@ -2479,6 +2504,53 @@ export function cloneToolExecuteResultWithExecution<TResult>(
   }) as ToolExecuteResult<TResult>;
 }
 
+/**
+ * Add current list getter declarations when replaying a result written by an
+ * older catalog. Stored declarations keep their precedence; the canonical
+ * provider response remains the source for any newly exposed getters.
+ */
+export function mergeToolExecuteResultListGetterMetadata(
+  value: ToolExecuteResult,
+  currentMetadata: Pick<
+    ToolResultMetadataInput,
+    'listExtractorPaths' | 'listExtractorAliases'
+  >,
+): ToolExecuteResult {
+  const serialized = serializeToolExecuteResult(value);
+  const storedMetadata = serialized.metadata;
+  const listExtractorPaths = [
+    ...new Set([
+      ...(storedMetadata.listExtractorPaths ?? []),
+      ...(currentMetadata.listExtractorPaths ?? []),
+    ]),
+  ];
+  const listExtractorAliases = {
+    ...(currentMetadata.listExtractorAliases ?? {}),
+    ...(storedMetadata.listExtractorAliases ?? {}),
+  };
+  const hasAliases = Object.keys(listExtractorAliases).length > 0;
+  if (
+    sameJsonValue(
+      listExtractorPaths,
+      storedMetadata.listExtractorPaths ?? [],
+    ) &&
+    sameJsonValue(
+      hasAliases ? listExtractorAliases : undefined,
+      storedMetadata.listExtractorAliases,
+    )
+  ) {
+    return value;
+  }
+  return deserializeToolExecuteResult({
+    ...serialized,
+    metadata: {
+      ...storedMetadata,
+      listExtractorPaths,
+      ...(hasAliases ? { listExtractorAliases } : {}),
+    },
+  });
+}
+
 /**
  * Native splitters return the logical source response; the historical adapter
  * stores it inside an extra `{ data }` layer. Keep that immutable receipt and