deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts

@@ -1,4 +1,5 @@
 import { hydrateRetainedPlayPipelines } from '../../plays/execution-pipeline';
+import { isInternalWorkReceiptKey } from '../work-receipt-storage-contract';
 /**
  * Runtime receipt gateway dispatch.
  *
@@ -962,6 +963,41 @@ export function createReceiptGatewayDispatcher(input: {
     );
     const scopeError = requireReceiptScope(scope, body);
     if (scopeError) return scopeError;
+    if (
+      body.action !== 'get_runtime_step_receipt' &&
+      body.action !== 'get_runtime_step_receipts'
+    ) {
+      // Raw callers may carry extra fields. No key-bearing field can shadow another.
+      const keys: string[] = [];
+      if ('key' in body && typeof body.key === 'string') keys.push(body.key);
+      if ('keys' in body && Array.isArray(body.keys)) {
+        keys.push(
+          ...body.keys.filter((key): key is string => typeof key === 'string'),
+        );
+      }
+      if ('receipts' in body && Array.isArray(body.receipts)) {
+        keys.push(
+          ...body.receipts.flatMap((receipt) =>
+            receipt &&
+            typeof receipt === 'object' &&
+            'key' in receipt &&
+            typeof receipt.key === 'string'
+              ? [receipt.key]
+              : [],
+          ),
+        );
+      }
+      // Reject before value persistence or an injected/coalesced commit path.
+      if (keys.some((key) => isInternalWorkReceiptKey(key.trim()))) {
+        return receiptGatewayJsonResponse(
+          {
+            error:
+              'Internal runtime facts cannot be mutated through generic work receipt operations.',
+          },
+          { status: 400 },
+        );
+      }
+    }
     const runId = String(body.runId || scope.runId);
     // Additive read presentation. Older callers omit this and receive the
     // exact hydrated body they did before; only a Foundation cache probe asks

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts

@@ -15,6 +15,8 @@ export type RunLedgerDispatchLaunch = Pick<
   | 'runLedgerExecutorToken'
   | 'baseUrl'
   | 'runtimeApiBaseUrl'
+  | 'arrayInputTerminalRetirementSupported'
+  | 'arrayInputTerminalProjectionOrigin'
   | 'runtimeTokenOrigin'
   | 'orgId'
   | 'runtimeAuthority'
@@ -33,6 +35,8 @@ export function runLedgerDispatchLaunchSql(alias: string): string {
     'runLedgerExecutorToken',
     'baseUrl',
     'runtimeApiBaseUrl',
+    'arrayInputTerminalRetirementSupported',
+    'arrayInputTerminalProjectionOrigin',
     'runtimeTokenOrigin',
     'orgId',
     'runtimeAuthority',

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -12,6 +12,7 @@ import { isIsolatedRuntimeSchedulerSchema } from '../runtime-scheduler-topology'
 import {
   dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope,
   postgresSchedulerNotifyChannel,
+  readPostgresSchedulerOutboxAdmissionHints,
   readPostgresSchedulerOutboxNextDueAt,
   type PostgresSchedulerOptions,
   type PostgresSchedulerOutboxProjectionResult,
@@ -19,6 +20,7 @@ import {
   type RunLedgerOutboxHeadPriority,
   type RunLedgerOutboxTokenAuthority,
   type RunLedgerOutboxWithClient,
+  type RunLedgerOutboxAdmissionHint,
 } from './postgres';
 import {
   acquirePgNotifyHub,
@@ -74,7 +76,11 @@ export const RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT = 32;
 // The higher exported ceiling remains available to isolated recovery callers,
 // but it is not the steady-state production setting.
 const RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_IN_FLIGHT = 8;
-const RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT = 4;
+// Production runs two projector replicas, so four index deliveries per
+// replica provide eight shared slots. Preview has one projector replica; give
+// it the same total bounded search-index capacity.
+const RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_INDEX_IN_FLIGHT = 4;
+const RUN_LEDGER_OUTBOX_RECOVERY_PREVIEW_INDEX_IN_FLIGHT = 8;
 // Admissions retain three of four claim opportunities. The fourth selects
 // the oldest eligible causal head across all runs, preventing terminal work
 // from starving behind continuous new admissions. An empty class never costs
@@ -94,6 +100,11 @@ export type RunLedgerOutboxRecoveryDeps = {
     headPriority: RunLedgerOutboxHeadPriority,
     runId?: string,
   ) => Promise<PostgresSchedulerOutboxProjectionResult>;
+  /** Discardable admission candidates after a productive global delivery. */
+  discoverAdmissionHints?: (input: {
+    limit: number;
+    excludeRunIds: readonly string[];
+  }) => Promise<RunLedgerOutboxAdmissionHint[]>;
   /** Independent index deliveries share this controller and notification source. */
   dispatchObservabilityOnce?: (
     runId?: string,
@@ -134,6 +145,14 @@ export function runLedgerOutboxRecoveryParallelism(
     : RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_IN_FLIGHT;
 }
 
+export function runLedgerOutboxObservabilityParallelism(
+  environment: 'preview' | 'production',
+): number {
+  return environment === 'preview'
+    ? RUN_LEDGER_OUTBOX_RECOVERY_PREVIEW_INDEX_IN_FLIGHT
+    : RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_INDEX_IN_FLIGHT;
+}
+
 export type RunLedgerOutboxRecoveryController = {
   /**
    * Ask the singleton projector to re-read durable outbox work. This is
@@ -264,15 +283,21 @@ export function startRunLedgerOutboxRecovery(
   let nextDispatchId = 0;
   let lastDispatchSettledAt: number | null = null;
   let admissionTurns = 0;
+  type LifecycleOrigin = 'global' | 'producer_hint' | 'admission_snapshot';
   const dispatchOnce = async (
     observability = false,
     runId?: string,
+    origin: LifecycleOrigin = runId === undefined ? 'global' : 'producer_hint',
   ): Promise<PostgresSchedulerOutboxProjectionResult> => {
     const oldestReady =
       admissionTurns === ADMISSION_TURNS_PER_OLDEST_READY_TURN;
-    // Scoped claims cannot change which run wins, so they must not consume
-    // the global drain's starvation-prevention turns.
-    if (!observability && runId === undefined)
+    // Producer hints do not choose a global winner. Admission snapshots do,
+    // even though their final claim is scoped and can lose to another owner.
+    if (!observability && origin === 'admission_snapshot' && oldestReady)
+      throw new Error(
+        'The oldest-ready opportunity requires a fresh global claim.',
+      );
+    if (!observability && origin !== 'producer_hint')
       admissionTurns = oldestReady ? 0 : admissionTurns + 1;
     const id = nextDispatchId++;
     inFlightStartedAt.set(id, now());
@@ -373,7 +398,7 @@ export function startRunLedgerOutboxRecovery(
   };
 
   const scheduleFailureRetry = (
-    stage: 'dispatch' | 'next_due',
+    stage: 'dispatch' | 'next_due' | 'admission_discovery',
     error: unknown,
   ) => {
     failureCount += 1;
@@ -549,13 +574,24 @@ export function startRunLedgerOutboxRecovery(
       id: symbol;
       observability: boolean;
       runId?: string;
+      origin: LifecycleOrigin;
       result:
         | PromiseSettledResult<PostgresSchedulerOutboxProjectionResult>
         | { status: 'skipped' };
     };
     const pending = new Set<symbol>();
     const pendingObservability = new Set<symbol>();
+    const pendingRunIds = new Map<symbol, string>();
     const deliveries: Delivery[] = [];
+    let admissionHints: RunLedgerOutboxAdmissionHint[] = [];
+    let discoveryPending: Promise<void> | null = null;
+    let admissionWindowGeneration = 0;
+    // Reserve scheduling order without consuming fairness turns before the
+    // actual handoff. launch() defers dispatch by one microtask.
+    let scheduledGlobalTurns = 0;
+    const nextGlobalTurn = () =>
+      (admissionTurns + scheduledGlobalTurns) %
+      (ADMISSION_TURNS_PER_OLDEST_READY_TURN + 1);
     let refill = false;
     let observabilityRefill = false;
     let dispatchFailed = false;
@@ -583,21 +619,41 @@ export function startRunLedgerOutboxRecovery(
       completed = attempted = projected = retrying = blocked = 0;
     };
 
-    const launch = (observability = false, runId?: string) => {
+    const launch = (
+      observability = false,
+      runId?: string,
+      origin: LifecycleOrigin = runId === undefined
+        ? 'global'
+        : 'producer_hint',
+    ) => {
       const id = Symbol();
+      if (
+        !observability &&
+        origin === 'global' &&
+        nextGlobalTurn() === ADMISSION_TURNS_PER_OLDEST_READY_TURN
+      ) {
+        // A newly eligible oldest run must win from a fresh global snapshot.
+        // Nothing read for the preceding admission window survives this turn.
+        admissionHints = [];
+        admissionWindowGeneration++;
+      }
       pending.add(id);
       if (observability) pendingObservability.add(id);
+      if (!observability && runId !== undefined) pendingRunIds.set(id, runId);
+      if (!observability && origin !== 'producer_hint') scheduledGlobalTurns++;
       const settle = (result: Delivery['result']) => {
         if (result.status === 'rejected' && !dispatchFailed) {
           dispatchFailed = true;
           scheduleFailureRetry('dispatch', result.reason);
         }
-        deliveries.push({ id, observability, runId, result });
+        deliveries.push({ id, observability, runId, origin, result });
         wakeActiveDrain?.();
       };
       // Subscribe once per delivery. Repeated Promise.race calls would retain
       // a new reaction on a slow callback every time a faster slot completes.
       void Promise.resolve().then(async () => {
+        if (!observability && origin !== 'producer_hint')
+          scheduledGlobalTurns--;
         if (closed || dispatchFailed || cooldownUntilMs !== null) {
           settle({ status: 'skipped' });
           return;
@@ -605,7 +661,7 @@ export function startRunLedgerOutboxRecovery(
         try {
           settle({
             status: 'fulfilled',
-            value: await dispatchOnce(observability, runId),
+            value: await dispatchOnce(observability, runId, origin),
           });
         } catch (reason) {
           settle({ status: 'rejected', reason });
@@ -613,6 +669,51 @@ export function startRunLedgerOutboxRecovery(
       });
     };
 
+    const discover = (limit: number) => {
+      const generation = admissionWindowGeneration;
+      discoveryPending = Promise.resolve()
+        .then(() =>
+          deps.discoverAdmissionHints!({
+            limit,
+            excludeRunIds: [...new Set(pendingRunIds.values())],
+          }),
+        )
+        .then((hints) => {
+          if (
+            closed ||
+            dispatchFailed ||
+            cooldownUntilMs !== null ||
+            generation !== admissionWindowGeneration
+          )
+            return;
+          if (
+            hints.length > limit ||
+            hints.some(
+              (hint) => hint.origin !== 'admission_snapshot' || !hint.runId,
+            ) ||
+            new Set(hints.map((hint) => hint.runId)).size !== hints.length
+          )
+            throw new Error(
+              'Outbox admission discovery returned an invalid bounded window.',
+            );
+          admissionHints = hints;
+          if (hints.length === 0) {
+            refill = false;
+            void scheduleNextDue();
+          }
+        })
+        .catch((error) => {
+          if (!dispatchFailed) {
+            dispatchFailed = true;
+            scheduleFailureRetry('admission_discovery', error);
+          }
+        })
+        .finally(() => {
+          discoveryPending = null;
+          wakeActiveDrain?.();
+        });
+    };
+
     const canProbeLifecycle = () =>
       !closed &&
       !dispatchFailed &&
@@ -659,12 +760,33 @@ export function startRunLedgerOutboxRecovery(
           while (
             pending.size < parallelism &&
             pending.size - pendingObservability.size < lifecycleParallelism
-          )
-            launch();
-          refill = false;
+          ) {
+            if (!deps.discoverAdmissionHints) {
+              launch();
+              continue;
+            }
+            const turn = nextGlobalTurn();
+            if (turn === ADMISSION_TURNS_PER_OLDEST_READY_TURN) {
+              launch();
+              continue;
+            }
+            const hint = admissionHints.shift();
+            if (hint) {
+              launch(false, hint.runId, hint.origin);
+              continue;
+            }
+            if (!discoveryPending)
+              discover(ADMISSION_TURNS_PER_OLDEST_READY_TURN - turn);
+            break;
+          }
+          if (!discoveryPending)
+            refill =
+              admissionHints.length > 0 ||
+              (deps.discoverAdmissionHints !== undefined &&
+                nextGlobalTurn() === ADMISSION_TURNS_PER_OLDEST_READY_TURN);
         }
       }
-      if (pending.size === 0) {
+      if (pending.size === 0 && discoveryPending === null) {
         reportDeliveries();
         // Metadata discovery is single-flight, but never owns a claim slot.
         // A slow snapshot must not delay a producer's next scoped claim.
@@ -678,7 +800,14 @@ export function startRunLedgerOutboxRecovery(
         if (
           deliveries.length > 0 ||
           canProbeLifecycle() ||
-          canProbeObservability()
+          canProbeObservability() ||
+          (!closed &&
+            !dispatchFailed &&
+            cooldownUntilMs === null &&
+            refill &&
+            discoveryPending === null &&
+            pending.size < parallelism &&
+            pending.size - pendingObservability.size < lifecycleParallelism)
         )
           resolve();
       });
@@ -687,6 +816,7 @@ export function startRunLedgerOutboxRecovery(
       if (!delivery) continue;
       pending.delete(delivery.id);
       pendingObservability.delete(delivery.id);
+      pendingRunIds.delete(delivery.id);
       if (delivery.result.status !== 'fulfilled') {
         // Stop new claims but settle every already-started delivery before
         // cooldown or close can hand ownership to another drain.
@@ -706,7 +836,7 @@ export function startRunLedgerOutboxRecovery(
       }
       if (
         result.retrying > 0 ||
-        (delivery.runId === undefined && result.attempted > 0)
+        (delivery.origin !== 'producer_hint' && result.attempted > 0)
       )
         dueClockKnownLane = null;
       if (result.retrying > 0) dueClockGeneration += 1;
@@ -731,7 +861,7 @@ export function startRunLedgerOutboxRecovery(
         void scheduleNextDue(result.retrying > 0);
       // Retryable rows own durable retry clocks. The SQL claim keeps their
       // run ordered while successful slots keep unrelated runs moving.
-      if (delivery.runId !== undefined) {
+      if (delivery.runId !== undefined && delivery.origin === 'producer_hint') {
         // Continue only the notified run. Refilling with global claims here
         // would turn one productive scoped claim back into whole-outbox scans.
         if (result.attempted > 0) {
@@ -749,7 +879,7 @@ export function startRunLedgerOutboxRecovery(
         }
       } else if (delivery.observability) {
         observabilityRefill = result.attempted > 0;
-      } else refill = result.attempted > 0;
+      } else if (result.attempted > 0) refill = true;
     }
   };
 
@@ -1037,11 +1167,14 @@ export function startRunLedgerOutboxRecoveryController(input: {
   onDelivered?: (deliveries: readonly RunLedgerOutboxDeliveredRow[]) => void;
 }): RunLedgerOutboxRecoveryController {
   const schedulerOptions = input.schedulerOptions ?? {};
+  const environment = input.environment ?? 'production';
+  const observabilityParallelism =
+    runLedgerOutboxObservabilityParallelism(environment);
   const observabilityPoolOptions: RuntimeSchedulerPoolOptions = {
     ...input.poolOptions,
     purpose: 'observability',
     applicationName: 'deepline-runtime-observability',
-    maxClients: RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT,
+    maxClients: observabilityParallelism,
   };
   const dispatch = (
     headPriority: RunLedgerOutboxHeadPriority,
@@ -1088,15 +1221,21 @@ export function startRunLedgerOutboxRecoveryController(input: {
             }),
           ),
     dispatchOnce: (headPriority, runId) => dispatch(headPriority, false, runId),
+    discoverAdmissionHints: (discovery) =>
+      withRuntimeSchedulerClient(input.poolOptions, (client) =>
+        readPostgresSchedulerOutboxAdmissionHints(
+          client,
+          discovery,
+          schedulerOptions,
+        ),
+      ),
     dispatchObservabilityOnce: (runId) => dispatch('oldest_ready', true, runId),
-    observabilityParallelism: RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT,
+    observabilityParallelism,
     // `dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope` releases each
     // short scheduler checkout before it calls the Vercel gateway. Production
-    // retains two replicas at eight slots each; one isolated Preview replica
-    // uses the same controller's existing hard ceiling of 32.
-    parallelism: runLedgerOutboxRecoveryParallelism(
-      input.environment ?? 'production',
-    ),
+    // retains two replicas at eight lifecycle and four search slots each; one
+    // Preview replica uses 32 lifecycle and eight search slots.
+    parallelism: runLedgerOutboxRecoveryParallelism(environment),
     notifyUnavailableSafetyWakeMs: runLedgerOutboxNotifySafetyWakeMs(
       schedulerOptions.schema,
     ),