deepline 0.3.246 → 0.3.248

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts

@@ -361,13 +361,21 @@ function requireReceiptScope(
   if (!scope.orgId || !scope.playName || !scope.runId) {
     return receiptGatewayJsonResponse(
       { error: 'Executor token is missing org/run/play scope.' },
-      { status: 403 },
+      {
+        status: 403,
+        headers: { 'x-deepline-error-code': 'runtime_receipt_scope_missing' },
+      },
     );
   }
   if (body.playName !== scope.playName) {
     return receiptGatewayJsonResponse(
       { error: 'Receipt request playName does not match executor token.' },
-      { status: 403 },
+      {
+        status: 403,
+        headers: {
+          'x-deepline-error-code': 'runtime_receipt_play_scope_mismatch',
+        },
+      },
     );
   }
   if (body.runId && body.runId !== scope.runId) {
@@ -378,7 +386,12 @@ function requireReceiptScope(
     });
     return receiptGatewayJsonResponse(
       { error: 'Receipt request runId does not match executor token.' },
-      { status: 403 },
+      {
+        status: 403,
+        headers: {
+          'x-deepline-error-code': 'runtime_receipt_run_scope_mismatch',
+        },
+      },
     );
   }
   return null;
@@ -403,7 +416,12 @@ function requireSameRunReceipts(
   return invalid
     ? receiptGatewayJsonResponse(
         { error: 'Receipt runId does not match executor token scope.' },
-        { status: 403 },
+        {
+          status: 403,
+          headers: {
+            'x-deepline-error-code': 'runtime_receipt_run_scope_mismatch',
+          },
+        },
       )
     : null;
 }

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -41,7 +41,7 @@ const MAX_TIMEOUT_MS = 2_147_483_647;
  * therefore a network/projection concurrency budget, deliberately independent
  * of the projector's much smaller Postgres pool.
  *
- * A single successful probe starts a bounded wave; an idle projector still
+ * A single successful probe fills bounded delivery slots; an idle projector still
  * performs only one indexed head read per safety wake. `SKIP LOCKED` leases
  * distinct runs, retaining strict causal ordering within every run.
  */
@@ -125,6 +125,7 @@ export function startRunLedgerOutboxRecovery(
   let failureCount = 0;
   let timer: RecoveryTimer | null = null;
   let drainPromise: Promise<void> | null = null;
+  let wakeActiveDrain: (() => void) | null = null;
   let cooldownUntilMs: number | null = null;
   let hubLease: PgNotifyHubLease | null = null;
   let unsubscribeNotify: (() => void) | undefined;
@@ -162,7 +163,10 @@ export function startRunLedgerOutboxRecovery(
     requested = true;
     if (cooldownUntilMs !== null && now() < cooldownUntilMs) return;
     clearWakeTimer();
-    if (drainPromise) return;
+    if (drainPromise) {
+      wakeActiveDrain?.();
+      return;
+    }
     drainPromise = drain(reason)
       .catch((error) => {
         // drain() handles expected database/projection faults itself. This is a
@@ -179,7 +183,6 @@ export function startRunLedgerOutboxRecovery(
   };
 
   const recordProjectionRetry = (retrying: number) => {
-    failureCount += 1;
     logger.warn('[run-ledger-outbox.recovery_retry]', {
       stage: 'projection',
       retrying,
@@ -205,6 +208,7 @@ export function startRunLedgerOutboxRecovery(
       delayMs,
       error: error instanceof Error ? error.message : String(error),
     });
+    if (closed) return;
     clearWakeTimer();
     cooldownUntilMs = now() + delayMs;
     timer = setTimer(() => {
@@ -216,6 +220,7 @@ export function startRunLedgerOutboxRecovery(
   };
 
   const scheduleNextDue = async (): Promise<void> => {
+    if (closed) return;
     let dueAt: Date | null;
     try {
       dueAt = await deps.readNextDueAt();
@@ -247,79 +252,26 @@ export function startRunLedgerOutboxRecovery(
   };
 
   const drain = async (reason: string): Promise<void> => {
-    while (requested && !closed) {
-      requested = false;
-      // Probe first so an idle projector issues one small indexed read per
-      // wake rather than `parallelism` identical empty scans. Once the probe
-      // claims a row, the remaining slots can safely lease unrelated runs;
-      // each dispatcher releases its database client before gateway I/O.
-      let first: PostgresSchedulerOutboxProjectionResult;
-      try {
-        first = await deps.dispatchOnce();
-      } catch (error) {
-        scheduleFailureRetry('dispatch', error);
-        return;
-      }
-      const settled = [
-        {
-          status: 'fulfilled' as const,
-          value: first,
-        },
-        ...(first.attempted === 0
-          ? []
-          : await Promise.allSettled(
-              Array.from({ length: parallelism - 1 }, () =>
-                deps.dispatchOnce(),
-              ),
-            )),
-      ];
-      const rejected = settled.find(
-        (result): result is PromiseRejectedResult =>
-          result.status === 'rejected',
-      );
-      if (rejected) {
-        scheduleFailureRetry('dispatch', rejected.reason);
-        return;
-      }
-
-      const results = settled.map(
-        (result) =>
-          (
-            result as PromiseFulfilledResult<PostgresSchedulerOutboxProjectionResult>
-          ).value,
-      );
-      const retrying = results.reduce(
-        (total, result) => total + result.retrying,
-        0,
-      );
-      const attempted = results.reduce(
-        (total, result) => total + result.attempted,
-        0,
-      );
-      const projected = results.reduce(
-        (total, result) => total + result.projected,
-        0,
-      );
-      const blocked = results.reduce(
-        (total, result) => total + result.failed,
-        0,
-      );
-
-      if (retrying > 0) {
-        // The failed row owns its own durable retry clock. Do NOT stop this
-        // drain: the claim query preserves causal order within a run, while
-        // independent runs are safe to project now. Returning here turns one
-        // transient projection failure into global head-of-line blocking when
-        // this is the only healthy projector replica.
-        recordProjectionRetry(retrying);
-      } else {
-        failureCount = 0;
-      }
+    type Delivery = {
+      id: symbol;
+      result: PromiseSettledResult<PostgresSchedulerOutboxProjectionResult> | { status: 'skipped' };
+    };
+    const pending = new Set<symbol>();
+    const deliveries: Delivery[] = [];
+    let refill = false;
+    let dispatchFailed = false;
+    let completed = 0;
+    let attempted = 0;
+    let projected = 0;
+    let retrying = 0;
+    let blocked = 0;
 
+    const reportDeliveries = () => {
+      if (retrying > 0) recordProjectionRetry(retrying);
+      else if (!dispatchFailed) failureCount = 0;
       if (attempted > 0) {
-        // This is the durable queue's high-signal throughput event. It makes
-        // a delayed public Run observable as a projector/outbox problem,
-        // without emitting one noisy log line per normal database checkout.
+        // Retain the existing throughput event, aggregated over at most one
+        // slot-set of completions without making that set a delivery barrier.
         logger.info('[run-ledger-outbox.recovery_wave]', {
           reason,
           parallelism,
@@ -329,16 +281,88 @@ export function startRunLedgerOutboxRecovery(
           blocked,
         });
       }
+      completed = attempted = projected = retrying = blocked = 0;
+    };
 
-      // Continue until no event was claimable. A transient failure schedules
-      // its own durable retry, so the next pass naturally skips that run while
-      // moving unrelated runs forward. A permanent failure is settled once and
-      // likewise cannot block another run's causal prefix.
-      if (attempted > 0) {
-        requested = true;
+    const launch = () => {
+      const id = Symbol();
+      pending.add(id);
+      const settle = (result: Delivery['result']) => {
+        if (result.status === 'rejected' && !dispatchFailed) {
+          dispatchFailed = true;
+          scheduleFailureRetry('dispatch', result.reason);
+        }
+        deliveries.push({ id, result });
+        wakeActiveDrain?.();
+      };
+      // Subscribe once per delivery. Repeated Promise.race calls would retain
+      // a new reaction on a slow callback every time a faster slot completes.
+      void Promise.resolve().then(async () => {
+        if (closed || dispatchFailed) {
+          settle({ status: 'skipped' });
+          return;
+        }
+        try {
+          settle({ status: 'fulfilled', value: await deps.dispatchOnce() });
+        } catch (reason) {
+          settle({ status: 'rejected', reason });
+        }
+      });
+    };
+
+    for (;;) {
+      if (!closed && !dispatchFailed) {
+        // A wake grants one indexed probe, including when other slots are
+        // still delivering. Keep it pending when full so an empty result
+        // cannot consume a notification that arrived after its query began.
+        if (requested && pending.size < parallelism) {
+          requested = false;
+          launch();
+        }
+        // Productive completion permits immediate use of every free slot.
+        // Empty results grant no refill, so idle work cannot spin on scans.
+        if (refill) {
+          while (pending.size < parallelism) launch();
+          refill = false;
+        }
+      }
+      if (pending.size === 0) {
+        reportDeliveries();
+        if (!closed && !dispatchFailed) await scheduleNextDue();
+        return;
+      }
+
+      await new Promise<void>((resolve) => {
+        // Register before checking queued completions and notifications.
+        wakeActiveDrain = resolve;
+        if (
+          deliveries.length > 0 ||
+          (requested &&
+            !closed &&
+            !dispatchFailed &&
+            pending.size < parallelism)
+        )
+          resolve();
+      });
+      wakeActiveDrain = null;
+      const delivery = deliveries.shift();
+      if (!delivery) continue;
+      pending.delete(delivery.id);
+      if (delivery.result.status !== 'fulfilled') {
+        // Stop new claims but settle every already-started delivery before
+        // cooldown or close can hand ownership to another drain.
         continue;
       }
-      await scheduleNextDue();
+      const result = delivery.result.value;
+      completed += 1;
+      attempted += result.attempted;
+      projected += result.projected;
+      retrying += result.retrying;
+      blocked += result.failed;
+      if (completed >= parallelism) reportDeliveries();
+      // Retryable rows own durable retry clocks. The SQL claim keeps their
+      // run ordered while successful slots keep unrelated runs moving.
+      refill = result.attempted > 0;
     }
   };
 
@@ -353,6 +377,7 @@ export function startRunLedgerOutboxRecovery(
     close: async () => {
       if (closed) return;
       closed = true;
+      wakeActiveDrain?.();
       clearWakeTimer();
       cooldownUntilMs = null;
       unsubscribeNotify?.();