deepline 0.3.247 → 0.3.249

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts

@@ -361,13 +361,21 @@ function requireReceiptScope(
   if (!scope.orgId || !scope.playName || !scope.runId) {
     return receiptGatewayJsonResponse(
       { error: 'Executor token is missing org/run/play scope.' },
-      { status: 403 },
+      {
+        status: 403,
+        headers: { 'x-deepline-error-code': 'runtime_receipt_scope_missing' },
+      },
     );
   }
   if (body.playName !== scope.playName) {
     return receiptGatewayJsonResponse(
       { error: 'Receipt request playName does not match executor token.' },
-      { status: 403 },
+      {
+        status: 403,
+        headers: {
+          'x-deepline-error-code': 'runtime_receipt_play_scope_mismatch',
+        },
+      },
     );
   }
   if (body.runId && body.runId !== scope.runId) {
@@ -378,7 +386,12 @@ function requireReceiptScope(
     });
     return receiptGatewayJsonResponse(
       { error: 'Receipt request runId does not match executor token.' },
-      { status: 403 },
+      {
+        status: 403,
+        headers: {
+          'x-deepline-error-code': 'runtime_receipt_run_scope_mismatch',
+        },
+      },
     );
   }
   return null;
@@ -403,7 +416,12 @@ function requireSameRunReceipts(
   return invalid
     ? receiptGatewayJsonResponse(
         { error: 'Receipt runId does not match executor token scope.' },
-        { status: 403 },
+        {
+          status: 403,
+          headers: {
+            'x-deepline-error-code': 'runtime_receipt_run_scope_mismatch',
+          },
+        },
       )
     : null;
 }

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -41,7 +41,7 @@ const MAX_TIMEOUT_MS = 2_147_483_647;
  * therefore a network/projection concurrency budget, deliberately independent
  * of the projector's much smaller Postgres pool.
  *
- * A single successful probe opens bounded delivery slots; an idle projector still
+ * A single successful probe fills bounded delivery slots; an idle projector still
  * performs only one indexed head read per safety wake. `SKIP LOCKED` leases
  * distinct runs, retaining strict causal ordering within every run.
  */
@@ -125,12 +125,12 @@ export function startRunLedgerOutboxRecovery(
   let failureCount = 0;
   let timer: RecoveryTimer | null = null;
   let drainPromise: Promise<void> | null = null;
+  let wakeActiveDrain: (() => void) | null = null;
   let cooldownUntilMs: number | null = null;
   let hubLease: PgNotifyHubLease | null = null;
   let unsubscribeNotify: (() => void) | undefined;
   let unsubscribeState: (() => void) | undefined;
   let requestDrain: (reason: string) => void = () => undefined;
-  let wakeDrainer: (() => void) | null = null;
 
   const clearWakeTimer = () => {
     if (!timer) return;
@@ -164,7 +164,7 @@ export function startRunLedgerOutboxRecovery(
     if (cooldownUntilMs !== null && now() < cooldownUntilMs) return;
     clearWakeTimer();
     if (drainPromise) {
-      wakeDrainer?.();
+      wakeActiveDrain?.();
       return;
     }
     drainPromise = drain(reason)
@@ -251,135 +251,119 @@ export function startRunLedgerOutboxRecovery(
     timer.unref?.();
   };
 
-  type DeliveryOutcome = {
-    delivery: Promise<DeliveryOutcome>;
-  } & (
-    | { status: 'fulfilled'; value: PostgresSchedulerOutboxProjectionResult }
-    | { status: 'rejected' | 'skipped' }
-  );
-
   const drain = async (reason: string): Promise<void> => {
-    const volume = { attempted: 0, projected: 0, retrying: 0, blocked: 0 };
-    const reportVolume = () => {
-      if (volume.attempted === 0) return;
-      logger.info('[run-ledger-outbox.recovery_wave]', {
-        reason,
-        parallelism,
-        ...volume,
-      });
-      volume.attempted = 0;
-      volume.projected = 0;
-      volume.retrying = 0;
-      volume.blocked = 0;
-    };
-    const recordResult = (
-      result: PostgresSchedulerOutboxProjectionResult,
-      resetFailures = true,
-    ) => {
-      volume.attempted += result.attempted;
-      volume.projected += result.projected;
-      volume.retrying += result.retrying;
-      volume.blocked += result.failed;
-      if (result.retrying > 0) {
-        // Each retrying run owns its durable retry clock. Independent runs
-        // remain eligible; this is not a controller-wide delivery cooldown.
-        recordProjectionRetry(result.retrying);
-      } else if (resetFailures) {
-        failureCount = 0;
-      }
-      // Streaming delivery has no wave barrier at which to report progress.
-      // Keep logs bounded per committed volume, and flush at idle/shutdown.
-      if (volume.attempted >= parallelism) reportVolume();
+    type Delivery = {
+      id: symbol;
+      result: PromiseSettledResult<PostgresSchedulerOutboxProjectionResult> | { status: 'skipped' };
     };
+    const pending = new Set<symbol>();
+    const deliveries: Delivery[] = [];
+    let refill = false;
+    let dispatchFailed = false;
+    let completed = 0;
+    let attempted = 0;
+    let projected = 0;
+    let retrying = 0;
+    let blocked = 0;
 
-    while (requested && !closed) {
-      requested = false;
-      // Probe first so an idle projector issues one small indexed read per
-      // wake rather than `parallelism` identical empty scans. Once the probe
-      // claims a row, the remaining slots can safely lease unrelated runs;
-      // each dispatcher releases its database client before gateway I/O.
-      let first: PostgresSchedulerOutboxProjectionResult;
-      try {
-        first = await deps.dispatchOnce();
-      } catch (error) {
-        scheduleFailureRetry('dispatch', error);
-        return;
-      }
-      recordResult(first);
-      if (first.attempted === 0) {
-        await scheduleNextDue();
-        continue;
+    const reportDeliveries = () => {
+      if (retrying > 0) recordProjectionRetry(retrying);
+      else if (!dispatchFailed) failureCount = 0;
+      if (attempted > 0) {
+        // Retain the existing throughput event, aggregated over at most one
+        // slot-set of completions without making that set a delivery barrier.
+        logger.info('[run-ledger-outbox.recovery_wave]', {
+          reason,
+          parallelism,
+          attempted,
+          projected,
+          retrying,
+          blocked,
+        });
       }
+      completed = attempted = projected = retrying = blocked = 0;
+    };
 
-      const active = new Set<Promise<DeliveryOutcome>>();
-      let dispatchFailed = false;
-      const startDelivery = () => {
-        const delivery: Promise<DeliveryOutcome> = Promise.resolve().then(
-          async (): Promise<DeliveryOutcome> => {
-            if (closed || dispatchFailed)
-              return { delivery, status: 'skipped' };
-            try {
-              return {
-                delivery,
-                status: 'fulfilled',
-                value: await deps.dispatchOnce(),
-              };
-            } catch (error) {
-              // Stop refills as soon as this rejection is observed, including
-              // when another already-completed slot wins Promise.race first.
-              if (!dispatchFailed) {
-                dispatchFailed = true;
-                scheduleFailureRetry('dispatch', error);
-              }
-              return { delivery, status: 'rejected' };
-            }
-          },
-        );
-        active.add(delivery);
-      };
-      const fillSlots = () => {
-        if (closed || dispatchFailed) return;
-        while (active.size < parallelism) {
-          startDelivery();
+    const launch = () => {
+      const id = Symbol();
+      pending.add(id);
+      const settle = (result: Delivery['result']) => {
+        if (result.status === 'rejected' && !dispatchFailed) {
+          dispatchFailed = true;
+          scheduleFailureRetry('dispatch', result.reason);
         }
+        deliveries.push({ id, result });
+        wakeActiveDrain?.();
       };
-      fillSlots();
-      while (active.size > 0) {
-        const wake = new Promise<null>((resolve) => {
-          wakeDrainer = () => resolve(null);
-        });
-        const outcome = await Promise.race([...active, wake]);
-        wakeDrainer = null;
-        if (outcome !== null) {
-          active.delete(outcome.delivery);
-          if (outcome.status === 'fulfilled') {
-            recordResult(outcome.value, !dispatchFailed);
-            if (outcome.value.attempted > 0 && !closed && !dispatchFailed) {
-              // Refill this completed slot immediately. A slower independent
-              // run never owns another run's delivery capacity.
-              startDelivery();
-            }
-          }
+      // Subscribe once per delivery. Repeated Promise.race calls would retain
+      // a new reaction on a slow callback every time a faster slot completes.
+      void Promise.resolve().then(async () => {
+        if (closed || dispatchFailed) {
+          settle({ status: 'skipped' });
+          return;
         }
-        if (
-          requested &&
-          !closed &&
-          !dispatchFailed &&
-          active.size < parallelism
-        ) {
-          // A new notification can use slots that previously found no work,
-          // even while another slot is still awaiting its gateway response.
-          // Keep the hint while all slots are occupied: an already-started
-          // empty claim does not establish idle after this notification.
+        try {
+          settle({ status: 'fulfilled', value: await deps.dispatchOnce() });
+        } catch (reason) {
+          settle({ status: 'rejected', reason });
+        }
+      });
+    };
+
+    for (;;) {
+      if (!closed && !dispatchFailed) {
+        // A wake grants one indexed probe, including when other slots are
+        // still delivering. Keep it pending when full so an empty result
+        // cannot consume a notification that arrived after its query began.
+        if (requested && pending.size < parallelism) {
           requested = false;
-          fillSlots();
+          launch();
+        }
+        // Productive completion permits immediate use of every free slot.
+        // Empty results grant no refill, so idle work cannot spin on scans.
+        if (refill) {
+          while (pending.size < parallelism) launch();
+          refill = false;
         }
       }
-      reportVolume();
-      if (dispatchFailed || closed) return;
-      await scheduleNextDue();
+      if (pending.size === 0) {
+        reportDeliveries();
+        if (!closed && !dispatchFailed) await scheduleNextDue();
+        return;
+      }
+
+      await new Promise<void>((resolve) => {
+        // Register before checking queued completions and notifications.
+        wakeActiveDrain = resolve;
+        if (
+          deliveries.length > 0 ||
+          (requested &&
+            !closed &&
+            !dispatchFailed &&
+            pending.size < parallelism)
+        )
+          resolve();
+      });
+      wakeActiveDrain = null;
+      const delivery = deliveries.shift();
+      if (!delivery) continue;
+      pending.delete(delivery.id);
+      if (delivery.result.status !== 'fulfilled') {
+        // Stop new claims but settle every already-started delivery before
+        // cooldown or close can hand ownership to another drain.
+        continue;
+      }
+      const result = delivery.result.value;
+      completed += 1;
+      attempted += result.attempted;
+      projected += result.projected;
+      retrying += result.retrying;
+      blocked += result.failed;
+      if (completed >= parallelism) reportDeliveries();
+      // Retryable rows own durable retry clocks. The SQL claim keeps their
+      // run ordered while successful slots keep unrelated runs moving.
+      refill = result.attempted > 0;
     }
-    reportVolume();
   };
 
   // Subscribe first, then scan. Notifications are lossy by design, so every
@@ -393,7 +377,7 @@ export function startRunLedgerOutboxRecovery(
     close: async () => {
       if (closed) return;
       closed = true;
-      wakeDrainer?.();
+      wakeActiveDrain?.();
       clearWakeTimer();
       cooldownUntilMs = null;
       unsubscribeNotify?.();