deepline 0.3.183 → 0.3.185

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -1288,10 +1288,14 @@ export async function parkAndResolveSuspension(input: {
         boundaryId: suspension.boundaryId,
         eventKey: suspension.eventKey,
         timeoutMs: suspension.timeoutMs,
+        ...(suspension.eventExpiresAtMs !== undefined
+          ? { eventExpiresAtMs: suspension.eventExpiresAtMs }
+          : {}),
+        ...(suspension.postTimeoutPark ? { postTimeoutPark: true } : {}),
         ...(suspension.messageRef ? { messageRef: suspension.messageRef } : {}),
       },
       claimSignal: input.claimSignal,
-    });
+    }).then((result) => result.checkpoint);
   }
 
   if (suspension.kind === 'integration_event_batch') {
@@ -1299,13 +1303,25 @@ export async function parkAndResolveSuspension(input: {
     for (const boundary of [...suspension.boundaries].sort(
       (a, b) => a.timeoutMs - b.timeoutMs,
     )) {
-      resolvedCheckpoint = await resolveIntegrationEvent({
+      // An earlier event may have resolved while the provider timer elapsed.
+      // Do not enter another event wait after that provider row is due.
+      if (
+        suspension.providerWakeAtMs !== undefined &&
+        suspension.providerWakeAtMs <= Date.now() &&
+        boundary.eventExpiresAtMs !== undefined &&
+        boundary.eventExpiresAtMs > Date.now()
+      )
+        return resolvedCheckpoint;
+      const resolved = await resolveIntegrationEvent({
         ctx,
         runId,
         checkpoint: resolvedCheckpoint,
         boundary,
         claimSignal: input.claimSignal,
+        providerWakeAtMs: suspension.providerWakeAtMs,
       });
+      resolvedCheckpoint = resolved.checkpoint;
+      if (resolved.providerWake) return resolvedCheckpoint;
     }
     return resolvedCheckpoint;
   }
@@ -1382,13 +1398,16 @@ async function resolveIntegrationEvent(input: {
     boundaryId: string;
     eventKey: string;
     timeoutMs: number;
+    eventExpiresAtMs?: number;
+    postTimeoutPark?: boolean;
     messageRef?: { channel: string; ts: string };
   };
+  providerWakeAtMs?: number;
   claimSignal: (
     eventKey: string,
     boundaryId: string | null,
   ) => Promise<PostgresSchedulerSignalRecord | null>;
-}): Promise<PlayCheckpoint> {
+}): Promise<{ checkpoint: PlayCheckpoint; providerWake: boolean }> {
   const { ctx, runId, checkpoint, boundary } = input;
   // Suspensions written before `messageRef` became part of the suspension
   // contract still have the armed Slack message reference in their checkpoint.
@@ -1403,31 +1422,89 @@ async function resolveIntegrationEvent(input: {
     (checkpointBoundary?.kind === 'integration_event'
       ? checkpointBoundary.messageRef
       : undefined);
+  const checkpointOutput =
+    checkpointBoundary?.kind === 'integration_event'
+      ? checkpointBoundary.output
+      : null;
+  const postTimeoutPark =
+    boundary.postTimeoutPark === true ||
+    (checkpointOutput !== null &&
+      typeof checkpointOutput === 'object' &&
+      !Array.isArray(checkpointOutput) &&
+      (checkpointOutput as Record<string, unknown>).timed_out === true);
+  const originalEventExpiresAtMs =
+    boundary.eventExpiresAtMs ??
+    (checkpointBoundary?.kind === 'integration_event'
+      ? checkpointBoundary.eventExpiresAtMs
+      : undefined);
+  const terminalParkAnchorMs =
+    checkpointBoundary?.kind === 'integration_event'
+      ? (checkpointBoundary.completedAt ?? Date.now())
+      : Date.now();
+  const eventExpiresAtMs = postTimeoutPark
+    ? (boundary.eventExpiresAtMs ??
+      terminalParkAnchorMs + Math.max(0, boundary.timeoutMs))
+    : (originalEventExpiresAtMs ??
+      Date.now() + Math.max(0, boundary.timeoutMs));
+  // Old event checkpoints have only an Absurd step timer, not an absolute
+  // expiry. Reuse that original step instead of granting a new full timeout
+  // whenever a provider row wakes.
+  const providerWakeAtMs =
+    postTimeoutPark || originalEventExpiresAtMs === undefined
+      ? undefined
+      : input.providerWakeAtMs;
 
   // Belt-and-suspenders: an early durable signal short-circuits the wait.
-  const early = await input.claimSignal(boundary.eventKey, boundary.boundaryId);
+  const early = postTimeoutPark
+    ? null
+    : await input.claimSignal(boundary.eventKey, boundary.boundaryId);
   if (early) {
-    return absurdCheckpointWithIntegrationEvent({
-      checkpoint,
-      boundaryId: boundary.boundaryId,
-      eventKey: boundary.eventKey,
-      timeoutMs: boundary.timeoutMs,
-      ...(messageRef ? { messageRef } : {}),
-      output: absurdIntegrationEventBoundaryOutput({
+    return {
+      checkpoint: absurdCheckpointWithIntegrationEvent({
+        checkpoint,
+        boundaryId: boundary.boundaryId,
         eventKey: boundary.eventKey,
-        signalData: signalData(early),
-        messageRef,
-        timedOut: false,
+        timeoutMs: boundary.timeoutMs,
+        ...(messageRef ? { messageRef } : {}),
+        output: absurdIntegrationEventBoundaryOutput({
+          eventKey: boundary.eventKey,
+          signalData: signalData(early),
+          messageRef,
+          timedOut: false,
+        }),
       }),
-    });
+      providerWake: false,
+    };
+  }
+
+  if (
+    providerWakeAtMs !== undefined &&
+    providerWakeAtMs <= Date.now() &&
+    eventExpiresAtMs > Date.now()
+  ) {
+    return { checkpoint: absurdCheckpointBase(checkpoint), providerWake: true };
   }
 
   const eventName = absurdIntegrationEventName(runId, boundary.eventKey);
+  const eventWaitMs = Math.max(0, eventExpiresAtMs - Date.now());
+  const providerWaitMs =
+    providerWakeAtMs === undefined
+      ? Number.POSITIVE_INFINITY
+      : Math.max(0, providerWakeAtMs - Date.now());
+  const waitMs = Math.min(eventWaitMs, providerWaitMs);
   let timedOut = false;
   try {
     await ctx.awaitEvent(eventName, {
-      stepName: boundary.boundaryId,
-      timeout: Math.max(0, boundary.timeoutMs) / 1000,
+      // Absurd persists waits by step name. A provider timer must never
+      // inherit an earlier event timeout (or vice versa) on replay.
+      stepName: postTimeoutPark
+        ? `${boundary.boundaryId}:terminal-park:${terminalParkAnchorMs}`
+        : providerWakeAtMs === undefined
+          ? boundary.boundaryId
+          : `${boundary.boundaryId}:provider-wake:${providerWakeAtMs}`,
+      // Absurd floors seconds. Round up so a positive remainder never wakes
+      // before the provider's authoritative not-before time.
+      timeout: Math.ceil(waitMs / 1_000),
     });
   } catch (error) {
     if (error instanceof TimeoutError) {
@@ -1438,6 +1515,25 @@ async function resolveIntegrationEvent(input: {
     }
   }
 
+  if (postTimeoutPark) {
+    return {
+      checkpoint: absurdCheckpointWithIntegrationEvent({
+        checkpoint,
+        boundaryId: boundary.boundaryId,
+        eventKey: boundary.eventKey,
+        timeoutMs: boundary.timeoutMs,
+        ...(messageRef ? { messageRef } : {}),
+        output:
+          checkpointOutput !== null &&
+          typeof checkpointOutput === 'object' &&
+          !Array.isArray(checkpointOutput)
+            ? (checkpointOutput as Record<string, unknown>)
+            : { timed_out: true },
+      }),
+      providerWake: false,
+    };
+  }
+
   // Claim the authoritative durable signal row for the payload — the absurd
   // event carries no payload we trust. On timeout, sweep once more before
   // giving up (mirrors today's belt-and-suspenders resume).
@@ -1445,20 +1541,32 @@ async function resolveIntegrationEvent(input: {
     boundary.eventKey,
     boundary.boundaryId,
   );
+  if (
+    !claimed &&
+    timedOut &&
+    providerWakeAtMs !== undefined &&
+    providerWakeAtMs < eventExpiresAtMs &&
+    Date.now() < eventExpiresAtMs
+  ) {
+    return { checkpoint: absurdCheckpointBase(checkpoint), providerWake: true };
+  }
   const resolvedTimedOut = timedOut && !claimed;
-  return absurdCheckpointWithIntegrationEvent({
-    checkpoint,
-    boundaryId: boundary.boundaryId,
-    eventKey: boundary.eventKey,
-    timeoutMs: boundary.timeoutMs,
-    ...(messageRef ? { messageRef } : {}),
-    output: absurdIntegrationEventBoundaryOutput({
+  return {
+    checkpoint: absurdCheckpointWithIntegrationEvent({
+      checkpoint,
+      boundaryId: boundary.boundaryId,
       eventKey: boundary.eventKey,
-      signalData: signalData(claimed),
-      messageRef,
-      timedOut: resolvedTimedOut,
+      timeoutMs: boundary.timeoutMs,
+      ...(messageRef ? { messageRef } : {}),
+      output: absurdIntegrationEventBoundaryOutput({
+        eventKey: boundary.eventKey,
+        signalData: signalData(claimed),
+        messageRef,
+        timedOut: resolvedTimedOut,
+      }),
     }),
-  });
+    providerWake: false,
+  };
 }
 
 /**

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts

@@ -0,0 +1,80 @@
+/**
+ * One quota scope owns both learning evidence and the dispatch budget. Rule
+ * rows retain configured ceilings and concurrency; they must not independently
+ * learn from copies of the same provider feedback. These CTEs are shared by
+ * pure-rate and concurrency-bearing admission, in the same atomic grant SQL.
+ */
+const AIMD_RECOVERY_STEP_MS = 1_000;
+// Increase relative to the learned rate, not by a fixed RPS amount: a long
+// Retry-After can imply a very small quota, where even +1 RPS would be an
+// enormous and repeated over-probe. A 25% clean-success step still recovers a
+// healthy 1->5 RPS provider in eight observations instead of seventeen.
+const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
+
+export function acquireRateScopeSql(
+  table: string,
+  successesParameter: string,
+): string {
+  return `provider_hold_rows AS (
+    SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
+  ), provider_hold AS (
+    SELECT h.*,
+      LEAST(
+        GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, h.refill_per_sec),
+        h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * h.refill_per_sec / 1000.0
+      ) AS tokens_after_refill,
+      ${successesParameter}::int = 0
+        AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms)
+          > GREATEST(1000, 2000.0 / h.refill_per_sec) AS observation_was_idle,
+      CASE WHEN ${successesParameter}::int > 0
+          AND h.rate_observation_started_at_ms IS NOT NULL
+          AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
+            THEN LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})
+            ELSE h.rate_recovery_interval_ms END
+          AND h.retry_at_ms <= nv.now_ms
+        THEN LEAST(h.max_rps, CASE WHEN h.seen_backpressure
+          THEN h.refill_per_sec * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})
+          ELSE h.refill_per_sec * 2 END)
+        ELSE h.refill_per_sec END AS next_refill_per_sec,
+      CASE WHEN ${successesParameter}::int > 0
+          AND h.rate_observation_started_at_ms IS NOT NULL
+          AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
+            THEN LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})
+            ELSE h.rate_recovery_interval_ms END
+          AND h.retry_at_ms <= nv.now_ms
+        THEN nv.now_ms ELSE h.last_ramp_at_ms END AS next_ramp_at_ms
+    FROM provider_hold_rows h CROSS JOIN now_value nv
+  )`;
+}
+
+export function updateRateScopeSql(table: string): string {
+  return `updated_scope AS (
+    UPDATE ${table} h SET
+      tokens = ph.tokens_after_refill - (SELECT granted FROM grant_calc),
+      refilled_at_ms = nv.now_ms,
+      refill_per_sec = ph.next_refill_per_sec,
+      last_ramp_at_ms = ph.next_ramp_at_ms,
+      rate_observation_started_at_ms = CASE
+        WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN nv.now_ms
+        WHEN (SELECT granted FROM grant_calc) > 0 THEN COALESCE(ph.rate_observation_started_at_ms, nv.now_ms)
+        ELSE ph.rate_observation_started_at_ms END,
+      rate_successes = CASE
+        WHEN (SELECT granted FROM grant_calc) > 0
+          AND ph.rate_observation_started_at_ms IS NULL THEN 0
+        WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
+        ELSE ph.rate_successes END,
+      rate_throttle_ceiling_rps = CASE
+        WHEN (SELECT granted FROM grant_calc) > 0
+          AND ph.rate_observation_started_at_ms IS NULL THEN NULL
+        ELSE ph.rate_throttle_ceiling_rps END,
+      next_dispatch_at_ms = CASE WHEN (SELECT granted FROM grant_calc) > 0 THEN
+        (SELECT first_at_ms FROM schedule_calc)
+          + CEIL((SELECT granted FROM grant_calc)::numeric / (SELECT burst_size FROM schedule_calc))::bigint
+            * (SELECT spacing_ms FROM schedule_calc) * (SELECT burst_size FROM schedule_calc)
+        ELSE ph.next_dispatch_at_ms END
+    FROM provider_hold ph CROSS JOIN now_value nv
+    WHERE h.bucket_id = $1
+    RETURNING h.bucket_id, h.rate_observation_started_at_ms, h.rate_successes,
+      h.rate_recovery_interval_ms, h.seen_backpressure
+  )`;
+}