deepline 0.3.184 → 0.3.186

package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts

@@ -85,6 +85,8 @@ export function createRuntimeFlow<Input>(input: {
 export type RuntimeDispatchProviderPermit = RuntimeFlowLease & {
   cancel(): Promise<void>;
   park?(): Promise<void>;
+  /** Recheck a shared provider hold at the physical dispatch boundary. */
+  validate?(): Promise<boolean>;
 };
 
 export type RuntimeDispatchFlowInput = {
@@ -92,6 +94,11 @@ export type RuntimeDispatchFlowInput = {
   signal?: AbortSignal;
   /** Fixtures consume provider capacity but acquire app transport after delay. */
   requiresTransportPermit: boolean;
+  /**
+   * Defer shared-hold validation until the request callback reaches its actual
+   * outbound send. The callback must call the supplied boundary exactly once.
+   */
+  deferProviderPermitValidation?: boolean;
   /** Emits the request-side transport wait observed by the caller's metrics. */
   onTransportAdmitted?: (waitMs: number) => void;
 };
@@ -114,12 +121,18 @@ type RuntimeDispatchPermit<
  */
 export type RuntimeDispatchScope = {
   /**
-   * Run the one physical request. `beforeRelease` executes after the response
-   * is known but while the provider lease is still valid, so a caller can
-   * durably record response-bound feedback that the lease authorizes.
+   * Run the one physical request. Deferred-validation callers must invoke
+   * `beforeProviderDispatch` immediately before the outbound send, after any
+   * async setup/queueing. `beforeRelease` executes after the response is known
+   * but while the provider lease is still valid, so a caller can durably
+   * record response-bound feedback that the lease authorizes.
    */
   request<Output>(
-    work: () => Output | Promise<Output>,
+    work: (
+      beforeProviderDispatch: (
+        onPermitInvalidated?: () => void,
+      ) => boolean | Promise<boolean>,
+    ) => Output | Promise<Output>,
     beforeRelease?: () => void | Promise<void>,
   ): Promise<Output>;
 };
@@ -130,10 +143,12 @@ export type RuntimeDispatchFlow<
   /**
    * Admit and run exactly one physical provider request.
    *
-   * Work that exits before `request` returns the provider permit through its
-   * cancellation path. Once `request` starts, the permit is released after
-   * the request body is consumed, even if that body cannot be verified. This
-   * preserves the ambiguity boundary while making resource lifetime private.
+   * Work must call `beforeProviderDispatch` at the actual outbound-send edge.
+   * That is where shared Retry-After state is checked: not before async
+   * request preparation. Work that exits before that call returns the provider
+   * permit through its cancellation path. Once the send starts, the permit is
+   * released after the response body is consumed, even if it cannot be
+   * verified.
    */
   run<Output>(
     input: RuntimeDispatchFlowInput,
@@ -264,8 +279,10 @@ export function createRuntimeDispatchFlow<
       admission: RuntimeDispatchFlowInput,
       work: (scope: RuntimeDispatchScope) => Output | Promise<Output>,
     ): Promise<Output> {
-      const permit = await admit(admission);
+      let permit = await admit(admission);
       let transportLease = permit.integrationRequestLease;
+      let requiresTransportPermitForReacquire =
+        admission.requiresTransportPermit;
       let requestStarted = false;
       let requestUsed = false;
       let scopeOpen = true;
@@ -284,7 +301,11 @@ export function createRuntimeDispatchFlow<
       try {
         const output = await work({
           request: <RequestOutput>(
-            request: () => RequestOutput | Promise<RequestOutput>,
+            request: (
+              beforeProviderDispatch: (
+                onPermitInvalidated?: () => void,
+              ) => boolean | Promise<boolean>,
+            ) => RequestOutput | Promise<RequestOutput>,
             beforeRelease?: () => void | Promise<void>,
           ): Promise<RequestOutput> => {
             if (!scopeOpen) {
@@ -301,10 +322,69 @@ export function createRuntimeDispatchFlow<
             }
             requestUsed = true;
             requestPromise = (async () => {
+              let dispatchStarted = false;
+              const beforeProviderDispatch = (
+                onPermitInvalidated?: () => void,
+              ): boolean | Promise<boolean> => {
+                if (dispatchStarted) {
+                  throw new Error(
+                    'Runtime dispatch flow permits exactly one physical provider dispatch.',
+                  );
+                }
+                if (!permit.providerPermit.validate) {
+                  // Preserve the zero-round-trip in-memory path: deferred
+                  // callers still cross the exact send boundary, but do not
+                  // pay an artificial microtask when there is nothing shared
+                  // to revalidate.
+                  dispatchStarted = true;
+                  requestStarted = true;
+                  return false;
+                }
+                return validateAndStartProviderDispatch(onPermitInvalidated);
+              };
+              const validateAndStartProviderDispatch = async (
+                onPermitInvalidated?: () => void,
+              ): Promise<boolean> => {
+                let replaced = false;
+                while (
+                  permit.providerPermit.validate &&
+                  !(await permit.providerPermit.validate())
+                ) {
+                  requestStarted = false;
+                  onPermitInvalidated?.();
+                  await releasePermit();
+                  permit = await admit({
+                    ...admission,
+                    requiresTransportPermit:
+                      requiresTransportPermitForReacquire,
+                  });
+                  transportLease = permit.integrationRequestLease;
+                  requestStarted = false;
+                  permitReleased = false;
+                  replaced = true;
+                }
+                dispatchStarted = true;
+                if (admission.deferProviderPermitValidation) {
+                  requestStarted = true;
+                }
+                return replaced;
+              };
               try {
+                if (!admission.deferProviderPermitValidation) {
+                  // Preserve the ordinary API: its request callback begins
+                  // the physical send immediately. Callers with asynchronous
+                  // preparation opt into deferred validation and invoke this
+                  // same boundary at the final outbound edge.
+                  if (permit.providerPermit.validate) {
+                    await beforeProviderDispatch();
+                  } else {
+                    dispatchStarted = true;
+                  }
+                }
                 if (transportLease) {
                   admission.onTransportAdmitted?.(0);
                 } else {
+                  requiresTransportPermitForReacquire = true;
                   const transportStartedAt = Date.now();
                   transportLease = await input.waitForTransportSlot({
                     signal: admission.signal,
@@ -313,8 +393,18 @@ export function createRuntimeDispatchFlow<
                     Date.now() - transportStartedAt,
                   );
                 }
-                requestStarted = true;
-                return await request();
+                if (!admission.deferProviderPermitValidation) {
+                  // Preserve the cancellation boundary for a permit whose
+                  // transport wait fails before the request callback begins.
+                  requestStarted = true;
+                }
+                const result = await request(beforeProviderDispatch);
+                if (!dispatchStarted) {
+                  throw new Error(
+                    'Runtime dispatch request completed without crossing its provider-dispatch boundary.',
+                  );
+                }
+                return result;
               } finally {
                 // Feedback is response-bound, but it is not allowed to strand
                 // the capacity that made the response possible. Preserve the

package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts

@@ -47,6 +47,7 @@ export const RUNTIME_OPERATION_CAPABILITIES = {
   rate_state_acquire: 'rate.acquire',
   rate_state_release: 'rate.acquire',
   rate_state_penalize: 'rate.acquire',
+  rate_state_validate: 'rate.acquire',
   get_runtime_traffic_policy: 'rate.acquire',
   governor_budget_charge: 'budget.charge',
   resolve_secret: 'secret.resolve',
@@ -69,6 +70,13 @@ export const RUNTIME_OPERATION_CONTRACT: DeployedOperationContract =
         kind: 'equivalent-fallback',
         action: 'complete_runtime_step_receipts',
       },
+      // Permit validation cannot safely fall back to a cached ticket: an old
+      // receiver would ignore the durable provider Retry-After hold. New
+      // workers therefore fail closed with this typed compatibility result.
+      rate_state_validate: {
+        kind: 'feature-unavailable',
+        code: 'RATE_STATE_VALIDATE_UNAVAILABLE',
+      },
     },
   });
 

package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts

@@ -41,6 +41,12 @@ export type PlayRunLaunchConflictTelemetryInput = {
   artifactHash?: string | null;
   graphHash?: string | null;
   inputSha256?: string | null;
+  /** Named-intent digest of the incoming durable launch envelope. */
+  launchIdentityDigest?: string | null;
+  /** Admission-side conflict classification when available. */
+  conflictReason?: string | null;
+  /** Named launch-intent fields that differed, when the persisted envelope is available. */
+  conflictingFields?: readonly string[] | null;
   error?: unknown;
 };
 
@@ -113,6 +119,9 @@ export function buildPlayRunLaunchConflictTelemetry(
       revisionId: input.revisionId ?? null,
       artifactHash: input.artifactHash ?? null,
       graphHash: input.graphHash ?? null,
+      launchIdentityDigest: input.launchIdentityDigest ?? null,
+      conflictReason: input.conflictReason ?? null,
+      conflictingFields: input.conflictingFields ?? null,
       triage:
         level === 'error'
           ? {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -216,6 +216,13 @@ export type PlaySchedulerSubmitInput = {
   runtimeTestFaultHeader?: string | null;
   /** Request-scoped, dev-only runtime policy overrides for black-box durability tests. */
   testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
+  /**
+   * Digest of the named intent behind this launch. Runtime admission compares
+   * this optional field instead of treating transport and execution details as
+   * immutable identity. Older persisted rows omit it and keep the legacy
+   * comparison until they are gone.
+   */
+  launchIdentityDigest?: string | null;
   /** Millisecond epoch timestamp captured immediately before scheduler submit. */
   submittedAtMs?: number;
   /** Scheduler backend that owns orchestration for this run. */

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -1288,10 +1288,14 @@ export async function parkAndResolveSuspension(input: {
         boundaryId: suspension.boundaryId,
         eventKey: suspension.eventKey,
         timeoutMs: suspension.timeoutMs,
+        ...(suspension.eventExpiresAtMs !== undefined
+          ? { eventExpiresAtMs: suspension.eventExpiresAtMs }
+          : {}),
+        ...(suspension.postTimeoutPark ? { postTimeoutPark: true } : {}),
         ...(suspension.messageRef ? { messageRef: suspension.messageRef } : {}),
       },
       claimSignal: input.claimSignal,
-    });
+    }).then((result) => result.checkpoint);
   }
 
   if (suspension.kind === 'integration_event_batch') {
@@ -1299,13 +1303,25 @@ export async function parkAndResolveSuspension(input: {
     for (const boundary of [...suspension.boundaries].sort(
       (a, b) => a.timeoutMs - b.timeoutMs,
     )) {
-      resolvedCheckpoint = await resolveIntegrationEvent({
+      // An earlier event may have resolved while the provider timer elapsed.
+      // Do not enter another event wait after that provider row is due.
+      if (
+        suspension.providerWakeAtMs !== undefined &&
+        suspension.providerWakeAtMs <= Date.now() &&
+        boundary.eventExpiresAtMs !== undefined &&
+        boundary.eventExpiresAtMs > Date.now()
+      )
+        return resolvedCheckpoint;
+      const resolved = await resolveIntegrationEvent({
         ctx,
         runId,
         checkpoint: resolvedCheckpoint,
         boundary,
         claimSignal: input.claimSignal,
+        providerWakeAtMs: suspension.providerWakeAtMs,
       });
+      resolvedCheckpoint = resolved.checkpoint;
+      if (resolved.providerWake) return resolvedCheckpoint;
     }
     return resolvedCheckpoint;
   }
@@ -1382,13 +1398,16 @@ async function resolveIntegrationEvent(input: {
     boundaryId: string;
     eventKey: string;
     timeoutMs: number;
+    eventExpiresAtMs?: number;
+    postTimeoutPark?: boolean;
     messageRef?: { channel: string; ts: string };
   };
+  providerWakeAtMs?: number;
   claimSignal: (
     eventKey: string,
     boundaryId: string | null,
   ) => Promise<PostgresSchedulerSignalRecord | null>;
-}): Promise<PlayCheckpoint> {
+}): Promise<{ checkpoint: PlayCheckpoint; providerWake: boolean }> {
   const { ctx, runId, checkpoint, boundary } = input;
   // Suspensions written before `messageRef` became part of the suspension
   // contract still have the armed Slack message reference in their checkpoint.
@@ -1403,31 +1422,89 @@ async function resolveIntegrationEvent(input: {
     (checkpointBoundary?.kind === 'integration_event'
       ? checkpointBoundary.messageRef
       : undefined);
+  const checkpointOutput =
+    checkpointBoundary?.kind === 'integration_event'
+      ? checkpointBoundary.output
+      : null;
+  const postTimeoutPark =
+    boundary.postTimeoutPark === true ||
+    (checkpointOutput !== null &&
+      typeof checkpointOutput === 'object' &&
+      !Array.isArray(checkpointOutput) &&
+      (checkpointOutput as Record<string, unknown>).timed_out === true);
+  const originalEventExpiresAtMs =
+    boundary.eventExpiresAtMs ??
+    (checkpointBoundary?.kind === 'integration_event'
+      ? checkpointBoundary.eventExpiresAtMs
+      : undefined);
+  const terminalParkAnchorMs =
+    checkpointBoundary?.kind === 'integration_event'
+      ? (checkpointBoundary.completedAt ?? Date.now())
+      : Date.now();
+  const eventExpiresAtMs = postTimeoutPark
+    ? (boundary.eventExpiresAtMs ??
+      terminalParkAnchorMs + Math.max(0, boundary.timeoutMs))
+    : (originalEventExpiresAtMs ??
+      Date.now() + Math.max(0, boundary.timeoutMs));
+  // Old event checkpoints have only an Absurd step timer, not an absolute
+  // expiry. Reuse that original step instead of granting a new full timeout
+  // whenever a provider row wakes.
+  const providerWakeAtMs =
+    postTimeoutPark || originalEventExpiresAtMs === undefined
+      ? undefined
+      : input.providerWakeAtMs;
 
   // Belt-and-suspenders: an early durable signal short-circuits the wait.
-  const early = await input.claimSignal(boundary.eventKey, boundary.boundaryId);
+  const early = postTimeoutPark
+    ? null
+    : await input.claimSignal(boundary.eventKey, boundary.boundaryId);
   if (early) {
-    return absurdCheckpointWithIntegrationEvent({
-      checkpoint,
-      boundaryId: boundary.boundaryId,
-      eventKey: boundary.eventKey,
-      timeoutMs: boundary.timeoutMs,
-      ...(messageRef ? { messageRef } : {}),
-      output: absurdIntegrationEventBoundaryOutput({
+    return {
+      checkpoint: absurdCheckpointWithIntegrationEvent({
+        checkpoint,
+        boundaryId: boundary.boundaryId,
         eventKey: boundary.eventKey,
-        signalData: signalData(early),
-        messageRef,
-        timedOut: false,
+        timeoutMs: boundary.timeoutMs,
+        ...(messageRef ? { messageRef } : {}),
+        output: absurdIntegrationEventBoundaryOutput({
+          eventKey: boundary.eventKey,
+          signalData: signalData(early),
+          messageRef,
+          timedOut: false,
+        }),
       }),
-    });
+      providerWake: false,
+    };
+  }
+
+  if (
+    providerWakeAtMs !== undefined &&
+    providerWakeAtMs <= Date.now() &&
+    eventExpiresAtMs > Date.now()
+  ) {
+    return { checkpoint: absurdCheckpointBase(checkpoint), providerWake: true };
   }
 
   const eventName = absurdIntegrationEventName(runId, boundary.eventKey);
+  const eventWaitMs = Math.max(0, eventExpiresAtMs - Date.now());
+  const providerWaitMs =
+    providerWakeAtMs === undefined
+      ? Number.POSITIVE_INFINITY
+      : Math.max(0, providerWakeAtMs - Date.now());
+  const waitMs = Math.min(eventWaitMs, providerWaitMs);
   let timedOut = false;
   try {
     await ctx.awaitEvent(eventName, {
-      stepName: boundary.boundaryId,
-      timeout: Math.max(0, boundary.timeoutMs) / 1000,
+      // Absurd persists waits by step name. A provider timer must never
+      // inherit an earlier event timeout (or vice versa) on replay.
+      stepName: postTimeoutPark
+        ? `${boundary.boundaryId}:terminal-park:${terminalParkAnchorMs}`
+        : providerWakeAtMs === undefined
+          ? boundary.boundaryId
+          : `${boundary.boundaryId}:provider-wake:${providerWakeAtMs}`,
+      // Absurd floors seconds. Round up so a positive remainder never wakes
+      // before the provider's authoritative not-before time.
+      timeout: Math.ceil(waitMs / 1_000),
     });
   } catch (error) {
     if (error instanceof TimeoutError) {
@@ -1438,6 +1515,25 @@ async function resolveIntegrationEvent(input: {
     }
   }
 
+  if (postTimeoutPark) {
+    return {
+      checkpoint: absurdCheckpointWithIntegrationEvent({
+        checkpoint,
+        boundaryId: boundary.boundaryId,
+        eventKey: boundary.eventKey,
+        timeoutMs: boundary.timeoutMs,
+        ...(messageRef ? { messageRef } : {}),
+        output:
+          checkpointOutput !== null &&
+          typeof checkpointOutput === 'object' &&
+          !Array.isArray(checkpointOutput)
+            ? (checkpointOutput as Record<string, unknown>)
+            : { timed_out: true },
+      }),
+      providerWake: false,
+    };
+  }
+
   // Claim the authoritative durable signal row for the payload — the absurd
   // event carries no payload we trust. On timeout, sweep once more before
   // giving up (mirrors today's belt-and-suspenders resume).
@@ -1445,20 +1541,32 @@ async function resolveIntegrationEvent(input: {
     boundary.eventKey,
     boundary.boundaryId,
   );
+  if (
+    !claimed &&
+    timedOut &&
+    providerWakeAtMs !== undefined &&
+    providerWakeAtMs < eventExpiresAtMs &&
+    Date.now() < eventExpiresAtMs
+  ) {
+    return { checkpoint: absurdCheckpointBase(checkpoint), providerWake: true };
+  }
   const resolvedTimedOut = timedOut && !claimed;
-  return absurdCheckpointWithIntegrationEvent({
-    checkpoint,
-    boundaryId: boundary.boundaryId,
-    eventKey: boundary.eventKey,
-    timeoutMs: boundary.timeoutMs,
-    ...(messageRef ? { messageRef } : {}),
-    output: absurdIntegrationEventBoundaryOutput({
+  return {
+    checkpoint: absurdCheckpointWithIntegrationEvent({
+      checkpoint,
+      boundaryId: boundary.boundaryId,
       eventKey: boundary.eventKey,
-      signalData: signalData(claimed),
-      messageRef,
-      timedOut: resolvedTimedOut,
+      timeoutMs: boundary.timeoutMs,
+      ...(messageRef ? { messageRef } : {}),
+      output: absurdIntegrationEventBoundaryOutput({
+        eventKey: boundary.eventKey,
+        signalData: signalData(claimed),
+        messageRef,
+        timedOut: resolvedTimedOut,
+      }),
     }),
-  });
+    providerWake: false,
+  };
 }
 
 /**

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts

@@ -0,0 +1,80 @@
+/**
+ * One quota scope owns both learning evidence and the dispatch budget. Rule
+ * rows retain configured ceilings and concurrency; they must not independently
+ * learn from copies of the same provider feedback. These CTEs are shared by
+ * pure-rate and concurrency-bearing admission, in the same atomic grant SQL.
+ */
+const AIMD_RECOVERY_STEP_MS = 1_000;
+// Increase relative to the learned rate, not by a fixed RPS amount: a long
+// Retry-After can imply a very small quota, where even +1 RPS would be an
+// enormous and repeated over-probe. A 25% clean-success step still recovers a
+// healthy 1->5 RPS provider in eight observations instead of seventeen.
+const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
+
+export function acquireRateScopeSql(
+  table: string,
+  successesParameter: string,
+): string {
+  return `provider_hold_rows AS (
+    SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
+  ), provider_hold AS (
+    SELECT h.*,
+      LEAST(
+        GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, h.refill_per_sec),
+        h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * h.refill_per_sec / 1000.0
+      ) AS tokens_after_refill,
+      ${successesParameter}::int = 0
+        AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms)
+          > GREATEST(1000, 2000.0 / h.refill_per_sec) AS observation_was_idle,
+      CASE WHEN ${successesParameter}::int > 0
+          AND h.rate_observation_started_at_ms IS NOT NULL
+          AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
+            THEN LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})
+            ELSE h.rate_recovery_interval_ms END
+          AND h.retry_at_ms <= nv.now_ms
+        THEN LEAST(h.max_rps, CASE WHEN h.seen_backpressure
+          THEN h.refill_per_sec * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})
+          ELSE h.refill_per_sec * 2 END)
+        ELSE h.refill_per_sec END AS next_refill_per_sec,
+      CASE WHEN ${successesParameter}::int > 0
+          AND h.rate_observation_started_at_ms IS NOT NULL
+          AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
+            THEN LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})
+            ELSE h.rate_recovery_interval_ms END
+          AND h.retry_at_ms <= nv.now_ms
+        THEN nv.now_ms ELSE h.last_ramp_at_ms END AS next_ramp_at_ms
+    FROM provider_hold_rows h CROSS JOIN now_value nv
+  )`;
+}
+
+export function updateRateScopeSql(table: string): string {
+  return `updated_scope AS (
+    UPDATE ${table} h SET
+      tokens = ph.tokens_after_refill - (SELECT granted FROM grant_calc),
+      refilled_at_ms = nv.now_ms,
+      refill_per_sec = ph.next_refill_per_sec,
+      last_ramp_at_ms = ph.next_ramp_at_ms,
+      rate_observation_started_at_ms = CASE
+        WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN nv.now_ms
+        WHEN (SELECT granted FROM grant_calc) > 0 THEN COALESCE(ph.rate_observation_started_at_ms, nv.now_ms)
+        ELSE ph.rate_observation_started_at_ms END,
+      rate_successes = CASE
+        WHEN (SELECT granted FROM grant_calc) > 0
+          AND ph.rate_observation_started_at_ms IS NULL THEN 0
+        WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
+        ELSE ph.rate_successes END,
+      rate_throttle_ceiling_rps = CASE
+        WHEN (SELECT granted FROM grant_calc) > 0
+          AND ph.rate_observation_started_at_ms IS NULL THEN NULL
+        ELSE ph.rate_throttle_ceiling_rps END,
+      next_dispatch_at_ms = CASE WHEN (SELECT granted FROM grant_calc) > 0 THEN
+        (SELECT first_at_ms FROM schedule_calc)
+          + CEIL((SELECT granted FROM grant_calc)::numeric / (SELECT burst_size FROM schedule_calc))::bigint
+            * (SELECT spacing_ms FROM schedule_calc) * (SELECT burst_size FROM schedule_calc)
+        ELSE ph.next_dispatch_at_ms END
+    FROM provider_hold ph CROSS JOIN now_value nv
+    WHERE h.bucket_id = $1
+    RETURNING h.bucket_id, h.rate_observation_started_at_ms, h.rate_successes,
+      h.rate_recovery_interval_ms, h.seen_backpressure
+  )`;
+}