deepline 0.3.184 → 0.3.186

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -157,6 +157,11 @@ import {
 import { LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS } from '../governor/policy';
 import { requireRuntimeAuthorityDescriptor } from '../execution-capabilities';
 import { stringifyPostgresJson } from '../postgres-json';
+import {
+  buildPlayLaunchIdentity,
+  withPlayLaunchIdentityDigest,
+} from '../launch-identity';
+import { PlayRunLaunchConflictError } from '../play-runs/admission/contracts';
 import { mintPlayRunAttemptRuntimeToken } from '../runtime-authority';
 import { withActiveSpan } from '../../observability/tracing';
 import type {
@@ -269,7 +274,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
 // deployment retries do not re-enter the old cross-table DDL transaction.
 // v45 adds the bucket-leading expiration index used by rate-reservation
 // cleanup. Shared schemas build it concurrently through the deploy migrator.
-export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 45;
+// v46 adds a bounded provider-scope Retry-After hold with shared adaptive-rate
+// state. The scope owns learning and dispatch across runs and tool rules;
+// existing deadline rows expand additively and initialize on their first seed.
+export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 46;
 const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
   'scheduler_schema_migrations';
 export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -1065,6 +1073,8 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
   cronScheduleSchema?: boolean;
   /** v43 added per-run fair-share tracking for rate buckets; v42 ignores it. */
   rateRequesterFairShare?: boolean;
+  /** v45 added durable Retry-After holds shared across rules and runs. */
+  rateScopeRetryAfter?: boolean;
 }>;
 
 export async function hasCurrentPostgresSchedulerSchemaShape(
@@ -1371,6 +1381,31 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
             )
           )
         )
+        AND (
+          $36::boolean = false
+          OR (
+            to_regclass($37) IS NOT NULL
+            AND (
+              SELECT count(DISTINCT column_name)
+              FROM information_schema.columns
+              WHERE table_schema = $1
+                AND table_name = 'rate_scope_holds'
+                AND column_name IN (
+                  'bucket_id', 'retry_at_ms', 'refill_per_sec', 'capacity', 'max_rps',
+                  'tokens', 'refilled_at_ms', 'next_dispatch_at_ms', 'seen_backpressure',
+                  'cool_until_ms', 'last_ramp_at_ms', 'rate_observation_started_at_ms',
+                  'rate_successes', 'rate_recovery_interval_ms',
+                  'rate_throttle_ceiling_rps'
+                )
+            ) = 15
+            AND (
+              SELECT count(DISTINCT column_name)
+              FROM information_schema.columns
+              WHERE table_schema = $1 AND table_name = 'rate_bucket'
+                AND column_name IN ('rate_observation_started_at_ms', 'rate_successes', 'rate_recovery_interval_ms')
+            ) = 3
+          )
+        )
         AND (
           $32::boolean = false
           OR (
@@ -1695,6 +1730,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
       `${quotedSchema}.${quoteIdent(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE)}`,
       requirements.rateRequesterFairShare !== false,
       `${quotedSchema}.${quoteIdent('rate_requesters')}`,
+      requirements.rateScopeRetryAfter !== false,
+      `${quotedSchema}.${quoteIdent('rate_scope_holds')}`,
     ],
   );
   if (currentShape.rows[0]?.ready !== true) return false;
@@ -2166,6 +2203,99 @@ async function ensurePostgresSchedulerCronScheduleSchema(
   `);
 }
 
+/**
+ * Install the launch-admission functions on every bootstrap path.
+ *
+ * These are function-body compatibility changes, not schema DDL. An already
+ * current scheduler schema takes the fast path below, so keeping these only in
+ * the full bootstrap leaves that estate with the old whole-envelope equality.
+ */
+async function ensurePostgresSchedulerLaunchAdmissionFunctions(
+  client: PostgresSchedulerQueryClient,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<void> {
+  const quotedSchema = quoteIdent(schemaName(options));
+  await client.query(`
+    CREATE OR REPLACE FUNCTION ${quotedSchema}.runtime_launch_admission_identity(
+      launch_json jsonb
+    ) RETURNS jsonb
+    LANGUAGE sql
+    IMMUTABLE
+    STRICT
+    PARALLEL SAFE
+    AS $$
+      SELECT jsonb_set(
+        jsonb_set(
+          launch_json - ARRAY[
+            'launchIdentityDigest',
+            'executorToken',
+            'inputFileId',
+            'preloadedDbSessionUnwrapKey',
+            'preloadedDbSessions',
+            'submittedAtMs',
+            'runtimeDeployVersion',
+            'absurdReleaseId',
+            'absurdReleaseEnvironment',
+            'coordinatorUrl',
+            'coordinatorInternalToken',
+            'baseUrl',
+            'runtimeApiBaseUrl',
+            'vercelProtectionBypassToken'
+          ]::text[],
+          '{maxConcurrentExternalCalls}',
+          coalesce(
+            launch_json->'maxConcurrentExternalCalls',
+            to_jsonb(${LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS}::integer)
+          ),
+          true
+        ),
+        '{maxConcurrentRows}',
+        coalesce(launch_json->'maxConcurrentRows', 'null'::jsonb),
+        true
+      )
+    $$
+  `);
+  await client.query(`
+    CREATE OR REPLACE FUNCTION ${quotedSchema}.runtime_launch_admission_matches(
+      existing_launch_json jsonb,
+      incoming_launch_json jsonb
+    ) RETURNS boolean
+    LANGUAGE sql
+    IMMUTABLE
+    STRICT
+    PARALLEL SAFE
+    AS $$
+      SELECT CASE
+        WHEN nullif(btrim(coalesce(existing_launch_json->>'launchIdentityDigest', '')), '') IS NOT NULL
+          AND nullif(btrim(coalesce(incoming_launch_json->>'launchIdentityDigest', '')), '') IS NOT NULL
+          THEN btrim(existing_launch_json->>'launchIdentityDigest')
+             = btrim(incoming_launch_json->>'launchIdentityDigest')
+            AND (
+              NOT (
+                existing_launch_json ? 'runtimeSandboxPlacementPolicyId'
+                AND incoming_launch_json ? 'runtimeSandboxPlacementPolicyId'
+              )
+              OR existing_launch_json->'runtimeSandboxPlacementPolicyId'
+                 IS NOT DISTINCT FROM incoming_launch_json->'runtimeSandboxPlacementPolicyId'
+            )
+        WHEN existing_launch_json ? 'runtimeSandboxPlacementPolicyId'
+          AND incoming_launch_json ? 'runtimeSandboxPlacementPolicyId'
+          THEN ${quotedSchema}.runtime_launch_admission_identity(
+            existing_launch_json - 'launchIdentityDigest'
+          ) = ${quotedSchema}.runtime_launch_admission_identity(
+            incoming_launch_json - 'launchIdentityDigest'
+          )
+        ELSE ${quotedSchema}.runtime_launch_admission_identity(
+          existing_launch_json - 'launchIdentityDigest'
+        ) - ARRAY['runtimeSandboxPlacementPolicyId', 'launchIdentityDigest']::text[] =
+          ${quotedSchema}.runtime_launch_admission_identity(
+            incoming_launch_json - 'launchIdentityDigest'
+          ) - ARRAY['runtimeSandboxPlacementPolicyId', 'launchIdentityDigest']::text[]
+      END
+    $$
+  `);
+}
+
 export async function ensurePostgresSchedulerSchema(
   client: PostgresSchedulerQueryClient,
   options?: Pick<PostgresSchedulerOptions, 'schema'> & {
@@ -2300,6 +2430,7 @@ export async function ensurePostgresSchedulerSchema(
     const hasLegacyAdmissionRows =
       legacyAdmissionRows?.rows[0]?.present === true;
     if (schemaCoreIsCurrent) {
+      await ensurePostgresSchedulerLaunchAdmissionFunctions(client, options);
       if (!schemaIsCurrent) {
         await ensurePostgresSchedulerTerminalComputeBillingOutboxTrigger(
           client,
@@ -3671,6 +3802,7 @@ export async function ensurePostgresSchedulerSchema(
       [POSTGRES_SCHEDULER_SCHEMA_VERSION],
     );
     await ensurePostgresSchedulerCronScheduleSchema(client, options);
+    await ensurePostgresSchedulerLaunchAdmissionFunctions(client, options);
     if (hadOutbox.rows[0]?.present !== true) {
       await client.query(
         `
@@ -4377,6 +4509,45 @@ export async function submitPostgresAbsurdPlayRun(
   };
 }
 
+function launchIdentityConflictFields(
+  existingLaunchJson: unknown,
+  incomingLaunch: PlaySchedulerSubmitInput,
+): string[] {
+  if (
+    !existingLaunchJson ||
+    typeof existingLaunchJson !== 'object' ||
+    Array.isArray(existingLaunchJson)
+  ) {
+    return [];
+  }
+  const existingIdentity = buildPlayLaunchIdentity(
+    existingLaunchJson as Record<string, unknown>,
+  );
+  const incomingIdentity = buildPlayLaunchIdentity(incomingLaunch);
+  const fields = Object.keys(incomingIdentity).filter(
+    (field) =>
+      stableStringify(existingIdentity[field]) !==
+      stableStringify(incomingIdentity[field]),
+  );
+  if (
+    Object.prototype.hasOwnProperty.call(
+      existingLaunchJson,
+      'runtimeSandboxPlacementPolicyId',
+    ) &&
+    Object.prototype.hasOwnProperty.call(
+      incomingLaunch,
+      'runtimeSandboxPlacementPolicyId',
+    ) &&
+    stableStringify(
+      (existingLaunchJson as Record<string, unknown>)
+        .runtimeSandboxPlacementPolicyId,
+    ) !== stableStringify(incomingLaunch.runtimeSandboxPlacementPolicyId)
+  ) {
+    fields.push('runtimeSandboxPlacementPolicyId');
+  }
+  return fields.sort();
+}
+
 async function submitPostgresSchedulerRunTransaction(
   client: PostgresSchedulerQueryClient,
   input: PlaySchedulerSubmitInput,
@@ -4415,7 +4586,10 @@ async function submitPostgresSchedulerRunTransaction(
     existingOnly: requestedExistingOnly,
     ...durableInput
   } = input;
-  input = durableInput;
+  // Every scheduler writer converges through this seam, including the cron
+  // successor path and Workpool retries. Stamp after ephemeral admission proof
+  // fields are removed so they can never change the durable launch identity.
+  input = withPlayLaunchIdentityDigest(durableInput);
   const existingOnlySql = requestedExistingOnly === true ? 'true' : 'false';
   const releaseId = input.absurdReleaseId?.trim() || null;
   const legacyPreviewRelease =
@@ -4499,6 +4673,7 @@ async function submitPostgresSchedulerRunTransaction(
     run_created: boolean;
     task_created: boolean | null;
     task_recovery_required: boolean;
+    existing_launch_json: unknown | null;
   }>(
     `
       WITH run_lock AS MATERIALIZED (
@@ -4546,6 +4721,7 @@ async function submitPostgresSchedulerRunTransaction(
             coalesce(runs.admission_json, runs.launch_json),
             $8::jsonb
           ) AS matches_launch,
+          coalesce(runs.admission_json, runs.launch_json) AS launch_json,
           runs.absurd_task_id,
           runs.status
         FROM admission_lock
@@ -4752,6 +4928,7 @@ async function submitPostgresSchedulerRunTransaction(
                    AND existing_run.absurd_task_id IS NULL
                )
              ) AS task_recovery_required,
+             (SELECT launch_json FROM existing_run) AS existing_launch_json,
              CASE WHEN EXISTS (SELECT 1 FROM run_insert)
                          OR EXISTS (SELECT 1 FROM run_recovery)
                THEN pg_notify($10, $11)
@@ -4808,9 +4985,13 @@ async function submitPostgresSchedulerRunTransaction(
     );
   }
   if (row?.accepted === false && row.existing_run && !row.matching_existing) {
-    throw new Error(
-      `Play run "${input.runId}" already exists with a different launch.`,
-    );
+    throw new PlayRunLaunchConflictError(input.runId, {
+      conflictReason: 'launch_identity_mismatch',
+      conflictingFields: launchIdentityConflictFields(
+        row.existing_launch_json,
+        input,
+      ),
+    });
   }
   if (row?.accepted === false && !row?.existing_run && requestedExistingOnly) {
     throw new Error(
@@ -5112,6 +5293,14 @@ export class PostgresSchedulerCronScheduleConfigurationError extends Error {
   }
 }
 
+/** A deterministic Run identity conflict cannot be repaired by retrying the same fire. */
+export class PostgresSchedulerCronLaunchConflictError extends PostgresSchedulerCronScheduleConfigurationError {
+  constructor(message: string) {
+    super(message);
+    this.name = 'PostgresSchedulerCronLaunchConflictError';
+  }
+}
+
 /**
  * The timer stores the facts needed to reconstruct a pinned admission, not a
  * previously prepared launch. Executor tokens, preloaded session keys,
@@ -5456,7 +5645,7 @@ function cronOccurrenceLaunch(input: {
       `Cron schedule "${input.schedule.scheduleId}" did not receive a fresh authority for its pinned occurrence.`,
     );
   }
-  return {
+  return withPlayLaunchIdentityDigest({
     ...input.launch,
     schedulerSchema:
       input.schedulerSchema ?? input.launch.schedulerSchema ?? null,
@@ -5469,7 +5658,7 @@ function cronOccurrenceLaunch(input: {
     ),
     triggerSource: 'cron',
     requiresRunCreatedProjection: true,
-  };
+  });
 }
 
 export type FirePostgresSchedulerCronScheduleInput = {
@@ -5492,14 +5681,26 @@ export type FirePostgresSchedulerCronScheduleInput = {
  * CTE inserts the immutable Run, its play-run task, run.created fact, next
  * cron-fire task, and next cursor. A task replay after commit sees a changed
  * cursor and is a harmless `superseded` no-op.
+ *
+ * A re-fire of the same occurrence whose deterministic Run already exists is
+ * an idempotent re-fire, not a no-op. The cursor advances when the existing
+ * launch identity matches, even if this statement did not create the Run.
+ * A genuine identity mismatch is returned as `stalled` so it cannot silently
+ * freeze the schedule behind an indistinguishable replay.
  */
 export async function firePostgresSchedulerCronSchedule(
   client: PostgresSchedulerQueryClient,
   input: FirePostgresSchedulerCronScheduleInput,
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<
-  | { status: 'admitted'; runId: string; successorDueAtMs: number }
+  | {
+      status: 'admitted';
+      runId: string;
+      successorDueAtMs: number;
+      reused: boolean;
+    }
   | { status: 'superseded' }
+  | { status: 'stalled'; runId: string; reason: string }
 > {
   assertPostgresSchedulerCronScheduleInput(input);
   const schedule = await readPostgresSchedulerCronSchedule(
@@ -5566,7 +5767,11 @@ export async function firePostgresSchedulerCronSchedule(
     orgId: launch.orgId,
   });
   const result = await client.query<{
-    admitted: boolean;
+    advanced: boolean;
+    run_created: boolean;
+    schedule_current: boolean;
+    existing_run: boolean;
+    matching_existing: boolean;
     release_accepting: boolean;
   }>(
     `
@@ -5591,16 +5796,28 @@ export async function firePostgresSchedulerCronSchedule(
           AND release_gate.accepting
         FOR UPDATE
       ), existing_run AS MATERIALIZED (
-        SELECT runs.run_id
+        SELECT runs.run_id,
+               ${quoteIdent(schemaName(options))}.runtime_launch_admission_matches(
+                 coalesce(runs.admission_json, runs.launch_json),
+                 $6::jsonb
+               ) AS matches_launch
         FROM matching_schedule
         JOIN ${tableName('runs', options)} AS runs ON runs.run_id = $5::text
+      ), occurrence AS MATERIALIZED (
+        SELECT EXISTS (SELECT 1 FROM matching_schedule) AS schedule_current,
+               EXISTS (SELECT 1 FROM existing_run) AS existing_run,
+               coalesce(
+                 (SELECT matches_launch FROM existing_run),
+                 false
+               ) AS matching_existing
       ), play_task_request AS (
         SELECT $7::text AS queue_name,
                $8::text AS task_name,
                $9::jsonb AS params,
                $10::jsonb AS options
-        WHERE EXISTS (SELECT 1 FROM matching_schedule)
-          AND NOT EXISTS (SELECT 1 FROM existing_run)
+        FROM occurrence
+        WHERE occurrence.schedule_current
+          AND NOT occurrence.existing_run
       ), play_task_spawn AS MATERIALIZED (
         SELECT spawned.task_id::text AS task_id
         FROM play_task_request
@@ -5636,15 +5853,25 @@ export async function firePostgresSchedulerCronSchedule(
                  'coordinatorInternalToken',
                  'vercelProtectionBypassToken'
                ]::text[]
-        WHERE EXISTS (SELECT 1 FROM matching_schedule)
-          AND NOT EXISTS (SELECT 1 FROM existing_run)
+        FROM occurrence
+        WHERE occurrence.schedule_current
+          AND NOT occurrence.existing_run
         RETURNING run_id
+      ), occurrence_admitted AS MATERIALIZED (
+        SELECT true AS admitted
+        FROM occurrence
+        WHERE EXISTS (SELECT 1 FROM run_insert)
+           OR (
+             occurrence.schedule_current
+             AND occurrence.existing_run
+             AND occurrence.matching_existing
+           )
       ), successor_task_request AS (
         SELECT $11::text AS queue_name,
                $12::text AS task_name,
                $13::jsonb AS params,
                $14::jsonb AS options
-        WHERE EXISTS (SELECT 1 FROM run_insert)
+        WHERE EXISTS (SELECT 1 FROM occurrence_admitted)
       ), successor_task_spawn AS MATERIALIZED (
         SELECT spawned.task_id::text AS task_id
         FROM successor_task_request
@@ -5718,10 +5945,14 @@ export async function firePostgresSchedulerCronSchedule(
           AND schedule.state = 'active'
           AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
           AND schedule.current_fire_generation = $15::bigint
-          AND EXISTS (SELECT 1 FROM run_insert)
+          AND EXISTS (SELECT 1 FROM occurrence_admitted)
         RETURNING schedule_id
       )
-      SELECT EXISTS (SELECT 1 FROM schedule_advance) AS admitted,
+      SELECT EXISTS (SELECT 1 FROM schedule_advance) AS advanced,
+             EXISTS (SELECT 1 FROM run_insert) AS run_created,
+             (SELECT schedule_current FROM occurrence) AS schedule_current,
+             (SELECT existing_run FROM occurrence) AS existing_run,
+             (SELECT matching_existing FROM occurrence) AS matching_existing,
              coalesce(
                (SELECT accepting
                   FROM ${tableName('runtime_release_admission', options)}
@@ -5748,19 +5979,34 @@ export async function firePostgresSchedulerCronSchedule(
       releaseAdmissionKey,
     ],
   );
-  if (result.rows[0]?.release_accepting !== true) {
+  const fired = result.rows[0];
+  if (fired?.release_accepting !== true) {
     throw new Error(
       `Cron schedule "${schedule.scheduleId}" cannot fire because Absurd release "${releaseAdmissionKey}" is not accepting admissions.`,
     );
   }
-  if (result.rows[0]?.admitted !== true) {
-    return { status: 'superseded' };
+  if (fired.advanced === true) {
+    return {
+      status: 'admitted',
+      runId: launch.runId,
+      successorDueAtMs,
+      reused: fired.run_created !== true,
+    };
   }
-  return {
-    status: 'admitted',
-    runId: launch.runId,
-    successorDueAtMs,
-  };
+  if (
+    fired.schedule_current === true &&
+    fired.existing_run === true &&
+    fired.matching_existing !== true
+  ) {
+    return {
+      status: 'stalled',
+      runId: launch.runId,
+      reason:
+        `Cron schedule "${schedule.scheduleId}" generation ${schedule.generation} cannot advance past ` +
+        `occurrence ${schedule.dueAtMs}: Run "${launch.runId}" already exists with a different launch identity.`,
+    };
+  }
+  return { status: 'superseded' };
 }
 
 /** Fence a malformed/invalid timer so it cannot repeatedly consume workers. */

package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts

@@ -14,6 +14,9 @@ export type PlayExecutionSuspension =
       boundaryId: string;
       eventKey: string;
       timeoutMs: number;
+      eventExpiresAtMs?: number;
+      /** A terminal timeout's deliberate long park, not its original event wait. */
+      postTimeoutPark?: boolean;
       messageRef?: {
         channel: string;
         ts: string;
@@ -21,10 +24,13 @@ export type PlayExecutionSuspension =
     }
   | {
       kind: 'integration_event_batch';
+      /** Earliest pending provider row; absence preserves older event-only waits. */
+      providerWakeAtMs?: number;
       boundaries: Array<{
         boundaryId: string;
         eventKey: string;
         timeoutMs: number;
+        eventExpiresAtMs?: number;
         messageRef?: {
           channel: string;
           ts: string;
@@ -123,6 +129,7 @@ export class PlayRowExecutionSuspendedError extends Error {
     boundaryId: string;
     eventKey: string;
     timeoutMs: number;
+    eventExpiresAtMs?: number;
     messageRef?: {
       channel: string;
       ts: string;
@@ -133,6 +140,7 @@ export class PlayRowExecutionSuspendedError extends Error {
     boundaryId: string;
     eventKey: string;
     timeoutMs: number;
+    eventExpiresAtMs?: number;
     messageRef?: {
       channel: string;
       ts: string;
@@ -146,6 +154,19 @@ export class PlayRowExecutionSuspendedError extends Error {
   }
 }
 
+/** A provider wait is a control handoff, never a failed cell or receipt. */
+export class PlayRowProviderBackpressureError extends Error {
+  constructor(
+    readonly provider: string,
+    readonly retryAtMs: number,
+  ) {
+    super(
+      `Play row is waiting for ${provider} provider backpressure to clear.`,
+    );
+    this.name = 'PlayRowProviderBackpressureError';
+  }
+}
+
 export function isPlayRowExecutionSuspendedError(
   value: unknown,
 ): value is PlayRowExecutionSuspendedError {