deepline 0.3.113 → 0.3.115

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +3 -1
  2. package/dist/bundling-sources/sdk/src/compat.ts +167 -11
  3. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  4. package/dist/bundling-sources/sdk/src/skills-version.ts +27 -1
  5. package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +6 -2
  6. package/dist/bundling-sources/shared_libs/observability/dlq.ts +942 -0
  7. package/dist/bundling-sources/shared_libs/observability/queue-health.ts +361 -6
  8. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +24 -24
  9. package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +2 -0
  10. package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +29 -1
  11. package/dist/bundling-sources/shared_libs/play-runtime/admission-transport-policy.ts +91 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +2 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/direct-admission-deadline.ts +34 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +325 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/play-admission-outcome.ts +134 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +13 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +4 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +13 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/play-webhook-admission-probe.ts +32 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/play-webhook-identity.ts +28 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-cron-fire-task.ts +100 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +313 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +111 -3
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +16 -0
  27. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +67 -8
  28. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1734 -147
  29. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +74 -13
  30. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts +245 -81
  31. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +79 -15
  32. package/dist/bundling-sources/shared_libs/play-runtime/webhook-receipt-retry.ts +21 -21
  33. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +4 -1
  34. package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +64 -0
  35. package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +1 -1
  36. package/dist/bundling-sources/shared_libs/product-notifications/events.ts +20 -1
  37. package/dist/bundling-sources/shared_libs/security/receipt-admission-token.ts +224 -0
  38. package/dist/cli/index.js +574 -114
  39. package/dist/cli/index.mjs +592 -133
  40. package/dist/index.d.mts +54 -1
  41. package/dist/index.d.ts +54 -1
  42. package/dist/index.js +8 -3
  43. package/dist/index.mjs +8 -3
  44. package/dist/install-integrity.json +8 -0
  45. package/dist/plays/bundle-play-file.mjs +1 -1
  46. package/dist/release.d.mts +1 -1
  47. package/dist/release.d.ts +1 -1
  48. package/dist/release.js +1 -1
  49. package/dist/release.mjs +1 -1
  50. package/package.json +1 -1
@@ -159,6 +159,11 @@ import {
159
159
  } from './scheduler-launch-read-observability';
160
160
  import { emitPlayTelemetry } from '../runtime-telemetry';
161
161
  import { buildAbsurdPlayRunTask } from './absurd-play-run-task';
162
+ import {
163
+ buildAbsurdCronFireTask,
164
+ buildCronFireRunIdentity,
165
+ } from './absurd-cron-fire-task';
166
+ import { advanceWorkflowCronSchedule } from '@deepline/workflows/scheduler';
162
167
 
163
168
  type QueryResult<
164
169
  Row extends Record<string, unknown> = Record<string, unknown>,
@@ -212,16 +217,24 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
212
217
  * interrupted deployment cannot claim a newer version without its required
213
218
  * objects.
214
219
  */
215
- // v36 adds a blocked-outbox oldest-first index. It bounds the operator health
216
- // probe even when permanently failed projection facts accumulate, while the
217
- // existing dead-letter index remains optimized for repair recency.
218
- export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 36;
220
+ // v38 adds bounded health indexes for the runtime-owned outbox, sandbox
221
+ // cleanup, and compute-billing queues. The indexes are additive: v37 cron
222
+ // scheduling, admission, and outbox projection remain valid while a deploy
223
+ // expands the shared schema, then the marker records the completed health
224
+ // capability for deployment tooling.
225
+ export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 38;
219
226
  const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
220
227
  'scheduler_schema_migrations';
221
228
  export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
222
229
  'runtime_scheduler_runs_open_recovery_idx';
223
230
  const POSTGRES_SCHEDULER_DEPENDENCY_CIRCUITS_TABLE =
224
231
  'runtime_dependency_circuits';
232
+ /**
233
+ * Runtime-owned cursor for a scheduled Play. Convex remains the authored
234
+ * control-plane projection; this row is the execution owner once a binding is
235
+ * cut over, so a Run and its successor can commit together.
236
+ */
237
+ const POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE = 'cron_schedules';
225
238
  const RECOVERY_CIRCUIT_OPEN_MS =
226
239
  RUNTIME_RELIABILITY_POLICY.recovery.circuitOpenMs;
227
240
  const RECOVERY_CIRCUIT_PROBE_LEASE_MS =
@@ -992,6 +1005,7 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
992
1005
  admissionPriorityIndex?: boolean;
993
1006
  terminalComputeBillingOutboxTrigger?: boolean;
994
1007
  outboxWakeTrigger?: boolean;
1008
+ cronScheduleSchema?: boolean;
995
1009
  }>;
996
1010
 
997
1011
  export async function hasCurrentPostgresSchedulerSchemaShape(
@@ -1240,6 +1254,49 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1240
1254
  AND to_regclass($20) IS NOT NULL
1241
1255
  AND to_regclass($21) IS NOT NULL
1242
1256
  AND to_regclass($22) IS NOT NULL
1257
+ AND (
1258
+ $25::boolean = false
1259
+ OR (
1260
+ to_regclass($26) IS NOT NULL
1261
+ AND (
1262
+ SELECT count(DISTINCT column_name)
1263
+ FROM information_schema.columns
1264
+ WHERE table_schema = $1
1265
+ AND table_name = '${POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE}'
1266
+ AND column_name IN (
1267
+ 'schedule_id',
1268
+ 'generation',
1269
+ 'state',
1270
+ 'cron',
1271
+ 'timezone',
1272
+ 'current_due_at',
1273
+ 'current_fire_task_id',
1274
+ 'current_fire_generation',
1275
+ 'current_fire_queue_name',
1276
+ 'last_fired_at',
1277
+ 'block_reason',
1278
+ 'cancellation_reason',
1279
+ 'source_pin_json',
1280
+ 'launch_json',
1281
+ 'created_at',
1282
+ 'updated_at'
1283
+ )
1284
+ ) = 16
1285
+ AND EXISTS (
1286
+ SELECT 1
1287
+ FROM pg_index AS index_row
1288
+ JOIN pg_class AS index_class
1289
+ ON index_class.oid = index_row.indexrelid
1290
+ JOIN pg_namespace AS index_namespace
1291
+ ON index_namespace.oid = index_class.relnamespace
1292
+ WHERE index_namespace.nspname = $1
1293
+ AND index_class.relname = 'runtime_scheduler_cron_schedules_due_idx'
1294
+ AND index_row.indrelid = to_regclass($26)
1295
+ AND index_row.indisready
1296
+ AND index_row.indisvalid
1297
+ )
1298
+ )
1299
+ )
1243
1300
  -- Canonical Integration Operation outcomes are additive to the
1244
1301
  -- transport receipt. Old byte-only rows remain readable by the
1245
1302
  -- gateway, but can never be mistaken for a canonical operation.
@@ -1483,6 +1540,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1483
1540
  `${quotedSchema}.${quoteIdent('outbox_repair_operations')}`,
1484
1541
  requirements.outboxWakeTrigger !== false,
1485
1542
  requirements.admissionPriorityIndex !== false,
1543
+ requirements.cronScheduleSchema !== false,
1544
+ `${quotedSchema}.${quoteIdent(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE)}`,
1486
1545
  ],
1487
1546
  );
1488
1547
  if (currentShape.rows[0]?.ready !== true) return false;
@@ -1509,6 +1568,87 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1509
1568
  AND index_row.indrelid = to_regclass($3)
1510
1569
  AND index_row.indisready AND index_row.indisvalid
1511
1570
  )
1571
+ AND EXISTS (
1572
+ SELECT 1 FROM pg_index AS index_row
1573
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1574
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1575
+ WHERE index_namespace.nspname = $1
1576
+ AND index_class.relname = 'runtime_scheduler_outbox_ready_health_idx'
1577
+ AND index_row.indrelid = to_regclass($3)
1578
+ AND index_row.indisready AND index_row.indisvalid
1579
+ )
1580
+ AND EXISTS (
1581
+ SELECT 1 FROM pg_index AS index_row
1582
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1583
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1584
+ WHERE index_namespace.nspname = $1
1585
+ AND index_class.relname = 'runtime_scheduler_outbox_dead_health_idx'
1586
+ AND index_row.indrelid = to_regclass($3)
1587
+ AND index_row.indisready AND index_row.indisvalid
1588
+ )
1589
+ AND EXISTS (
1590
+ SELECT 1 FROM pg_index AS index_row
1591
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1592
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1593
+ WHERE index_namespace.nspname = $1
1594
+ AND index_class.relname = 'runtime_scheduler_outbox_lease_health_idx'
1595
+ AND index_row.indrelid = to_regclass($3)
1596
+ AND index_row.indisready AND index_row.indisvalid
1597
+ )
1598
+ AND EXISTS (
1599
+ SELECT 1 FROM pg_index AS index_row
1600
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1601
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1602
+ WHERE index_namespace.nspname = $1
1603
+ AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_ready_health_idx'
1604
+ AND index_row.indrelid = to_regclass($7)
1605
+ AND index_row.indisready AND index_row.indisvalid
1606
+ )
1607
+ AND EXISTS (
1608
+ SELECT 1 FROM pg_index AS index_row
1609
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1610
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1611
+ WHERE index_namespace.nspname = $1
1612
+ AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_lease_health_idx'
1613
+ AND index_row.indrelid = to_regclass($7)
1614
+ AND index_row.indisready AND index_row.indisvalid
1615
+ )
1616
+ AND EXISTS (
1617
+ SELECT 1 FROM pg_index AS index_row
1618
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1619
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1620
+ WHERE index_namespace.nspname = $1
1621
+ AND index_class.relname = 'runtime_scheduler_compute_billing_ready_health_idx'
1622
+ AND index_row.indrelid = to_regclass($8)
1623
+ AND index_row.indisready AND index_row.indisvalid
1624
+ )
1625
+ AND EXISTS (
1626
+ SELECT 1 FROM pg_index AS index_row
1627
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1628
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1629
+ WHERE index_namespace.nspname = $1
1630
+ AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx'
1631
+ AND index_row.indrelid = to_regclass($7)
1632
+ AND index_row.indisready AND index_row.indisvalid
1633
+ )
1634
+ AND EXISTS (
1635
+ SELECT 1 FROM pg_index AS index_row
1636
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1637
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1638
+ WHERE index_namespace.nspname = $1
1639
+ AND index_class.relname = 'runtime_scheduler_compute_billing_lease_health_idx'
1640
+ AND index_row.indrelid = to_regclass($8)
1641
+ AND index_row.indisready AND index_row.indisvalid
1642
+ )
1643
+ AND EXISTS (
1644
+ SELECT 1 FROM pg_index AS index_row
1645
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1646
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1647
+ WHERE index_namespace.nspname = $1
1648
+ AND index_class.relname = 'runtime_scheduler_compute_billing_blocked_health_idx'
1649
+ AND index_row.indrelid = to_regclass($8)
1650
+ AND index_row.indisready AND index_row.indisvalid
1651
+ )
1512
1652
  AND (
1513
1653
  $5::boolean = false
1514
1654
  OR EXISTS (
@@ -1550,6 +1690,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1550
1690
  `${quotedSchema}.${quoteIdent('runs')}`,
1551
1691
  requirements.deliveredOutboxRetentionIndex !== false,
1552
1692
  requirements.blockedOutboxIndex !== false,
1693
+ `${quotedSchema}.${quoteIdent('sandbox_cleanup_jobs_v2')}`,
1694
+ `${quotedSchema}.${quoteIdent('compute_billing_jobs_v2')}`,
1553
1695
  ],
1554
1696
  );
1555
1697
  return runtimeAdminIndexes.rows[0]?.ready === true;
@@ -1712,6 +1854,68 @@ async function ensurePostgresSchedulerOutboxWakeTrigger(
1712
1854
  `);
1713
1855
  }
1714
1856
 
1857
+ /**
1858
+ * Additive, independent schema for the cron successor owner. It intentionally
1859
+ * does not participate in the global scheduler schema marker: older app and
1860
+ * worker builds neither read nor write this table, and making ordinary Run
1861
+ * admission wait on this rollout would turn an additive timer deployment into
1862
+ * an availability dependency.
1863
+ */
1864
+ async function ensurePostgresSchedulerCronScheduleSchema(
1865
+ client: PostgresSchedulerQueryClient,
1866
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
1867
+ ): Promise<void> {
1868
+ await client.query(`
1869
+ CREATE TABLE IF NOT EXISTS ${tableName(
1870
+ POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE,
1871
+ options,
1872
+ )} (
1873
+ schedule_id text PRIMARY KEY,
1874
+ generation bigint NOT NULL CHECK (generation > 0),
1875
+ state text NOT NULL CHECK (state IN ('active', 'blocked', 'cancelled')),
1876
+ cron text NOT NULL,
1877
+ timezone text,
1878
+ current_due_at timestamptz NOT NULL,
1879
+ current_fire_task_id text,
1880
+ current_fire_generation bigint NOT NULL DEFAULT 1 CHECK (current_fire_generation > 0),
1881
+ current_fire_queue_name text,
1882
+ last_fired_at timestamptz,
1883
+ block_reason text,
1884
+ cancellation_reason text,
1885
+ source_pin_json jsonb,
1886
+ launch_json jsonb NOT NULL,
1887
+ created_at timestamptz NOT NULL DEFAULT now(),
1888
+ updated_at timestamptz NOT NULL DEFAULT now()
1889
+ )
1890
+ `);
1891
+ await client.query(`
1892
+ ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
1893
+ ADD COLUMN IF NOT EXISTS block_reason text
1894
+ `);
1895
+ await client.query(`
1896
+ ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
1897
+ ADD COLUMN IF NOT EXISTS current_fire_generation bigint NOT NULL DEFAULT 1
1898
+ `);
1899
+ await client.query(`
1900
+ ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
1901
+ ADD COLUMN IF NOT EXISTS current_fire_queue_name text
1902
+ `);
1903
+ await client.query(`
1904
+ ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
1905
+ ADD COLUMN IF NOT EXISTS cancellation_reason text
1906
+ `);
1907
+ await client.query(`
1908
+ ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
1909
+ ADD COLUMN IF NOT EXISTS source_pin_json jsonb
1910
+ `);
1911
+ await client.query(`
1912
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_cron_schedules_due_idx
1913
+ ON ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
1914
+ (current_due_at, schedule_id)
1915
+ WHERE state = 'active'
1916
+ `);
1917
+ }
1918
+
1715
1919
  export async function ensurePostgresSchedulerSchema(
1716
1920
  client: PostgresSchedulerQueryClient,
1717
1921
  options?: Pick<PostgresSchedulerOptions, 'schema'> & {
@@ -1817,7 +2021,7 @@ export async function ensurePostgresSchedulerSchema(
1817
2021
  // trigger as a narrow migration: replaying historical schema DDL in one
1818
2022
  // transaction against a live worker fleet can invert table-lock order with
1819
2023
  // terminal writers.
1820
- const schemaIsCurrentWithoutAdditiveOutboxTriggers =
2024
+ const schemaIsCurrentWithoutAdditiveFastPathObjects =
1821
2025
  !schemaIsCurrent &&
1822
2026
  (await hasCurrentPostgresSchedulerSchemaShape(client, options, {
1823
2027
  openRecoveryIndex: !options?.deferOpenRecoveryIndex,
@@ -1825,17 +2029,10 @@ export async function ensurePostgresSchedulerSchema(
1825
2029
  admissionPriorityIndex: !options?.deferRuntimeAdminIndexes,
1826
2030
  terminalComputeBillingOutboxTrigger: false,
1827
2031
  outboxWakeTrigger: false,
1828
- }));
1829
- const schemaIsCurrentWithoutTerminalComputeBillingOutboxTrigger =
1830
- !schemaIsCurrent &&
1831
- (await hasCurrentPostgresSchedulerSchemaShape(client, options, {
1832
- openRecoveryIndex: !options?.deferOpenRecoveryIndex,
1833
- runtimeAdminIndexes: !options?.deferRuntimeAdminIndexes,
1834
- admissionPriorityIndex: !options?.deferRuntimeAdminIndexes,
1835
- terminalComputeBillingOutboxTrigger: false,
2032
+ cronScheduleSchema: false,
1836
2033
  }));
1837
2034
  const schemaCoreIsCurrent =
1838
- schemaIsCurrent || schemaIsCurrentWithoutAdditiveOutboxTriggers;
2035
+ schemaIsCurrent || schemaIsCurrentWithoutAdditiveFastPathObjects;
1839
2036
  // Older workers can still create a release-only fence after the current
1840
2037
  // schema marker has been written. The metadata is current, but the data
1841
2038
  // must be copied before a new worker trusts the fast path.
@@ -1853,13 +2050,14 @@ export async function ensurePostgresSchedulerSchema(
1853
2050
  const hasLegacyAdmissionRows =
1854
2051
  legacyAdmissionRows?.rows[0]?.present === true;
1855
2052
  if (schemaCoreIsCurrent) {
1856
- if (schemaIsCurrentWithoutTerminalComputeBillingOutboxTrigger) {
2053
+ if (!schemaIsCurrent) {
1857
2054
  await ensurePostgresSchedulerTerminalComputeBillingOutboxTrigger(
1858
2055
  client,
1859
2056
  options,
1860
2057
  );
1861
2058
  }
1862
2059
  await ensurePostgresSchedulerOutboxWakeTrigger(client, options);
2060
+ await ensurePostgresSchedulerCronScheduleSchema(client, options);
1863
2061
  if (hasLegacyAdmissionRows) {
1864
2062
  await migrateLegacyReleaseAdmissionRows(client, options);
1865
2063
  }
@@ -3020,6 +3218,30 @@ export async function ensurePostgresSchedulerSchema(
3020
3218
  (available_at, created_at, provider, sandbox_id)
3021
3219
  WHERE status <> 'deleted'
3022
3220
  `);
3221
+ if (!options?.deferRuntimeAdminIndexes) {
3222
+ await client.query(`
3223
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_ready_health_idx
3224
+ ON ${tableName('sandbox_cleanup_jobs_v2', options)}
3225
+ (available_at, created_at, provider, sandbox_id)
3226
+ WHERE status <> 'deleted'
3227
+ AND blocked_at IS NULL
3228
+ AND lease_owner IS NULL
3229
+ AND status <> 'deleting'
3230
+ `);
3231
+ await client.query(`
3232
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_lease_health_idx
3233
+ ON ${tableName('sandbox_cleanup_jobs_v2', options)}
3234
+ (lease_expires_at, created_at, provider, sandbox_id)
3235
+ WHERE status <> 'deleted' AND lease_owner IS NOT NULL
3236
+ `);
3237
+ await client.query(`
3238
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx
3239
+ ON ${tableName('sandbox_cleanup_jobs_v2', options)}
3240
+ (blocked_at, lease_expires_at, created_at, provider, sandbox_id)
3241
+ WHERE status <> 'deleted'
3242
+ AND (blocked_at IS NOT NULL OR status = 'deleting')
3243
+ `);
3244
+ }
3023
3245
  await client.query(`
3024
3246
  CREATE INDEX IF NOT EXISTS ${POSTGRES_SCHEDULER_COMPUTE_BILLING_PENDING_INDEX}
3025
3247
  ON ${tableName('compute_billing_jobs_v2', options)}
@@ -3073,6 +3295,27 @@ export async function ensurePostgresSchedulerSchema(
3073
3295
  ON ${tableName('outbox', options)} (created_at, event_id)
3074
3296
  WHERE delivered_at IS NULL AND failed_at IS NULL
3075
3297
  `);
3298
+ await client.query(`
3299
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lease_health_idx
3300
+ ON ${tableName('outbox', options)}
3301
+ (lease_expires_at, created_at, event_id)
3302
+ WHERE delivered_at IS NULL
3303
+ AND failed_at IS NULL
3304
+ AND lease_owner IS NOT NULL
3305
+ `);
3306
+ await client.query(`
3307
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_ready_health_idx
3308
+ ON ${tableName('outbox', options)}
3309
+ (coalesce(next_attempt_at, created_at), created_at, event_id)
3310
+ WHERE delivered_at IS NULL
3311
+ AND failed_at IS NULL
3312
+ AND lease_owner IS NULL
3313
+ `);
3314
+ await client.query(`
3315
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_dead_health_idx
3316
+ ON ${tableName('outbox', options)} (failed_at, event_id)
3317
+ WHERE delivered_at IS NULL AND failed_at IS NOT NULL
3318
+ `);
3076
3319
  await client.query(`
3077
3320
  CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_delivered_retention_idx
3078
3321
  ON ${tableName('outbox', options)} (delivered_at, event_id)
@@ -3084,6 +3327,28 @@ export async function ensurePostgresSchedulerSchema(
3084
3327
  WHERE delivered_at IS NULL AND failed_at IS NOT NULL
3085
3328
  `);
3086
3329
  }
3330
+ if (!options?.deferRuntimeAdminIndexes) {
3331
+ await client.query(`
3332
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_ready_health_idx
3333
+ ON ${tableName('compute_billing_jobs_v2', options)}
3334
+ (available_at, created_at, run_id)
3335
+ WHERE status <> 'settled'
3336
+ AND blocked_at IS NULL
3337
+ AND lease_owner IS NULL
3338
+ `);
3339
+ await client.query(`
3340
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_lease_health_idx
3341
+ ON ${tableName('compute_billing_jobs_v2', options)}
3342
+ (lease_expires_at, created_at, run_id)
3343
+ WHERE status <> 'settled' AND lease_owner IS NOT NULL
3344
+ `);
3345
+ await client.query(`
3346
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_blocked_health_idx
3347
+ ON ${tableName('compute_billing_jobs_v2', options)}
3348
+ (blocked_at, lease_expires_at, created_at, run_id)
3349
+ WHERE status <> 'settled' AND blocked_at IS NOT NULL
3350
+ `);
3351
+ }
3087
3352
  await client.query(`
3088
3353
  CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_run_created_idx
3089
3354
  ON ${tableName('outbox', options)} (run_id, created_at, event_id)
@@ -3111,6 +3376,7 @@ export async function ensurePostgresSchedulerSchema(
3111
3376
  )} (version) VALUES ($1) ON CONFLICT (version) DO NOTHING`,
3112
3377
  [POSTGRES_SCHEDULER_SCHEMA_VERSION],
3113
3378
  );
3379
+ await ensurePostgresSchedulerCronScheduleSchema(client, options);
3114
3380
  if (hadOutbox.rows[0]?.present !== true) {
3115
3381
  await client.query(
3116
3382
  `
@@ -3159,6 +3425,28 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
3159
3425
  definition: `(created_at, event_id)
3160
3426
  WHERE delivered_at IS NULL AND failed_at IS NULL`,
3161
3427
  },
3428
+ {
3429
+ name: 'runtime_scheduler_outbox_lease_health_idx',
3430
+ table: 'outbox',
3431
+ definition: `(lease_expires_at, created_at, event_id)
3432
+ WHERE delivered_at IS NULL
3433
+ AND failed_at IS NULL
3434
+ AND lease_owner IS NOT NULL`,
3435
+ },
3436
+ {
3437
+ name: 'runtime_scheduler_outbox_ready_health_idx',
3438
+ table: 'outbox',
3439
+ definition: `(coalesce(next_attempt_at, created_at), created_at, event_id)
3440
+ WHERE delivered_at IS NULL
3441
+ AND failed_at IS NULL
3442
+ AND lease_owner IS NULL`,
3443
+ },
3444
+ {
3445
+ name: 'runtime_scheduler_outbox_dead_health_idx',
3446
+ table: 'outbox',
3447
+ definition: `(failed_at, event_id)
3448
+ WHERE delivered_at IS NULL AND failed_at IS NOT NULL`,
3449
+ },
3162
3450
  {
3163
3451
  name: 'runtime_scheduler_outbox_admission_pending_idx',
3164
3452
  table: 'outbox',
@@ -3182,6 +3470,48 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
3182
3470
  definition: `(created_at, event_id)
3183
3471
  WHERE delivered_at IS NULL AND failed_at IS NOT NULL`,
3184
3472
  },
3473
+ {
3474
+ name: 'runtime_scheduler_sandbox_cleanup_v2_ready_health_idx',
3475
+ table: 'sandbox_cleanup_jobs_v2',
3476
+ definition: `(available_at, created_at, provider, sandbox_id)
3477
+ WHERE status <> 'deleted'
3478
+ AND blocked_at IS NULL
3479
+ AND lease_owner IS NULL
3480
+ AND status <> 'deleting'`,
3481
+ },
3482
+ {
3483
+ name: 'runtime_scheduler_sandbox_cleanup_v2_lease_health_idx',
3484
+ table: 'sandbox_cleanup_jobs_v2',
3485
+ definition: `(lease_expires_at, created_at, provider, sandbox_id)
3486
+ WHERE status <> 'deleted' AND lease_owner IS NOT NULL`,
3487
+ },
3488
+ {
3489
+ name: 'runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx',
3490
+ table: 'sandbox_cleanup_jobs_v2',
3491
+ definition: `(blocked_at, lease_expires_at, created_at, provider, sandbox_id)
3492
+ WHERE status <> 'deleted'
3493
+ AND (blocked_at IS NOT NULL OR status = 'deleting')`,
3494
+ },
3495
+ {
3496
+ name: 'runtime_scheduler_compute_billing_ready_health_idx',
3497
+ table: 'compute_billing_jobs_v2',
3498
+ definition: `(available_at, created_at, run_id)
3499
+ WHERE status <> 'settled'
3500
+ AND blocked_at IS NULL
3501
+ AND lease_owner IS NULL`,
3502
+ },
3503
+ {
3504
+ name: 'runtime_scheduler_compute_billing_lease_health_idx',
3505
+ table: 'compute_billing_jobs_v2',
3506
+ definition: `(lease_expires_at, created_at, run_id)
3507
+ WHERE status <> 'settled' AND lease_owner IS NOT NULL`,
3508
+ },
3509
+ {
3510
+ name: 'runtime_scheduler_compute_billing_blocked_health_idx',
3511
+ table: 'compute_billing_jobs_v2',
3512
+ definition: `(blocked_at, lease_expires_at, created_at, run_id)
3513
+ WHERE status <> 'settled' AND blocked_at IS NOT NULL`,
3514
+ },
3185
3515
  {
3186
3516
  name: 'runtime_scheduler_runs_terminal_finished_idx',
3187
3517
  table: 'runs',
@@ -3518,17 +3848,24 @@ export async function ensureAbsurdSchema(
3518
3848
  client: PostgresSchedulerQueryClient,
3519
3849
  ): Promise<void> {
3520
3850
  const [lockClassId, lockObjectId] = absurdSchemaInitAdvisoryLockKey();
3521
- await client.query('BEGIN');
3851
+ let stage = 'begin transaction';
3852
+ let transactionStarted = false;
3522
3853
  let committed = false;
3523
3854
  try {
3855
+ await client.query('BEGIN');
3856
+ transactionStarted = true;
3857
+
3858
+ stage = 'set bootstrap statement timeout';
3524
3859
  await client.query(
3525
3860
  `SELECT set_config('statement_timeout', '${POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS}', true)`,
3526
3861
  );
3862
+ stage = 'acquire Absurd schema bootstrap advisory lock';
3527
3863
  await client.query(
3528
3864
  `SELECT pg_advisory_xact_lock($1::integer, $2::integer)`,
3529
3865
  [lockClassId, lockObjectId],
3530
3866
  );
3531
3867
 
3868
+ stage = 'read Absurd schema catalog marker';
3532
3869
  const present = await client.query<{ present: boolean }>(
3533
3870
  `SELECT EXISTS (
3534
3871
  SELECT 1
@@ -3540,11 +3877,13 @@ export async function ensureAbsurdSchema(
3540
3877
  );
3541
3878
 
3542
3879
  if (present.rows[0]?.present) {
3880
+ stage = 'read installed Absurd schema version';
3543
3881
  const versionRow = await client.query<{ v: string | null }>(
3544
3882
  `SELECT absurd.get_schema_version() AS v`,
3545
3883
  );
3546
3884
  const installed = versionRow.rows[0]?.v ?? null;
3547
3885
  if (installed === ABSURD_SCHEMA_VERSION) {
3886
+ stage = 'read Absurd compatibility marker catalog entry';
3548
3887
  const marker = await client.query<{ present: boolean }>(
3549
3888
  `SELECT EXISTS (
3550
3889
  SELECT 1
@@ -3554,6 +3893,7 @@ export async function ensureAbsurdSchema(
3554
3893
  AND p.proname = 'deepline_no_scheduler_org_cap_compatibility_version'
3555
3894
  ) AS present`,
3556
3895
  );
3896
+ stage = 'read Absurd compatibility marker version';
3557
3897
  const markerVersion = marker.rows[0]?.present
3558
3898
  ? await client.query<{ v: string | null }>(
3559
3899
  `SELECT absurd.deepline_no_scheduler_org_cap_compatibility_version() AS v`,
@@ -3563,13 +3903,16 @@ export async function ensureAbsurdSchema(
3563
3903
  markerVersion?.rows[0]?.v !==
3564
3904
  ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION
3565
3905
  ) {
3906
+ stage = 'apply Absurd no-org-cap compatibility function patch';
3566
3907
  await client.query(
3567
3908
  readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(),
3568
3909
  );
3910
+ stage = 'apply Absurd no-org-cap compatibility marker patch';
3569
3911
  await client.query(
3570
3912
  readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(),
3571
3913
  );
3572
3914
  }
3915
+ stage = 'commit verified Absurd schema bootstrap';
3573
3916
  await client.query('COMMIT');
3574
3917
  committed = true;
3575
3918
  return;
@@ -3579,32 +3922,42 @@ export async function ensureAbsurdSchema(
3579
3922
  `"${ABSURD_SCHEMA_VERSION}". Continuing; the next SQL operation ` +
3580
3923
  'will report any missing relation, column, or function.',
3581
3924
  );
3925
+ stage = 'commit non-current Absurd schema marker';
3582
3926
  await client.query('COMMIT');
3583
3927
  committed = true;
3584
3928
  return;
3585
3929
  }
3586
3930
 
3931
+ stage = 'apply full Absurd schema';
3587
3932
  await client.query(readAbsurdSchemaSql());
3933
+ stage = 'apply Absurd no-org-cap compatibility function patch';
3588
3934
  await client.query(readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql());
3935
+ stage = 'apply Absurd no-org-cap compatibility marker patch';
3589
3936
  await client.query(
3590
3937
  readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(),
3591
3938
  );
3939
+ stage = 'commit newly initialized Absurd schema';
3592
3940
  await client.query('COMMIT');
3593
3941
  committed = true;
3594
3942
  } catch (error) {
3595
- if (!committed) {
3943
+ const primaryError = new Error(
3944
+ `Failed to initialize Absurd schema during ${stage}. ` +
3945
+ `Primary error: ${errorMessage(error)}.`,
3946
+ { cause: error },
3947
+ );
3948
+ if (transactionStarted && !committed) {
3596
3949
  try {
3597
3950
  await client.query('ROLLBACK');
3598
3951
  } catch (rollbackError) {
3599
3952
  throw new AggregateError(
3600
- [error, rollbackError],
3953
+ [primaryError, rollbackError],
3601
3954
  'Failed to initialize absurd schema and roll back the failed transaction. ' +
3602
- `Primary error: ${errorMessage(error)}. ` +
3955
+ `${primaryError.message} ` +
3603
3956
  `Rollback error: ${errorMessage(rollbackError)}.`,
3604
3957
  );
3605
3958
  }
3606
3959
  }
3607
- throw error;
3960
+ throw primaryError;
3608
3961
  }
3609
3962
  }
3610
3963
 
@@ -3629,7 +3982,12 @@ export async function submitPostgresAbsurdPlayRun(
3629
3982
  taskRecoveryRequired: boolean;
3630
3983
  }> {
3631
3984
  const task = buildAbsurdPlayRunTask({
3632
- launch: input,
3985
+ // This fence applies to admission only. A repaired historic task must
3986
+ // retain the persisted launch envelope, not the one-shot retry policy.
3987
+ launch: (() => {
3988
+ const { existingOnly: _existingOnly, ...launch } = input;
3989
+ return launch;
3990
+ })(),
3633
3991
  schedulerSchema: options?.schema,
3634
3992
  });
3635
3993
  const submitted = await submitPostgresSchedulerRunTransaction(
@@ -3689,9 +4047,11 @@ async function submitPostgresSchedulerRunTransaction(
3689
4047
  asyncParentRunAttempt: _asyncParentRunAttempt,
3690
4048
  asyncParentRunLeaseToken: _asyncParentRunLeaseToken,
3691
4049
  asyncParentAttemptRequired: _asyncParentAttemptRequired,
4050
+ existingOnly: requestedExistingOnly,
3692
4051
  ...durableInput
3693
4052
  } = input;
3694
4053
  input = durableInput;
4054
+ const existingOnlySql = requestedExistingOnly === true ? 'true' : 'false';
3695
4055
  const releaseId = input.absurdReleaseId?.trim() || null;
3696
4056
  const legacyPreviewRelease =
3697
4057
  releaseId === 'dev' || releaseId?.startsWith('dev-') === true;
@@ -3734,6 +4094,7 @@ async function submitPostgresSchedulerRunTransaction(
3734
4094
  NOT admission.existing_run
3735
4095
  AND admission.accepting_new
3736
4096
  AND admission.parent_accepting
4097
+ AND NOT ${existingOnlySql}
3737
4098
  ) OR (
3738
4099
  ${recoverExistingTaskSql}
3739
4100
  AND admission.existing_run
@@ -3921,6 +4282,7 @@ async function submitPostgresSchedulerRunTransaction(
3921
4282
  WHERE NOT existing_run
3922
4283
  AND accepting_new
3923
4284
  AND parent_accepting
4285
+ AND NOT ${existingOnlySql}
3924
4286
  ON CONFLICT (run_id) DO NOTHING
3925
4287
  RETURNING run_id, absurd_task_id
3926
4288
  ), run_recovery AS (
@@ -3948,153 +4310,1292 @@ async function submitPostgresSchedulerRunTransaction(
3948
4310
  run_insert.run_id,
3949
4311
  'run.created',
3950
4312
  jsonb_strip_nulls(jsonb_build_object(
3951
- 'orgId', $2::text,
3952
- 'playId', $3::text,
3953
- 'playName', $4::text,
3954
- -- Optional lineage makes a projected async child discoverable
3955
- -- from its terminal parent. Older envelopes omit these keys.
3956
- 'parentRunId', $8::jsonb ->> 'parentRunId',
3957
- 'rootRunId', $8::jsonb ->> 'rootRunId',
3958
- 'asyncLaunchKey', $8::jsonb ->> 'asyncLaunchKey',
3959
- 'asyncInvocationKey', $8::jsonb ->> 'asyncInvocationKey',
3960
- 'asyncRequestFingerprint', $8::jsonb ->> 'asyncRequestFingerprint',
3961
- 'asyncAncestryPlayIds', $8::jsonb -> 'asyncAncestryPlayIds',
3962
- 'definitionName', $8::jsonb ->> 'definitionName',
3963
- 'playReference', $8::jsonb ->> 'playReference',
3964
- 'definitionScope', $8::jsonb ->> 'definitionScope',
3965
- 'revisionId', $8::jsonb ->> 'revisionId',
3966
- 'workflowFamilyKey', $8::jsonb ->> 'workflowFamilyKey',
3967
- 'artifactStorageKey', $7::text,
3968
- 'artifactHash', $8::jsonb ->> 'artifactHash',
3969
- 'graphHash', $5::text,
3970
- 'runtimeBackend', $6::text,
3971
- 'schedulerBackend', $8::jsonb ->> 'schedulerBackend',
3972
- 'schedulerSchema', $8::jsonb ->> 'schedulerSchema',
3973
- 'executionProfile', $8::jsonb ->> 'executionProfile',
3974
- 'runtimeReleaseId', $8::jsonb ->> 'runtimeReleaseId',
3975
- 'runtimeDeployVersion', $8::jsonb ->> 'runtimeDeployVersion',
3976
- 'runtimeCallbackBaseUrl', $8::jsonb ->> 'runtimeCallbackBaseUrl',
3977
- 'coordinatorUrl', $8::jsonb ->> 'coordinatorUrl',
3978
- 'coordinatorWorkerName', $8::jsonb ->> 'coordinatorWorkerName',
3979
- 'runtimeWorkflowName', $8::jsonb ->> 'runtimeWorkflowName',
3980
- 'runtimeHarnessWorkerName', $8::jsonb ->> 'runtimeHarnessWorkerName',
3981
- 'maxCreditsPerRun', $8::jsonb -> 'maxCreditsPerRun',
3982
- 'staticPipeline', $8::jsonb -> 'staticPipeline',
3983
- 'source', $8::jsonb ->> 'source',
3984
- 'triggerSource', $8::jsonb ->> 'triggerSource',
3985
- 'inputFileId', $8::jsonb ->> 'inputFileId',
3986
- 'inputBytes', $8::jsonb -> 'inputBytes',
3987
- 'inputSha256', $8::jsonb ->> 'inputSha256',
3988
- 'replayedFromRunId', $8::jsonb ->> 'replayedFromRunId',
3989
- 'secretRefs', $8::jsonb -> 'secretRefs',
3990
- 'inputSummary', $8::jsonb -> 'inputSummary'
4313
+ 'orgId', $2::text,
4314
+ 'playId', $3::text,
4315
+ 'playName', $4::text,
4316
+ -- Optional lineage makes a projected async child discoverable
4317
+ -- from its terminal parent. Older envelopes omit these keys.
4318
+ 'parentRunId', $8::jsonb ->> 'parentRunId',
4319
+ 'rootRunId', $8::jsonb ->> 'rootRunId',
4320
+ 'asyncLaunchKey', $8::jsonb ->> 'asyncLaunchKey',
4321
+ 'asyncInvocationKey', $8::jsonb ->> 'asyncInvocationKey',
4322
+ 'asyncRequestFingerprint', $8::jsonb ->> 'asyncRequestFingerprint',
4323
+ 'asyncAncestryPlayIds', $8::jsonb -> 'asyncAncestryPlayIds',
4324
+ 'definitionName', $8::jsonb ->> 'definitionName',
4325
+ 'playReference', $8::jsonb ->> 'playReference',
4326
+ 'definitionScope', $8::jsonb ->> 'definitionScope',
4327
+ 'revisionId', $8::jsonb ->> 'revisionId',
4328
+ 'workflowFamilyKey', $8::jsonb ->> 'workflowFamilyKey',
4329
+ 'artifactStorageKey', $7::text,
4330
+ 'artifactHash', $8::jsonb ->> 'artifactHash',
4331
+ 'graphHash', $5::text,
4332
+ 'runtimeBackend', $6::text,
4333
+ 'schedulerBackend', $8::jsonb ->> 'schedulerBackend',
4334
+ 'schedulerSchema', $8::jsonb ->> 'schedulerSchema',
4335
+ 'executionProfile', $8::jsonb ->> 'executionProfile',
4336
+ 'runtimeReleaseId', $8::jsonb ->> 'runtimeReleaseId',
4337
+ 'runtimeDeployVersion', $8::jsonb ->> 'runtimeDeployVersion',
4338
+ 'runtimeCallbackBaseUrl', $8::jsonb ->> 'runtimeCallbackBaseUrl',
4339
+ 'coordinatorUrl', $8::jsonb ->> 'coordinatorUrl',
4340
+ 'coordinatorWorkerName', $8::jsonb ->> 'coordinatorWorkerName',
4341
+ 'runtimeWorkflowName', $8::jsonb ->> 'runtimeWorkflowName',
4342
+ 'runtimeHarnessWorkerName', $8::jsonb ->> 'runtimeHarnessWorkerName',
4343
+ 'maxCreditsPerRun', $8::jsonb -> 'maxCreditsPerRun',
4344
+ 'staticPipeline', $8::jsonb -> 'staticPipeline',
4345
+ 'source', $8::jsonb ->> 'source',
4346
+ 'triggerSource', $8::jsonb ->> 'triggerSource',
4347
+ 'inputFileId', $8::jsonb ->> 'inputFileId',
4348
+ 'inputBytes', $8::jsonb -> 'inputBytes',
4349
+ 'inputSha256', $8::jsonb ->> 'inputSha256',
4350
+ 'replayedFromRunId', $8::jsonb ->> 'replayedFromRunId',
4351
+ 'secretRefs', $8::jsonb -> 'secretRefs',
4352
+ 'inputSummary', $8::jsonb -> 'inputSummary'
4353
+ )),
4354
+ ${causalOutboxCreatedAt('run_insert.run_id', options)}
4355
+ FROM run_insert
4356
+ -- Old Vercel launchers do not carry the immutable admission envelope.
4357
+ -- During a Convex-first rollout they retain their established start
4358
+ -- projection; only the new launcher opts into queued projection.
4359
+ WHERE $8::jsonb ? 'playReference'
4360
+ ON CONFLICT (event_id) DO NOTHING
4361
+ RETURNING run_id
4362
+ )
4363
+ SELECT (
4364
+ (NOT admission.existing_run AND admission.accepting_new
4365
+ AND admission.parent_accepting AND NOT ${existingOnlySql})
4366
+ OR admission.matching_existing
4367
+ ) AS accepted,
4368
+ admission.accepting_new,
4369
+ admission.parent_accepting,
4370
+ admission.parent_attempt_current,
4371
+ admission.parent_terminal,
4372
+ admission.existing_run,
4373
+ admission.matching_existing,
4374
+ coalesce(
4375
+ (SELECT task_id FROM task_spawn),
4376
+ (SELECT absurd_task_id FROM existing_run)
4377
+ ) AS absurd_task_id,
4378
+ EXISTS (SELECT 1 FROM run_insert) AS run_created,
4379
+ (SELECT created FROM task_spawn) AS task_created,
4380
+ (
4381
+ admission.matching_existing
4382
+ AND NOT ${recoverExistingTaskSql}
4383
+ AND EXISTS (
4384
+ SELECT 1
4385
+ FROM existing_run
4386
+ WHERE existing_run.status = 'queued'
4387
+ AND existing_run.absurd_task_id IS NULL
4388
+ )
4389
+ ) AS task_recovery_required,
4390
+ CASE WHEN EXISTS (SELECT 1 FROM run_insert)
4391
+ OR EXISTS (SELECT 1 FROM run_recovery)
4392
+ THEN pg_notify($10, $11)
4393
+ END
4394
+ FROM admission
4395
+ `,
4396
+ [
4397
+ input.runId,
4398
+ input.orgId,
4399
+ input.playId,
4400
+ input.playName,
4401
+ input.graphHash,
4402
+ input.runtimeBackend,
4403
+ input.artifactStorageKey,
4404
+ stringifyPostgresJson(input),
4405
+ stringifyPostgresJson(input.input ?? {}),
4406
+ postgresSchedulerNotifyChannel('queue', options),
4407
+ JSON.stringify({ runId: input.runId, playName: input.playName }),
4408
+ releaseAdmissionKey,
4409
+ releaseAdmissionLockKey,
4410
+ useLegacyAdmissionKey,
4411
+ releaseId,
4412
+ parentRunId,
4413
+ asyncRootRunId,
4414
+ asyncParentRunAttempt,
4415
+ asyncParentRunLeaseToken,
4416
+ asyncParentAttemptRequired,
4417
+ ...(absurdTask
4418
+ ? [
4419
+ absurdTask.queueName,
4420
+ absurdTask.taskName,
4421
+ stringifyPostgresJson(absurdTask.params),
4422
+ stringifyPostgresJson(absurdTask.options),
4423
+ ]
4424
+ : []),
4425
+ ],
4426
+ );
4427
+ const row = submitted.rows[0];
4428
+ if (
4429
+ row?.accepted === false &&
4430
+ !row.existing_run &&
4431
+ row.parent_accepting === false
4432
+ ) {
4433
+ if (
4434
+ row?.parent_terminal === false &&
4435
+ row?.parent_attempt_current === false
4436
+ ) {
4437
+ throw new Error(
4438
+ `CTX_RUN_PLAY_ASYNC_PARENT_ATTEMPT_STALE: parent Run "${parentRunId}" no longer has the executor attempt that requested this child.`,
4439
+ );
4440
+ }
4441
+ throw new Error(
4442
+ `CTX_RUN_PLAY_ASYNC_PARENT_TERMINAL: parent Run "${parentRunId}" is terminal, missing, belongs to another organization, or is pinned to a different release.`,
4443
+ );
4444
+ }
4445
+ if (row?.accepted === false && row.existing_run && !row.matching_existing) {
4446
+ throw new Error(
4447
+ `Play run "${input.runId}" already exists with a different launch.`,
4448
+ );
4449
+ }
4450
+ if (row?.accepted === false && !row?.existing_run && requestedExistingOnly) {
4451
+ throw new Error(
4452
+ `Play run "${input.runId}" does not exist for an existing-only admission.`,
4453
+ );
4454
+ }
4455
+ if (row?.accepted === false) {
4456
+ throw new Error(
4457
+ `Absurd runtime release "${releaseId}" no longer accepts new launches.`,
4458
+ );
4459
+ }
4460
+ return {
4461
+ absurdTaskId: row?.absurd_task_id ?? null,
4462
+ runCreated: row?.run_created === true,
4463
+ taskCreated: row?.task_created === true,
4464
+ taskRecoveryRequired: row?.task_recovery_required === true,
4465
+ };
4466
+ }
4467
+
4468
+ /** Immutable Runtime Postgres state for an Absurd-owned Play cron timer. */
4469
+ export type PostgresSchedulerCronSchedule = {
4470
+ scheduleId: string;
4471
+ generation: number;
4472
+ state: 'active' | 'blocked' | 'cancelled';
4473
+ cron: string;
4474
+ timezone: string | null;
4475
+ dueAtMs: number;
4476
+ currentFireTaskId: string | null;
4477
+ currentFireGeneration: number;
4478
+ currentFireQueueName: string | null;
4479
+ /** Immutable source identity only; no executor token or release lane. */
4480
+ sourcePin: PostgresSchedulerCronSourcePin;
4481
+ };
4482
+
4483
+ export type PostgresSchedulerCronFireTaskState =
4484
+ | 'pending'
4485
+ | 'running'
4486
+ | 'sleeping'
4487
+ | 'completed'
4488
+ | 'failed'
4489
+ | 'cancelled';
4490
+
4491
+ /**
4492
+ * Read-only health view for the Runtime-owned cron successor contract.
4493
+ *
4494
+ * A schedule is the source-of-truth cursor. Its current fire task is merely
4495
+ * the durable delivery vehicle for that cursor, so a missing or terminal task
4496
+ * on an active row is an operator-visible integrity failure rather than a
4497
+ * reason to silently advance or recreate an occurrence.
4498
+ */
4499
+ export type PostgresSchedulerCronHealth = {
4500
+ observedAtMs: number;
4501
+ activeSchedules: number;
4502
+ blockedSchedules: number;
4503
+ cancelledSchedules: number;
4504
+ overdueSchedules: number;
4505
+ oldestOverdueDueAtMs: number | null;
4506
+ missingCurrentFireTasks: number;
4507
+ terminalCurrentFireTasks: number;
4508
+ issues: Array<{
4509
+ scheduleId: string;
4510
+ generation: number;
4511
+ state: 'active' | 'blocked' | 'cancelled';
4512
+ dueAtMs: number;
4513
+ currentFireGeneration: number;
4514
+ currentFireTaskId: string | null;
4515
+ currentFireQueueName: string | null;
4516
+ fireTaskState: PostgresSchedulerCronFireTaskState | null;
4517
+ overdue: boolean;
4518
+ missingCurrentFireTask: boolean;
4519
+ terminalCurrentFireTask: boolean;
4520
+ }>;
4521
+ };
4522
+
4523
+ type PostgresSchedulerCronHealthQueryRow = {
4524
+ schedule_id: string;
4525
+ generation: number | string;
4526
+ state: 'active' | 'blocked' | 'cancelled';
4527
+ due_at_ms: number | string;
4528
+ current_fire_generation: number | string;
4529
+ current_fire_task_id: string | null;
4530
+ current_fire_queue_name: string | null;
4531
+ fire_task_state: string | null;
4532
+ overdue: boolean;
4533
+ missing_current_fire_task: boolean;
4534
+ terminal_current_fire_task: boolean;
4535
+ };
4536
+
4537
+ const POSTGRES_SCHEDULER_CRON_FIRE_TASK_STATES =
4538
+ new Set<PostgresSchedulerCronFireTaskState>([
4539
+ 'pending',
4540
+ 'running',
4541
+ 'sleeping',
4542
+ 'completed',
4543
+ 'failed',
4544
+ 'cancelled',
4545
+ ]);
4546
+
4547
+ function postgresSchedulerCronFireTaskState(
4548
+ value: string | null,
4549
+ ): PostgresSchedulerCronFireTaskState | null {
4550
+ return value &&
4551
+ POSTGRES_SCHEDULER_CRON_FIRE_TASK_STATES.has(
4552
+ value as PostgresSchedulerCronFireTaskState,
4553
+ )
4554
+ ? (value as PostgresSchedulerCronFireTaskState)
4555
+ : null;
4556
+ }
4557
+
4558
+ function postgresSchedulerCronHealthInteger(
4559
+ value: number | string | null | undefined,
4560
+ ): number {
4561
+ const parsed = Number(value ?? 0);
4562
+ return Number.isSafeInteger(parsed) ? parsed : 0;
4563
+ }
4564
+
4565
+ function assertPostgresSchedulerCronHealthInput(input: {
4566
+ nowMs?: number;
4567
+ limit?: number;
4568
+ }): { nowMs: number; limit: number } {
4569
+ const nowMs = input.nowMs ?? Date.now();
4570
+ if (!Number.isSafeInteger(nowMs) || nowMs <= 0) {
4571
+ throw new Error('Cron health nowMs must be a positive safe integer.');
4572
+ }
4573
+ const limit = input.limit ?? 50;
4574
+ if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
4575
+ throw new Error(
4576
+ 'Cron health limit must be a safe integer from 1 through 100.',
4577
+ );
4578
+ }
4579
+ return { nowMs, limit };
4580
+ }
4581
+
4582
+ /**
4583
+ * Read a bounded, non-mutating cron integrity report.
4584
+ *
4585
+ * The query joins only registered Absurd queues before asking the engine for a
4586
+ * task. A retired/missing queue therefore becomes a `missingCurrentFireTask`
4587
+ * signal instead of causing this diagnostic endpoint to fail while it is most
4588
+ * needed. Repair remains an explicit, separately fenced operation.
4589
+ */
4590
+ export async function readPostgresSchedulerCronHealth(
4591
+ client: PostgresSchedulerQueryClient,
4592
+ input: { nowMs?: number; limit?: number } = {},
4593
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
4594
+ ): Promise<PostgresSchedulerCronHealth> {
4595
+ const { nowMs, limit } = assertPostgresSchedulerCronHealthInput(input);
4596
+ const schedules = tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options);
4597
+ const healthRowsSql = `
4598
+ WITH schedules AS MATERIALIZED (
4599
+ SELECT schedule_id,
4600
+ generation,
4601
+ state,
4602
+ floor(extract(epoch FROM current_due_at) * 1000)::bigint AS due_at_ms,
4603
+ current_fire_task_id,
4604
+ current_fire_generation,
4605
+ current_fire_queue_name
4606
+ FROM ${schedules}
4607
+ ), valid_timer_references AS MATERIALIZED (
4608
+ SELECT schedule.schedule_id,
4609
+ schedule.current_fire_task_id,
4610
+ schedule.current_fire_queue_name
4611
+ FROM schedules AS schedule
4612
+ JOIN absurd.queues AS queue
4613
+ ON queue.queue_name = schedule.current_fire_queue_name
4614
+ WHERE schedule.current_fire_task_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
4615
+ ), timer_states AS MATERIALIZED (
4616
+ SELECT reference.schedule_id,
4617
+ task.state AS fire_task_state
4618
+ FROM valid_timer_references AS reference
4619
+ CROSS JOIN LATERAL absurd.get_task_result(
4620
+ reference.current_fire_queue_name,
4621
+ reference.current_fire_task_id::uuid
4622
+ ) AS task
4623
+ ), health_rows AS MATERIALIZED (
4624
+ SELECT schedule.schedule_id,
4625
+ schedule.generation,
4626
+ schedule.state,
4627
+ schedule.due_at_ms,
4628
+ schedule.current_fire_generation,
4629
+ schedule.current_fire_task_id,
4630
+ schedule.current_fire_queue_name,
4631
+ timer.fire_task_state,
4632
+ (schedule.state = 'active' AND schedule.due_at_ms <= $1::bigint) AS overdue,
4633
+ (
4634
+ schedule.state = 'active'
4635
+ AND (
4636
+ schedule.current_fire_task_id IS NULL
4637
+ OR schedule.current_fire_queue_name IS NULL
4638
+ OR NOT COALESCE(
4639
+ schedule.current_fire_task_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$',
4640
+ false
4641
+ )
4642
+ OR timer.fire_task_state IS NULL
4643
+ )
4644
+ ) AS missing_current_fire_task,
4645
+ (
4646
+ schedule.state = 'active'
4647
+ AND timer.fire_task_state IN ('completed', 'failed', 'cancelled')
4648
+ ) AS terminal_current_fire_task
4649
+ FROM schedules AS schedule
4650
+ LEFT JOIN timer_states AS timer
4651
+ ON timer.schedule_id = schedule.schedule_id
4652
+ )`;
4653
+ const summary = await client.query<{
4654
+ active_schedules: number | string;
4655
+ blocked_schedules: number | string;
4656
+ cancelled_schedules: number | string;
4657
+ overdue_schedules: number | string;
4658
+ oldest_overdue_due_at_ms: number | string | null;
4659
+ missing_current_fire_tasks: number | string;
4660
+ terminal_current_fire_tasks: number | string;
4661
+ }>(
4662
+ `${healthRowsSql}
4663
+ SELECT count(*) FILTER (WHERE state = 'active')::text AS active_schedules,
4664
+ count(*) FILTER (WHERE state = 'blocked')::text AS blocked_schedules,
4665
+ count(*) FILTER (WHERE state = 'cancelled')::text AS cancelled_schedules,
4666
+ count(*) FILTER (WHERE overdue)::text AS overdue_schedules,
4667
+ min(due_at_ms) FILTER (WHERE overdue) AS oldest_overdue_due_at_ms,
4668
+ count(*) FILTER (WHERE missing_current_fire_task)::text AS missing_current_fire_tasks,
4669
+ count(*) FILTER (WHERE terminal_current_fire_task)::text AS terminal_current_fire_tasks
4670
+ FROM health_rows`,
4671
+ [nowMs],
4672
+ );
4673
+ const issueRows = await client.query<PostgresSchedulerCronHealthQueryRow>(
4674
+ `${healthRowsSql}
4675
+ SELECT schedule_id,
4676
+ generation,
4677
+ state,
4678
+ due_at_ms,
4679
+ current_fire_generation,
4680
+ current_fire_task_id,
4681
+ current_fire_queue_name,
4682
+ fire_task_state,
4683
+ overdue,
4684
+ missing_current_fire_task,
4685
+ terminal_current_fire_task
4686
+ FROM health_rows
4687
+ WHERE state = 'blocked'
4688
+ OR overdue
4689
+ OR missing_current_fire_task
4690
+ OR terminal_current_fire_task
4691
+ ORDER BY
4692
+ (state = 'blocked') DESC,
4693
+ missing_current_fire_task DESC,
4694
+ terminal_current_fire_task DESC,
4695
+ overdue DESC,
4696
+ due_at_ms ASC,
4697
+ schedule_id ASC
4698
+ LIMIT $2::int`,
4699
+ [nowMs, limit],
4700
+ );
4701
+ const row = summary.rows[0];
4702
+ return {
4703
+ observedAtMs: nowMs,
4704
+ activeSchedules: postgresSchedulerCronHealthInteger(row?.active_schedules),
4705
+ blockedSchedules: postgresSchedulerCronHealthInteger(
4706
+ row?.blocked_schedules,
4707
+ ),
4708
+ cancelledSchedules: postgresSchedulerCronHealthInteger(
4709
+ row?.cancelled_schedules,
4710
+ ),
4711
+ overdueSchedules: postgresSchedulerCronHealthInteger(
4712
+ row?.overdue_schedules,
4713
+ ),
4714
+ oldestOverdueDueAtMs:
4715
+ row?.oldest_overdue_due_at_ms == null
4716
+ ? null
4717
+ : postgresSchedulerCronHealthInteger(row.oldest_overdue_due_at_ms),
4718
+ missingCurrentFireTasks: postgresSchedulerCronHealthInteger(
4719
+ row?.missing_current_fire_tasks,
4720
+ ),
4721
+ terminalCurrentFireTasks: postgresSchedulerCronHealthInteger(
4722
+ row?.terminal_current_fire_tasks,
4723
+ ),
4724
+ issues: issueRows.rows.map((issue) => ({
4725
+ scheduleId: issue.schedule_id,
4726
+ generation: postgresSchedulerCronHealthInteger(issue.generation),
4727
+ state: issue.state,
4728
+ dueAtMs: postgresSchedulerCronHealthInteger(issue.due_at_ms),
4729
+ currentFireGeneration: postgresSchedulerCronHealthInteger(
4730
+ issue.current_fire_generation,
4731
+ ),
4732
+ currentFireTaskId: issue.current_fire_task_id,
4733
+ currentFireQueueName: issue.current_fire_queue_name,
4734
+ fireTaskState: postgresSchedulerCronFireTaskState(issue.fire_task_state),
4735
+ overdue: issue.overdue === true,
4736
+ missingCurrentFireTask: issue.missing_current_fire_task === true,
4737
+ terminalCurrentFireTask: issue.terminal_current_fire_task === true,
4738
+ })),
4739
+ };
4740
+ }
4741
+
4742
+ /** A pinned schedule can never become valid by retrying its same fire task. */
4743
+ export class PostgresSchedulerCronScheduleConfigurationError extends Error {
4744
+ constructor(message: string) {
4745
+ super(message);
4746
+ this.name = 'PostgresSchedulerCronScheduleConfigurationError';
4747
+ }
4748
+ }
4749
+
4750
+ /**
4751
+ * The timer stores the facts needed to reconstruct a pinned admission, not a
4752
+ * previously prepared launch. Executor tokens, preloaded session keys,
4753
+ * callback URLs, and release lanes are all intentionally absent: Vercel
4754
+ * creates them at the fire instant.
4755
+ */
4756
+ export type PostgresSchedulerCronSourcePin = {
4757
+ orgId: string;
4758
+ userId: string | null;
4759
+ playName: string;
4760
+ revisionId: string | null;
4761
+ artifactStorageKey: string;
4762
+ artifactKind: 'cjs_node20' | null;
4763
+ artifactHash: string | null;
4764
+ graphHash: string | null;
4765
+ staticPipeline: unknown | null;
4766
+ input: PlaySchedulerSubmitInput['input'];
4767
+ source: 'published' | 'draft';
4768
+ schedulerSchema: string | null;
4769
+ };
4770
+
4771
+ export function postgresSchedulerCronSourcePinFromLaunch(
4772
+ launch: PlaySchedulerSubmitInput,
4773
+ ): PostgresSchedulerCronSourcePin {
4774
+ const artifactKind =
4775
+ launch.runtimeArtifact &&
4776
+ typeof launch.runtimeArtifact === 'object' &&
4777
+ (launch.runtimeArtifact as { artifactKind?: unknown }).artifactKind ===
4778
+ 'cjs_node20'
4779
+ ? 'cjs_node20'
4780
+ : null;
4781
+ return {
4782
+ orgId: launch.orgId,
4783
+ userId: launch.userId ?? null,
4784
+ playName: launch.playName,
4785
+ revisionId: launch.revisionId ?? null,
4786
+ artifactStorageKey: launch.artifactStorageKey,
4787
+ artifactKind,
4788
+ artifactHash: launch.artifactHash || null,
4789
+ graphHash: launch.graphHash || null,
4790
+ staticPipeline: launch.staticPipeline ?? null,
4791
+ input: launch.input,
4792
+ source: launch.source === 'draft' ? 'draft' : 'published',
4793
+ schedulerSchema: launch.schedulerSchema ?? null,
4794
+ };
4795
+ }
4796
+
4797
+ export type SeedPostgresSchedulerCronScheduleInput = {
4798
+ scheduleId: string;
4799
+ generation: number;
4800
+ dueAtMs: number;
4801
+ cron: string;
4802
+ timezone?: string | null;
4803
+ sourcePin: PostgresSchedulerCronSourcePin;
4804
+ queueName: string;
4805
+ /** Explicit release registration gate; absence is never treated as ready. */
4806
+ releaseId: string;
4807
+ releaseEnvironment: 'preview' | 'production';
4808
+ };
4809
+
4810
+ function assertPostgresSchedulerCronScheduleInput(input: {
4811
+ scheduleId: string;
4812
+ generation: number;
4813
+ fireGeneration?: number;
4814
+ dueAtMs: number;
4815
+ cron?: string;
4816
+ }): void {
4817
+ if (!input.scheduleId.trim()) {
4818
+ throw new Error('Cron schedule requires a non-empty scheduleId.');
4819
+ }
4820
+ if (!Number.isSafeInteger(input.generation) || input.generation <= 0) {
4821
+ throw new Error(
4822
+ 'Cron schedule generation must be a positive safe integer.',
4823
+ );
4824
+ }
4825
+ if (
4826
+ input.fireGeneration !== undefined &&
4827
+ (!Number.isSafeInteger(input.fireGeneration) || input.fireGeneration <= 0)
4828
+ ) {
4829
+ throw new Error(
4830
+ 'Cron schedule fireGeneration must be a positive safe integer.',
4831
+ );
4832
+ }
4833
+ if (!Number.isSafeInteger(input.dueAtMs) || input.dueAtMs <= 0) {
4834
+ throw new Error('Cron schedule dueAtMs must be a positive safe integer.');
4835
+ }
4836
+ if (input.cron !== undefined && !input.cron.trim()) {
4837
+ throw new Error('Cron schedule requires a non-empty cron expression.');
4838
+ }
4839
+ }
4840
+
4841
+ /**
4842
+ * Persist (or generation-replace) one Runtime-owned cron cursor and its first
4843
+ * Absurd timer in a single statement. A caller must leave Convex as the owner
4844
+ * until this returns: the cross-store ownership flip is deliberately separate
4845
+ * from this durable timer primitive.
4846
+ */
4847
+ export async function seedPostgresSchedulerCronSchedule(
4848
+ client: PostgresSchedulerQueryClient,
4849
+ input: SeedPostgresSchedulerCronScheduleInput,
4850
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
4851
+ ): Promise<{
4852
+ created: boolean;
4853
+ scheduleId: string;
4854
+ fireTaskId: string | null;
4855
+ }> {
4856
+ assertPostgresSchedulerCronScheduleInput(input);
4857
+ if (!input.releaseId?.trim()) {
4858
+ throw new Error(
4859
+ 'Cron schedule requires an explicitly registered releaseId.',
4860
+ );
4861
+ }
4862
+ const sourcePin = input.sourcePin;
4863
+ const task = buildAbsurdCronFireTask({
4864
+ scheduleId: input.scheduleId.trim(),
4865
+ generation: input.generation,
4866
+ fireGeneration: 1,
4867
+ dueAtMs: input.dueAtMs,
4868
+ schedulerSchema: options?.schema ?? sourcePin.schedulerSchema ?? null,
4869
+ orgId: sourcePin.orgId,
4870
+ });
4871
+ const releaseAdmissionKey = `${input.releaseEnvironment}:${input.releaseId.trim()}`;
4872
+ const result = await client.query<{
4873
+ accepted: boolean;
4874
+ created: boolean;
4875
+ existing_generation: number | null;
4876
+ existing_matches_input: boolean | null;
4877
+ fire_task_id: string | null;
4878
+ }>(
4879
+ `
4880
+ WITH schedule_lock AS MATERIALIZED (
4881
+ SELECT pg_advisory_xact_lock(
4882
+ ${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
4883
+ ${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
4884
+ 'cron-schedule:' || $1::text
4885
+ )
4886
+ )
4887
+ ), existing AS MATERIALIZED (
4888
+ SELECT schedule.generation,
4889
+ (
4890
+ schedule.cron = $4::text
4891
+ AND schedule.timezone IS NOT DISTINCT FROM $5::text
4892
+ AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
4893
+ AND schedule.source_pin_json = $6::jsonb
4894
+ ) AS matches_input
4895
+ FROM schedule_lock
4896
+ JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
4897
+ ON schedule.schedule_id = $1::text
4898
+ FOR UPDATE
4899
+ ), release_gate AS MATERIALIZED (
4900
+ SELECT accepting
4901
+ FROM ${tableName('runtime_release_admission', options)}
4902
+ WHERE release_id = $11::text
4903
+ ), task_request AS (
4904
+ SELECT $7::text AS queue_name,
4905
+ $8::text AS task_name,
4906
+ $9::jsonb AS params,
4907
+ $10::jsonb AS options
4908
+ WHERE (
4909
+ NOT EXISTS (SELECT 1 FROM existing)
4910
+ OR (SELECT generation FROM existing) < $2::bigint
4911
+ )
4912
+ AND coalesce((SELECT accepting FROM release_gate), false)
4913
+ ), task_spawn AS MATERIALIZED (
4914
+ SELECT spawned.task_id::text AS task_id
4915
+ FROM task_request
4916
+ CROSS JOIN LATERAL absurd.spawn_task(
4917
+ task_request.queue_name,
4918
+ task_request.task_name,
4919
+ task_request.params,
4920
+ task_request.options
4921
+ ) AS spawned
4922
+ ), schedule_upsert AS (
4923
+ INSERT INTO ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule (
4924
+ schedule_id, generation, state, cron, timezone, current_due_at,
4925
+ current_fire_task_id, current_fire_generation, current_fire_queue_name,
4926
+ source_pin_json, launch_json
4927
+ )
4928
+ SELECT $1::text, $2::bigint, 'active', $4::text, $5::text,
4929
+ to_timestamp($3::double precision / 1000.0),
4930
+ (SELECT task_id FROM task_spawn), 1, $7::text, $6::jsonb, $6::jsonb
4931
+ WHERE (
4932
+ NOT EXISTS (SELECT 1 FROM existing)
4933
+ OR (SELECT generation FROM existing) < $2::bigint
4934
+ )
4935
+ AND coalesce((SELECT accepting FROM release_gate), false)
4936
+ ON CONFLICT (schedule_id) DO UPDATE
4937
+ SET generation = excluded.generation,
4938
+ state = excluded.state,
4939
+ cron = excluded.cron,
4940
+ timezone = excluded.timezone,
4941
+ current_due_at = excluded.current_due_at,
4942
+ current_fire_task_id = excluded.current_fire_task_id,
4943
+ current_fire_generation = excluded.current_fire_generation,
4944
+ current_fire_queue_name = excluded.current_fire_queue_name,
4945
+ source_pin_json = excluded.source_pin_json,
4946
+ -- Keep the legacy non-null column free of credentials while old
4947
+ -- schedules drain. New readers use source_pin_json exclusively.
4948
+ launch_json = excluded.launch_json,
4949
+ last_fired_at = NULL,
4950
+ updated_at = now()
4951
+ WHERE schedule.generation < excluded.generation
4952
+ RETURNING schedule_id
4953
+ )
4954
+ SELECT
4955
+ (EXISTS (SELECT 1 FROM schedule_upsert) OR EXISTS (SELECT 1 FROM existing))
4956
+ AS accepted,
4957
+ EXISTS (SELECT 1 FROM schedule_upsert) AS created,
4958
+ (SELECT generation FROM existing) AS existing_generation,
4959
+ (SELECT matches_input FROM existing) AS existing_matches_input,
4960
+ coalesce((SELECT accepting FROM release_gate), false) AS release_accepting,
4961
+ (SELECT task_id FROM task_spawn) AS fire_task_id
4962
+ `,
4963
+ [
4964
+ input.scheduleId.trim(),
4965
+ input.generation,
4966
+ input.dueAtMs,
4967
+ input.cron.trim(),
4968
+ input.timezone?.trim() || null,
4969
+ stringifyPostgresJson(sourcePin),
4970
+ input.queueName,
4971
+ task.taskName,
4972
+ stringifyPostgresJson(task.params),
4973
+ stringifyPostgresJson(task.options),
4974
+ releaseAdmissionKey,
4975
+ ],
4976
+ );
4977
+ const row = result.rows[0];
4978
+ if (!row?.accepted) {
4979
+ throw new Error(
4980
+ `Cron schedule "${input.scheduleId}" was not accepted for generation ${input.generation}.`,
4981
+ );
4982
+ }
4983
+ if (
4984
+ typeof row.existing_generation === 'number' &&
4985
+ row.existing_generation > input.generation
4986
+ ) {
4987
+ throw new Error(
4988
+ `Cron schedule "${input.scheduleId}" is already owned by newer generation ${row.existing_generation}.`,
4989
+ );
4990
+ }
4991
+ if (
4992
+ row.existing_generation === input.generation &&
4993
+ row.existing_matches_input !== true
4994
+ ) {
4995
+ throw new Error(
4996
+ `Cron schedule "${input.scheduleId}" generation ${input.generation} already exists with a different immutable configuration.`,
4997
+ );
4998
+ }
4999
+ return {
5000
+ created: row.created === true,
5001
+ scheduleId: input.scheduleId.trim(),
5002
+ fireTaskId: row.fire_task_id ?? null,
5003
+ };
5004
+ }
5005
+
5006
+ export async function readPostgresSchedulerCronSchedule(
5007
+ client: PostgresSchedulerQueryClient,
5008
+ scheduleId: string,
5009
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
5010
+ ): Promise<PostgresSchedulerCronSchedule | null> {
5011
+ const result = await client.query<{
5012
+ schedule_id: string;
5013
+ generation: number;
5014
+ state: 'active' | 'blocked' | 'cancelled';
5015
+ cron: string;
5016
+ timezone: string | null;
5017
+ due_at_ms: number;
5018
+ current_fire_task_id: string | null;
5019
+ current_fire_generation: number;
5020
+ current_fire_queue_name: string | null;
5021
+ source_pin_json: unknown;
5022
+ launch_json: unknown;
5023
+ }>(
5024
+ `
5025
+ SELECT schedule_id,
5026
+ generation,
5027
+ state,
5028
+ cron,
5029
+ timezone,
5030
+ floor(extract(epoch FROM current_due_at) * 1000)::bigint AS due_at_ms,
5031
+ current_fire_task_id,
5032
+ current_fire_generation,
5033
+ current_fire_queue_name,
5034
+ source_pin_json,
5035
+ launch_json
5036
+ FROM ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
5037
+ WHERE schedule_id = $1::text
5038
+ `,
5039
+ [scheduleId.trim()],
5040
+ );
5041
+ const row = result.rows[0];
5042
+ if (!row) return null;
5043
+ let sourcePin: PostgresSchedulerCronSourcePin;
5044
+ try {
5045
+ sourcePin = row.source_pin_json
5046
+ ? parsePostgresSchedulerCronSourcePin(row.source_pin_json)
5047
+ : postgresSchedulerCronSourcePinFromLaunch(
5048
+ parseLaunchJson(row.launch_json),
5049
+ );
5050
+ } catch (error) {
5051
+ throw new PostgresSchedulerCronScheduleConfigurationError(
5052
+ `Cron schedule "${row.schedule_id}" has malformed pinned source data: ${
5053
+ error instanceof Error ? error.message : String(error)
5054
+ }`,
5055
+ );
5056
+ }
5057
+ return {
5058
+ scheduleId: row.schedule_id,
5059
+ generation: Number(row.generation),
5060
+ state: row.state,
5061
+ cron: row.cron,
5062
+ timezone: row.timezone,
5063
+ dueAtMs: Number(row.due_at_ms),
5064
+ currentFireTaskId: row.current_fire_task_id ?? null,
5065
+ currentFireGeneration: Number(row.current_fire_generation),
5066
+ currentFireQueueName: row.current_fire_queue_name ?? null,
5067
+ sourcePin,
5068
+ };
5069
+ }
5070
+
5071
+ function cronOccurrenceLaunch(input: {
5072
+ schedule: PostgresSchedulerCronSchedule;
5073
+ launch: PlaySchedulerSubmitInput;
5074
+ schedulerSchema?: string | null;
5075
+ absurdReleaseId?: string | null;
5076
+ }): PlaySchedulerSubmitInput {
5077
+ const identity = buildCronFireRunIdentity({
5078
+ scheduleId: input.schedule.scheduleId,
5079
+ generation: input.schedule.generation,
5080
+ dueAtMs: input.schedule.dueAtMs,
5081
+ playName: input.schedule.sourcePin.playName,
5082
+ });
5083
+ const authority = input.launch.runtimeAuthority;
5084
+ if (
5085
+ !authority ||
5086
+ input.launch.runId !== identity.runId ||
5087
+ authority.workflowId !== identity.workflowId ||
5088
+ authority.runId !== identity.runId
5089
+ ) {
5090
+ throw new PostgresSchedulerCronScheduleConfigurationError(
5091
+ `Cron schedule "${input.schedule.scheduleId}" did not receive a fresh authority for its pinned occurrence.`,
5092
+ );
5093
+ }
5094
+ return {
5095
+ ...input.launch,
5096
+ schedulerSchema:
5097
+ input.schedulerSchema ?? input.launch.schedulerSchema ?? null,
5098
+ ...(input.absurdReleaseId?.trim()
5099
+ ? { absurdReleaseId: input.absurdReleaseId.trim() }
5100
+ : {}),
5101
+ runtimeAuthority: authority,
5102
+ runtimeProtocolVersion: protocolForNewPlayRuntimeLaunch(
5103
+ input.launch.runtimeProtocolVersion,
5104
+ ),
5105
+ triggerSource: 'cron',
5106
+ requiresRunCreatedProjection: true,
5107
+ };
5108
+ }
5109
+
5110
+ export type FirePostgresSchedulerCronScheduleInput = {
5111
+ scheduleId: string;
5112
+ generation: number;
5113
+ /** Matches the durable timer-repair fence for this exact occurrence. */
5114
+ fireGeneration: number;
5115
+ dueAtMs: number;
5116
+ queueName: string;
5117
+ /** Built at the due instant by Vercel; never reuse the seed-time authority. */
5118
+ launch: PlaySchedulerSubmitInput;
5119
+ /** Current accepting lane, resolved by the waking worker rather than seed. */
5120
+ absurdReleaseId?: string | null;
5121
+ /** Environment scopes the release-admission fence. */
5122
+ releaseEnvironment?: 'preview' | 'production';
5123
+ };
5124
+
5125
+ /**
5126
+ * The cron OCC boundary. The matching schedule row is locked while this one
5127
+ * CTE inserts the immutable Run, its play-run task, run.created fact, next
5128
+ * cron-fire task, and next cursor. A task replay after commit sees a changed
5129
+ * cursor and is a harmless `superseded` no-op.
5130
+ */
5131
+ export async function firePostgresSchedulerCronSchedule(
5132
+ client: PostgresSchedulerQueryClient,
5133
+ input: FirePostgresSchedulerCronScheduleInput,
5134
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
5135
+ ): Promise<
5136
+ | { status: 'admitted'; runId: string; successorDueAtMs: number }
5137
+ | { status: 'superseded' }
5138
+ > {
5139
+ assertPostgresSchedulerCronScheduleInput(input);
5140
+ const schedule = await readPostgresSchedulerCronSchedule(
5141
+ client,
5142
+ input.scheduleId,
5143
+ options,
5144
+ );
5145
+ if (
5146
+ !schedule ||
5147
+ schedule.state !== 'active' ||
5148
+ schedule.generation !== input.generation ||
5149
+ schedule.dueAtMs !== input.dueAtMs ||
5150
+ schedule.currentFireGeneration !== input.fireGeneration
5151
+ ) {
5152
+ return { status: 'superseded' };
5153
+ }
5154
+ let successorDueAtMs: number;
5155
+ try {
5156
+ successorDueAtMs = advanceWorkflowCronSchedule(
5157
+ schedule.cron,
5158
+ Math.max(Date.now(), schedule.dueAtMs),
5159
+ schedule.timezone,
5160
+ );
5161
+ } catch (error) {
5162
+ throw new PostgresSchedulerCronScheduleConfigurationError(
5163
+ `Cron schedule "${schedule.scheduleId}" has an invalid pinned expression or timezone: ${
5164
+ error instanceof Error ? error.message : String(error)
5165
+ }`,
5166
+ );
5167
+ }
5168
+ if (
5169
+ !Number.isSafeInteger(successorDueAtMs) ||
5170
+ successorDueAtMs <= schedule.dueAtMs
5171
+ ) {
5172
+ throw new PostgresSchedulerCronScheduleConfigurationError(
5173
+ `Cron schedule "${schedule.scheduleId}" did not produce a future successor occurrence.`,
5174
+ );
5175
+ }
5176
+ const launch = cronOccurrenceLaunch({
5177
+ schedule,
5178
+ launch: input.launch,
5179
+ schedulerSchema: options?.schema,
5180
+ absurdReleaseId: input.absurdReleaseId,
5181
+ });
5182
+ const releaseId = launch.absurdReleaseId?.trim();
5183
+ if (!releaseId) {
5184
+ throw new PostgresSchedulerCronScheduleConfigurationError(
5185
+ `Cron schedule "${schedule.scheduleId}" has no fresh Absurd release lane.`,
5186
+ );
5187
+ }
5188
+ const releaseEnvironment =
5189
+ input.releaseEnvironment ?? launch.absurdReleaseEnvironment ?? 'production';
5190
+ const releaseAdmissionKey = `${releaseEnvironment}:${releaseId}`;
5191
+ const playTask = buildAbsurdPlayRunTask({
5192
+ launch,
5193
+ schedulerSchema: options?.schema,
5194
+ });
5195
+ const successorTask = buildAbsurdCronFireTask({
5196
+ scheduleId: schedule.scheduleId,
5197
+ generation: schedule.generation,
5198
+ fireGeneration: 1,
5199
+ dueAtMs: successorDueAtMs,
5200
+ schedulerSchema: options?.schema ?? launch.schedulerSchema ?? null,
5201
+ orgId: launch.orgId,
5202
+ });
5203
+ const result = await client.query<{
5204
+ admitted: boolean;
5205
+ release_accepting: boolean;
5206
+ }>(
5207
+ `
5208
+ WITH schedule_lock AS MATERIALIZED (
5209
+ SELECT pg_advisory_xact_lock(
5210
+ ${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
5211
+ ${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
5212
+ 'cron-schedule:' || $1::text
5213
+ )
5214
+ )
5215
+ ), matching_schedule AS MATERIALIZED (
5216
+ SELECT schedule.schedule_id
5217
+ FROM schedule_lock
5218
+ JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
5219
+ ON schedule.schedule_id = $1::text
5220
+ JOIN ${tableName('runtime_release_admission', options)} AS release_gate
5221
+ ON release_gate.release_id = $16::text
5222
+ WHERE schedule.generation = $2::bigint
5223
+ AND schedule.state = 'active'
5224
+ AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
5225
+ AND schedule.current_fire_generation = $15::bigint
5226
+ AND release_gate.accepting
5227
+ FOR UPDATE
5228
+ ), existing_run AS MATERIALIZED (
5229
+ SELECT runs.run_id
5230
+ FROM matching_schedule
5231
+ JOIN ${tableName('runs', options)} AS runs ON runs.run_id = $5::text
5232
+ ), play_task_request AS (
5233
+ SELECT $7::text AS queue_name,
5234
+ $8::text AS task_name,
5235
+ $9::jsonb AS params,
5236
+ $10::jsonb AS options
5237
+ WHERE EXISTS (SELECT 1 FROM matching_schedule)
5238
+ AND NOT EXISTS (SELECT 1 FROM existing_run)
5239
+ ), play_task_spawn AS MATERIALIZED (
5240
+ SELECT spawned.task_id::text AS task_id
5241
+ FROM play_task_request
5242
+ CROSS JOIN LATERAL absurd.spawn_task(
5243
+ play_task_request.queue_name,
5244
+ play_task_request.task_name,
5245
+ play_task_request.params,
5246
+ play_task_request.options
5247
+ ) AS spawned
5248
+ ), run_insert AS (
5249
+ INSERT INTO ${tableName('runs', options)} (
5250
+ run_id, org_id, play_id, play_name, graph_hash, runtime_profile,
5251
+ artifact_storage_key, launch_json, input_json, status,
5252
+ absurd_task_id, admission_json
5253
+ )
5254
+ SELECT $5::text,
5255
+ $6::jsonb ->> 'orgId',
5256
+ NULLIF($6::jsonb ->> 'playId', ''),
5257
+ $6::jsonb ->> 'playName',
5258
+ $6::jsonb ->> 'graphHash',
5259
+ $6::jsonb ->> 'runtimeBackend',
5260
+ $6::jsonb ->> 'artifactStorageKey',
5261
+ $6::jsonb,
5262
+ coalesce($6::jsonb -> 'input', '{}'::jsonb),
5263
+ 'queued',
5264
+ (SELECT task_id FROM play_task_spawn),
5265
+ $6::jsonb - ARRAY[
5266
+ 'executorToken',
5267
+ 'preloadedDbSessionUnwrapKey',
5268
+ 'preloadedDbSessions',
5269
+ 'submittedAtMs',
5270
+ 'runtimeDeployVersion',
5271
+ 'coordinatorInternalToken',
5272
+ 'vercelProtectionBypassToken'
5273
+ ]::text[]
5274
+ WHERE EXISTS (SELECT 1 FROM matching_schedule)
5275
+ AND NOT EXISTS (SELECT 1 FROM existing_run)
5276
+ RETURNING run_id
5277
+ ), successor_task_request AS (
5278
+ SELECT $11::text AS queue_name,
5279
+ $12::text AS task_name,
5280
+ $13::jsonb AS params,
5281
+ $14::jsonb AS options
5282
+ WHERE EXISTS (SELECT 1 FROM run_insert)
5283
+ ), successor_task_spawn AS MATERIALIZED (
5284
+ SELECT spawned.task_id::text AS task_id
5285
+ FROM successor_task_request
5286
+ CROSS JOIN LATERAL absurd.spawn_task(
5287
+ successor_task_request.queue_name,
5288
+ successor_task_request.task_name,
5289
+ successor_task_request.params,
5290
+ successor_task_request.options
5291
+ ) AS spawned
5292
+ ), outbox_insert AS (
5293
+ INSERT INTO ${tableName('outbox', options)} (
5294
+ event_id, run_id, event_type, payload_json, created_at
5295
+ )
5296
+ SELECT
5297
+ 'run-created:' || run_insert.run_id,
5298
+ run_insert.run_id,
5299
+ 'run.created',
5300
+ jsonb_strip_nulls(jsonb_build_object(
5301
+ 'orgId', $6::jsonb ->> 'orgId',
5302
+ 'playId', $6::jsonb ->> 'playId',
5303
+ 'playName', $6::jsonb ->> 'playName',
5304
+ 'definitionName', $6::jsonb ->> 'definitionName',
5305
+ 'playReference', $6::jsonb ->> 'playReference',
5306
+ 'definitionScope', $6::jsonb ->> 'definitionScope',
5307
+ 'revisionId', $6::jsonb ->> 'revisionId',
5308
+ 'workflowFamilyKey', $6::jsonb ->> 'workflowFamilyKey',
5309
+ 'artifactStorageKey', $6::jsonb ->> 'artifactStorageKey',
5310
+ 'artifactHash', $6::jsonb ->> 'artifactHash',
5311
+ 'graphHash', $6::jsonb ->> 'graphHash',
5312
+ 'runtimeBackend', $6::jsonb ->> 'runtimeBackend',
5313
+ 'schedulerBackend', $6::jsonb ->> 'schedulerBackend',
5314
+ 'schedulerSchema', $6::jsonb ->> 'schedulerSchema',
5315
+ 'executionProfile', $6::jsonb ->> 'executionProfile',
5316
+ 'runtimeReleaseId', $6::jsonb ->> 'runtimeReleaseId',
5317
+ 'runtimeDeployVersion', $6::jsonb ->> 'runtimeDeployVersion',
5318
+ 'runtimeCallbackBaseUrl', $6::jsonb ->> 'runtimeCallbackBaseUrl',
5319
+ 'coordinatorUrl', $6::jsonb ->> 'coordinatorUrl',
5320
+ 'coordinatorWorkerName', $6::jsonb ->> 'coordinatorWorkerName',
5321
+ 'runtimeWorkflowName', $6::jsonb ->> 'runtimeWorkflowName',
5322
+ 'runtimeHarnessWorkerName', $6::jsonb ->> 'runtimeHarnessWorkerName',
5323
+ 'maxCreditsPerRun', $6::jsonb -> 'maxCreditsPerRun',
5324
+ 'triggerSource', $6::jsonb ->> 'triggerSource',
5325
+ 'source', $6::jsonb ->> 'source',
5326
+ 'staticPipeline', $6::jsonb -> 'staticPipeline',
5327
+ 'inputFileId', $6::jsonb ->> 'inputFileId',
5328
+ 'inputBytes', $6::jsonb -> 'inputBytes',
5329
+ 'inputSha256', $6::jsonb ->> 'inputSha256',
5330
+ 'replayedFromRunId', $6::jsonb ->> 'replayedFromRunId',
5331
+ 'parentRunId', $6::jsonb ->> 'parentRunId',
5332
+ 'rootRunId', $6::jsonb ->> 'rootRunId',
5333
+ 'asyncLaunchKey', $6::jsonb ->> 'asyncLaunchKey',
5334
+ 'asyncInvocationKey', $6::jsonb ->> 'asyncInvocationKey',
5335
+ 'asyncRequestFingerprint', $6::jsonb ->> 'asyncRequestFingerprint',
5336
+ 'asyncAncestryPlayIds', $6::jsonb -> 'asyncAncestryPlayIds',
5337
+ 'secretRefs', $6::jsonb -> 'secretRefs',
5338
+ 'inputSummary', $6::jsonb -> 'inputSummary'
3991
5339
  )),
3992
5340
  ${causalOutboxCreatedAt('run_insert.run_id', options)}
3993
5341
  FROM run_insert
3994
- -- Old Vercel launchers do not carry the immutable admission envelope.
3995
- -- During a Convex-first rollout they retain their established start
3996
- -- projection; only the new launcher opts into queued projection.
3997
- WHERE $8::jsonb ? 'playReference'
3998
5342
  ON CONFLICT (event_id) DO NOTHING
3999
- RETURNING run_id
5343
+ ), schedule_advance AS (
5344
+ UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
5345
+ SET current_due_at = to_timestamp($4::double precision / 1000.0),
5346
+ current_fire_task_id = (SELECT task_id FROM successor_task_spawn),
5347
+ current_fire_generation = 1,
5348
+ current_fire_queue_name = $11::text,
5349
+ last_fired_at = clock_timestamp(),
5350
+ updated_at = now()
5351
+ WHERE schedule.schedule_id = $1::text
5352
+ AND schedule.generation = $2::bigint
5353
+ AND schedule.state = 'active'
5354
+ AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
5355
+ AND schedule.current_fire_generation = $15::bigint
5356
+ AND EXISTS (SELECT 1 FROM run_insert)
5357
+ RETURNING schedule_id
4000
5358
  )
4001
- SELECT (
4002
- (NOT admission.existing_run AND admission.accepting_new
4003
- AND admission.parent_accepting)
4004
- OR admission.matching_existing
4005
- ) AS accepted,
4006
- admission.accepting_new,
4007
- admission.parent_accepting,
4008
- admission.parent_attempt_current,
4009
- admission.parent_terminal,
4010
- admission.existing_run,
4011
- admission.matching_existing,
5359
+ SELECT EXISTS (SELECT 1 FROM schedule_advance) AS admitted,
4012
5360
  coalesce(
4013
- (SELECT task_id FROM task_spawn),
4014
- (SELECT absurd_task_id FROM existing_run)
4015
- ) AS absurd_task_id,
4016
- EXISTS (SELECT 1 FROM run_insert) AS run_created,
4017
- (SELECT created FROM task_spawn) AS task_created,
4018
- (
4019
- admission.matching_existing
4020
- AND NOT ${recoverExistingTaskSql}
4021
- AND EXISTS (
4022
- SELECT 1
4023
- FROM existing_run
4024
- WHERE existing_run.status = 'queued'
4025
- AND existing_run.absurd_task_id IS NULL
4026
- )
4027
- ) AS task_recovery_required,
4028
- CASE WHEN EXISTS (SELECT 1 FROM run_insert)
4029
- OR EXISTS (SELECT 1 FROM run_recovery)
4030
- THEN pg_notify($10, $11)
4031
- END
4032
- FROM admission
5361
+ (SELECT accepting
5362
+ FROM ${tableName('runtime_release_admission', options)}
5363
+ WHERE release_id = $16::text),
5364
+ false
5365
+ ) AS release_accepting
4033
5366
  `,
4034
5367
  [
4035
- input.runId,
4036
- input.orgId,
4037
- input.playId,
4038
- input.playName,
4039
- input.graphHash,
4040
- input.runtimeBackend,
4041
- input.artifactStorageKey,
4042
- stringifyPostgresJson(input),
4043
- stringifyPostgresJson(input.input ?? {}),
4044
- postgresSchedulerNotifyChannel('queue', options),
4045
- JSON.stringify({ runId: input.runId, playName: input.playName }),
5368
+ schedule.scheduleId,
5369
+ schedule.generation,
5370
+ schedule.dueAtMs,
5371
+ successorDueAtMs,
5372
+ launch.runId,
5373
+ stringifyPostgresJson(launch),
5374
+ input.queueName,
5375
+ playTask.taskName,
5376
+ stringifyPostgresJson(playTask.params),
5377
+ stringifyPostgresJson(playTask.options),
5378
+ input.queueName,
5379
+ successorTask.taskName,
5380
+ stringifyPostgresJson(successorTask.params),
5381
+ stringifyPostgresJson(successorTask.options),
5382
+ schedule.currentFireGeneration,
4046
5383
  releaseAdmissionKey,
4047
- releaseAdmissionLockKey,
4048
- useLegacyAdmissionKey,
4049
- releaseId,
4050
- parentRunId,
4051
- asyncRootRunId,
4052
- asyncParentRunAttempt,
4053
- asyncParentRunLeaseToken,
4054
- asyncParentAttemptRequired,
4055
- ...(absurdTask
4056
- ? [
4057
- absurdTask.queueName,
4058
- absurdTask.taskName,
4059
- stringifyPostgresJson(absurdTask.params),
4060
- stringifyPostgresJson(absurdTask.options),
4061
- ]
4062
- : []),
4063
5384
  ],
4064
5385
  );
4065
- const row = submitted.rows[0];
4066
- if (
4067
- row?.accepted === false &&
4068
- !row.existing_run &&
4069
- row.parent_accepting === false
4070
- ) {
4071
- if (
4072
- row?.parent_terminal === false &&
4073
- row?.parent_attempt_current === false
4074
- ) {
4075
- throw new Error(
4076
- `CTX_RUN_PLAY_ASYNC_PARENT_ATTEMPT_STALE: parent Run "${parentRunId}" no longer has the executor attempt that requested this child.`,
4077
- );
4078
- }
5386
+ if (result.rows[0]?.release_accepting !== true) {
4079
5387
  throw new Error(
4080
- `CTX_RUN_PLAY_ASYNC_PARENT_TERMINAL: parent Run "${parentRunId}" is terminal, missing, belongs to another organization, or is pinned to a different release.`,
5388
+ `Cron schedule "${schedule.scheduleId}" cannot fire because Absurd release "${releaseAdmissionKey}" is not accepting admissions.`,
4081
5389
  );
4082
5390
  }
4083
- if (row?.accepted === false && row.existing_run && !row.matching_existing) {
5391
+ if (result.rows[0]?.admitted !== true) {
5392
+ return { status: 'superseded' };
5393
+ }
5394
+ return {
5395
+ status: 'admitted',
5396
+ runId: launch.runId,
5397
+ successorDueAtMs,
5398
+ };
5399
+ }
5400
+
5401
+ /** Fence a malformed/invalid timer so it cannot repeatedly consume workers. */
5402
+ export async function blockPostgresSchedulerCronSchedule(
5403
+ client: PostgresSchedulerQueryClient,
5404
+ input: Pick<
5405
+ FirePostgresSchedulerCronScheduleInput,
5406
+ 'scheduleId' | 'generation' | 'fireGeneration' | 'dueAtMs'
5407
+ > & {
5408
+ reason: string;
5409
+ },
5410
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
5411
+ ): Promise<boolean> {
5412
+ const result = await client.query<{ schedule_id: string }>(
5413
+ `
5414
+ UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
5415
+ SET state = 'blocked',
5416
+ block_reason = left($4::text, 1_000),
5417
+ updated_at = now()
5418
+ WHERE schedule_id = $1::text
5419
+ AND generation = $2::bigint
5420
+ AND state = 'active'
5421
+ AND current_due_at = to_timestamp($3::double precision / 1000.0)
5422
+ AND current_fire_generation = $5::bigint
5423
+ RETURNING schedule_id
5424
+ `,
5425
+ [
5426
+ input.scheduleId,
5427
+ input.generation,
5428
+ input.dueAtMs,
5429
+ input.reason,
5430
+ input.fireGeneration,
5431
+ ],
5432
+ );
5433
+ return result.rows.length > 0;
5434
+ }
5435
+
5436
+ /**
5437
+ * Fence a Runtime-owned schedule before its source binding is disabled,
5438
+ * archived, suspended, or replaced. The advisory lock is shared with fire and
5439
+ * seed, so an already-sleeping task either commits before the lifecycle write
5440
+ * (a run valid at that instant) or observes `cancelled` and becomes a
5441
+ * superseded no-op. It can never admit after this mutation commits.
5442
+ */
5443
+ export async function cancelPostgresSchedulerCronSchedule(
5444
+ client: PostgresSchedulerQueryClient,
5445
+ input: {
5446
+ scheduleId: string;
5447
+ generation: number;
5448
+ reason: string;
5449
+ },
5450
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
5451
+ ): Promise<boolean> {
5452
+ if (!input.scheduleId.trim()) {
4084
5453
  throw new Error(
4085
- `Play run "${input.runId}" already exists with a different launch.`,
5454
+ 'Cron schedule cancellation requires a non-empty scheduleId.',
4086
5455
  );
4087
5456
  }
4088
- if (row?.accepted === false) {
5457
+ if (!Number.isSafeInteger(input.generation) || input.generation <= 0) {
4089
5458
  throw new Error(
4090
- `Absurd runtime release "${releaseId}" no longer accepts new launches.`,
5459
+ 'Cron schedule cancellation requires a positive generation.',
4091
5460
  );
4092
5461
  }
5462
+ const result = await client.query<{ schedule_id: string }>(
5463
+ `
5464
+ WITH schedule_lock AS MATERIALIZED (
5465
+ SELECT pg_advisory_xact_lock(
5466
+ ${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
5467
+ ${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
5468
+ 'cron-schedule:' || $1::text
5469
+ )
5470
+ )
5471
+ )
5472
+ UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
5473
+ SET state = 'cancelled',
5474
+ cancellation_reason = left($3::text, 1_000),
5475
+ updated_at = now()
5476
+ FROM schedule_lock
5477
+ WHERE schedule.schedule_id = $1::text
5478
+ AND schedule.generation = $2::bigint
5479
+ AND schedule.state = 'active'
5480
+ RETURNING schedule.schedule_id
5481
+ `,
5482
+ [input.scheduleId.trim(), input.generation, input.reason],
5483
+ );
5484
+ return result.rows.length > 0;
5485
+ }
5486
+
5487
+ /**
5488
+ * Replace an observed missing, cancelled, or exhausted timer without changing
5489
+ * its scheduled occurrence. Callers must first inspect the engine task and
5490
+ * pass the observed `fireGeneration`; the CTE then advances that fence and
5491
+ * spawns the replacement in one transaction. A late old timer is superseded
5492
+ * by the generation comparison in `firePostgresSchedulerCronSchedule`.
5493
+ */
5494
+ export async function repairPostgresSchedulerCronFire(
5495
+ client: PostgresSchedulerQueryClient,
5496
+ input: {
5497
+ scheduleId: string;
5498
+ generation: number;
5499
+ fireGeneration: number;
5500
+ dueAtMs: number;
5501
+ queueName: string;
5502
+ },
5503
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
5504
+ ): Promise<{
5505
+ repaired: boolean;
5506
+ fireTaskId: string | null;
5507
+ fireGeneration: number | null;
5508
+ }> {
5509
+ assertPostgresSchedulerCronScheduleInput(input);
5510
+ const schedule = await readPostgresSchedulerCronSchedule(
5511
+ client,
5512
+ input.scheduleId,
5513
+ options,
5514
+ );
5515
+ if (
5516
+ !schedule ||
5517
+ schedule.state !== 'active' ||
5518
+ schedule.generation !== input.generation ||
5519
+ schedule.dueAtMs !== input.dueAtMs ||
5520
+ schedule.currentFireGeneration !== input.fireGeneration
5521
+ ) {
5522
+ return { repaired: false, fireTaskId: null, fireGeneration: null };
5523
+ }
5524
+ const replacementGeneration = schedule.currentFireGeneration + 1;
5525
+ const task = buildAbsurdCronFireTask({
5526
+ scheduleId: schedule.scheduleId,
5527
+ generation: schedule.generation,
5528
+ fireGeneration: replacementGeneration,
5529
+ dueAtMs: schedule.dueAtMs,
5530
+ schedulerSchema:
5531
+ options?.schema ?? schedule.sourcePin.schedulerSchema ?? null,
5532
+ orgId: schedule.sourcePin.orgId,
5533
+ });
5534
+ const result = await client.query<{
5535
+ repaired: boolean;
5536
+ fire_task_id: string | null;
5537
+ }>(
5538
+ `
5539
+ WITH schedule_lock AS MATERIALIZED (
5540
+ SELECT pg_advisory_xact_lock(
5541
+ ${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
5542
+ ${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
5543
+ 'cron-schedule:' || $1::text
5544
+ )
5545
+ )
5546
+ ), matching_schedule AS MATERIALIZED (
5547
+ SELECT schedule.schedule_id
5548
+ FROM schedule_lock
5549
+ JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
5550
+ ON schedule.schedule_id = $1::text
5551
+ WHERE schedule.generation = $2::bigint
5552
+ AND schedule.state = 'active'
5553
+ AND schedule.current_due_at = to_timestamp($4::double precision / 1000.0)
5554
+ AND schedule.current_fire_generation = $3::bigint
5555
+ FOR UPDATE
5556
+ ), task_spawn AS MATERIALIZED (
5557
+ SELECT spawned.task_id::text AS task_id
5558
+ FROM matching_schedule
5559
+ CROSS JOIN LATERAL absurd.spawn_task(
5560
+ $5::text,
5561
+ $6::text,
5562
+ $7::jsonb,
5563
+ $8::jsonb
5564
+ ) AS spawned
5565
+ ), repaired_schedule AS (
5566
+ UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
5567
+ SET current_fire_task_id = (SELECT task_id FROM task_spawn),
5568
+ current_fire_generation = $9::bigint,
5569
+ current_fire_queue_name = $5::text,
5570
+ updated_at = now()
5571
+ WHERE schedule.schedule_id = $1::text
5572
+ AND schedule.generation = $2::bigint
5573
+ AND schedule.state = 'active'
5574
+ AND schedule.current_due_at = to_timestamp($4::double precision / 1000.0)
5575
+ AND schedule.current_fire_generation = $3::bigint
5576
+ AND EXISTS (SELECT 1 FROM task_spawn)
5577
+ RETURNING schedule.schedule_id
5578
+ )
5579
+ SELECT EXISTS (SELECT 1 FROM repaired_schedule) AS repaired,
5580
+ (SELECT task_id FROM task_spawn) AS fire_task_id
5581
+ `,
5582
+ [
5583
+ schedule.scheduleId,
5584
+ schedule.generation,
5585
+ schedule.currentFireGeneration,
5586
+ schedule.dueAtMs,
5587
+ input.queueName,
5588
+ task.taskName,
5589
+ stringifyPostgresJson(task.params),
5590
+ stringifyPostgresJson(task.options),
5591
+ replacementGeneration,
5592
+ ],
5593
+ );
5594
+ const row = result.rows[0];
4093
5595
  return {
4094
- absurdTaskId: row?.absurd_task_id ?? null,
4095
- runCreated: row?.run_created === true,
4096
- taskCreated: row?.task_created === true,
4097
- taskRecoveryRequired: row?.task_recovery_required === true,
5596
+ repaired: row?.repaired === true,
5597
+ fireTaskId: row?.fire_task_id ?? null,
5598
+ fireGeneration: row?.repaired === true ? replacementGeneration : null,
4098
5599
  };
4099
5600
  }
4100
5601
 
@@ -5923,6 +7424,58 @@ export async function readPostgresSchedulerLaunch(
5923
7424
  }
5924
7425
  }
5925
7426
 
7427
+ /**
7428
+ * One round-trip replay inspection for externally retried admissions. The
7429
+ * caller uses this only to return an already-committed Run without rebuilding
7430
+ * mutable launch dependencies; it is never an admission authority.
7431
+ */
7432
+ export type PostgresSchedulerAdmissionSnapshot = {
7433
+ runId: string;
7434
+ /** Empty only for historic rows that predate an owning organization. */
7435
+ orgId: string;
7436
+ status: string;
7437
+ absurdTaskId: string | null;
7438
+ launch: PlaySchedulerSubmitInput;
7439
+ };
7440
+
7441
+ export async function readPostgresSchedulerAdmission(
7442
+ client: PostgresSchedulerQueryClient,
7443
+ runId: string,
7444
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
7445
+ ): Promise<PostgresSchedulerAdmissionSnapshot | null> {
7446
+ const result = await client.query<{
7447
+ run_id: string;
7448
+ org_id: string | null;
7449
+ status: string;
7450
+ absurd_task_id: string | null;
7451
+ launch_json: unknown;
7452
+ }>(
7453
+ `
7454
+ SELECT runs.run_id,
7455
+ runs.org_id,
7456
+ runs.status,
7457
+ runs.absurd_task_id,
7458
+ runs.launch_json || coalesce(runs.admission_json, '{}'::jsonb)
7459
+ AS launch_json
7460
+ FROM ${tableName('runs', options)} AS runs
7461
+ WHERE runs.run_id = $1
7462
+ `,
7463
+ [runId],
7464
+ );
7465
+ const row = result.rows[0] ?? null;
7466
+ if (!row) return null;
7467
+ return {
7468
+ runId: row.run_id,
7469
+ orgId: row.org_id ?? '',
7470
+ status: row.status,
7471
+ absurdTaskId:
7472
+ typeof row.absurd_task_id === 'string' && row.absurd_task_id.trim()
7473
+ ? row.absurd_task_id
7474
+ : null,
7475
+ launch: parseLaunchJson(row.launch_json),
7476
+ };
7477
+ }
7478
+
5926
7479
  /**
5927
7480
  * Resolve the immutable launch only when the caller proves ownership of the
5928
7481
  * currently live scheduler attempt. Used by the Vercel token issuer so a Fly
@@ -14411,6 +15964,40 @@ function parseLaunchJson(value: unknown): PlaySchedulerSubmitInput {
14411
15964
  return launch as PlaySchedulerSubmitInput;
14412
15965
  }
14413
15966
 
15967
+ function parsePostgresSchedulerCronSourcePin(
15968
+ value: unknown,
15969
+ ): PostgresSchedulerCronSourcePin {
15970
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
15971
+ throw new Error('Cron source pin is not an object.');
15972
+ }
15973
+ const pin = value as Partial<PostgresSchedulerCronSourcePin>;
15974
+ if (
15975
+ typeof pin.orgId !== 'string' ||
15976
+ typeof pin.playName !== 'string' ||
15977
+ typeof pin.artifactStorageKey !== 'string' ||
15978
+ !('input' in pin) ||
15979
+ (pin.source !== 'published' && pin.source !== 'draft')
15980
+ ) {
15981
+ throw new Error('Cron source pin is missing required immutable fields.');
15982
+ }
15983
+ return {
15984
+ orgId: pin.orgId,
15985
+ userId: typeof pin.userId === 'string' ? pin.userId : null,
15986
+ playName: pin.playName,
15987
+ revisionId: typeof pin.revisionId === 'string' ? pin.revisionId : null,
15988
+ artifactStorageKey: pin.artifactStorageKey,
15989
+ artifactKind: pin.artifactKind === 'cjs_node20' ? 'cjs_node20' : null,
15990
+ artifactHash:
15991
+ typeof pin.artifactHash === 'string' ? pin.artifactHash : null,
15992
+ graphHash: typeof pin.graphHash === 'string' ? pin.graphHash : null,
15993
+ staticPipeline: pin.staticPipeline ?? null,
15994
+ input: pin.input as PlaySchedulerSubmitInput['input'],
15995
+ source: pin.source,
15996
+ schedulerSchema:
15997
+ typeof pin.schedulerSchema === 'string' ? pin.schedulerSchema : null,
15998
+ };
15999
+ }
16000
+
14414
16001
  function errorMessage(error: unknown): string {
14415
16002
  if (error && typeof error === 'object') {
14416
16003
  const message = (error as { message?: unknown }).message;