deepline 0.3.113 → 0.3.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +3 -1
- package/dist/bundling-sources/sdk/src/compat.ts +167 -11
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/skills-version.ts +27 -1
- package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +6 -2
- package/dist/bundling-sources/shared_libs/observability/dlq.ts +942 -0
- package/dist/bundling-sources/shared_libs/observability/queue-health.ts +361 -6
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +24 -24
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +2 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +29 -1
- package/dist/bundling-sources/shared_libs/play-runtime/admission-transport-policy.ts +91 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/direct-admission-deadline.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +325 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-admission-outcome.ts +134 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-webhook-admission-probe.ts +32 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-webhook-identity.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-cron-fire-task.ts +100 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +313 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +111 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +67 -8
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1734 -147
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +74 -13
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts +245 -81
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +79 -15
- package/dist/bundling-sources/shared_libs/play-runtime/webhook-receipt-retry.ts +21 -21
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +4 -1
- package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +64 -0
- package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/product-notifications/events.ts +20 -1
- package/dist/bundling-sources/shared_libs/security/receipt-admission-token.ts +224 -0
- package/dist/cli/index.js +574 -114
- package/dist/cli/index.mjs +592 -133
- package/dist/index.d.mts +54 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.js +8 -3
- package/dist/index.mjs +8 -3
- package/dist/install-integrity.json +8 -0
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -159,6 +159,11 @@ import {
|
|
|
159
159
|
} from './scheduler-launch-read-observability';
|
|
160
160
|
import { emitPlayTelemetry } from '../runtime-telemetry';
|
|
161
161
|
import { buildAbsurdPlayRunTask } from './absurd-play-run-task';
|
|
162
|
+
import {
|
|
163
|
+
buildAbsurdCronFireTask,
|
|
164
|
+
buildCronFireRunIdentity,
|
|
165
|
+
} from './absurd-cron-fire-task';
|
|
166
|
+
import { advanceWorkflowCronSchedule } from '@deepline/workflows/scheduler';
|
|
162
167
|
|
|
163
168
|
type QueryResult<
|
|
164
169
|
Row extends Record<string, unknown> = Record<string, unknown>,
|
|
@@ -212,16 +217,24 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
|
|
|
212
217
|
* interrupted deployment cannot claim a newer version without its required
|
|
213
218
|
* objects.
|
|
214
219
|
*/
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
|
|
220
|
+
// v38 adds bounded health indexes for the runtime-owned outbox, sandbox
|
|
221
|
+
// cleanup, and compute-billing queues. The indexes are additive: v37 cron
|
|
222
|
+
// scheduling, admission, and outbox projection remain valid while a deploy
|
|
223
|
+
// expands the shared schema, then the marker records the completed health
|
|
224
|
+
// capability for deployment tooling.
|
|
225
|
+
export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 38;
|
|
219
226
|
const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
|
|
220
227
|
'scheduler_schema_migrations';
|
|
221
228
|
export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
|
|
222
229
|
'runtime_scheduler_runs_open_recovery_idx';
|
|
223
230
|
const POSTGRES_SCHEDULER_DEPENDENCY_CIRCUITS_TABLE =
|
|
224
231
|
'runtime_dependency_circuits';
|
|
232
|
+
/**
|
|
233
|
+
* Runtime-owned cursor for a scheduled Play. Convex remains the authored
|
|
234
|
+
* control-plane projection; this row is the execution owner once a binding is
|
|
235
|
+
* cut over, so a Run and its successor can commit together.
|
|
236
|
+
*/
|
|
237
|
+
const POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE = 'cron_schedules';
|
|
225
238
|
const RECOVERY_CIRCUIT_OPEN_MS =
|
|
226
239
|
RUNTIME_RELIABILITY_POLICY.recovery.circuitOpenMs;
|
|
227
240
|
const RECOVERY_CIRCUIT_PROBE_LEASE_MS =
|
|
@@ -992,6 +1005,7 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
|
|
|
992
1005
|
admissionPriorityIndex?: boolean;
|
|
993
1006
|
terminalComputeBillingOutboxTrigger?: boolean;
|
|
994
1007
|
outboxWakeTrigger?: boolean;
|
|
1008
|
+
cronScheduleSchema?: boolean;
|
|
995
1009
|
}>;
|
|
996
1010
|
|
|
997
1011
|
export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
@@ -1240,6 +1254,49 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1240
1254
|
AND to_regclass($20) IS NOT NULL
|
|
1241
1255
|
AND to_regclass($21) IS NOT NULL
|
|
1242
1256
|
AND to_regclass($22) IS NOT NULL
|
|
1257
|
+
AND (
|
|
1258
|
+
$25::boolean = false
|
|
1259
|
+
OR (
|
|
1260
|
+
to_regclass($26) IS NOT NULL
|
|
1261
|
+
AND (
|
|
1262
|
+
SELECT count(DISTINCT column_name)
|
|
1263
|
+
FROM information_schema.columns
|
|
1264
|
+
WHERE table_schema = $1
|
|
1265
|
+
AND table_name = '${POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE}'
|
|
1266
|
+
AND column_name IN (
|
|
1267
|
+
'schedule_id',
|
|
1268
|
+
'generation',
|
|
1269
|
+
'state',
|
|
1270
|
+
'cron',
|
|
1271
|
+
'timezone',
|
|
1272
|
+
'current_due_at',
|
|
1273
|
+
'current_fire_task_id',
|
|
1274
|
+
'current_fire_generation',
|
|
1275
|
+
'current_fire_queue_name',
|
|
1276
|
+
'last_fired_at',
|
|
1277
|
+
'block_reason',
|
|
1278
|
+
'cancellation_reason',
|
|
1279
|
+
'source_pin_json',
|
|
1280
|
+
'launch_json',
|
|
1281
|
+
'created_at',
|
|
1282
|
+
'updated_at'
|
|
1283
|
+
)
|
|
1284
|
+
) = 16
|
|
1285
|
+
AND EXISTS (
|
|
1286
|
+
SELECT 1
|
|
1287
|
+
FROM pg_index AS index_row
|
|
1288
|
+
JOIN pg_class AS index_class
|
|
1289
|
+
ON index_class.oid = index_row.indexrelid
|
|
1290
|
+
JOIN pg_namespace AS index_namespace
|
|
1291
|
+
ON index_namespace.oid = index_class.relnamespace
|
|
1292
|
+
WHERE index_namespace.nspname = $1
|
|
1293
|
+
AND index_class.relname = 'runtime_scheduler_cron_schedules_due_idx'
|
|
1294
|
+
AND index_row.indrelid = to_regclass($26)
|
|
1295
|
+
AND index_row.indisready
|
|
1296
|
+
AND index_row.indisvalid
|
|
1297
|
+
)
|
|
1298
|
+
)
|
|
1299
|
+
)
|
|
1243
1300
|
-- Canonical Integration Operation outcomes are additive to the
|
|
1244
1301
|
-- transport receipt. Old byte-only rows remain readable by the
|
|
1245
1302
|
-- gateway, but can never be mistaken for a canonical operation.
|
|
@@ -1483,6 +1540,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1483
1540
|
`${quotedSchema}.${quoteIdent('outbox_repair_operations')}`,
|
|
1484
1541
|
requirements.outboxWakeTrigger !== false,
|
|
1485
1542
|
requirements.admissionPriorityIndex !== false,
|
|
1543
|
+
requirements.cronScheduleSchema !== false,
|
|
1544
|
+
`${quotedSchema}.${quoteIdent(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE)}`,
|
|
1486
1545
|
],
|
|
1487
1546
|
);
|
|
1488
1547
|
if (currentShape.rows[0]?.ready !== true) return false;
|
|
@@ -1509,6 +1568,87 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1509
1568
|
AND index_row.indrelid = to_regclass($3)
|
|
1510
1569
|
AND index_row.indisready AND index_row.indisvalid
|
|
1511
1570
|
)
|
|
1571
|
+
AND EXISTS (
|
|
1572
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1573
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1574
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1575
|
+
WHERE index_namespace.nspname = $1
|
|
1576
|
+
AND index_class.relname = 'runtime_scheduler_outbox_ready_health_idx'
|
|
1577
|
+
AND index_row.indrelid = to_regclass($3)
|
|
1578
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1579
|
+
)
|
|
1580
|
+
AND EXISTS (
|
|
1581
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1582
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1583
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1584
|
+
WHERE index_namespace.nspname = $1
|
|
1585
|
+
AND index_class.relname = 'runtime_scheduler_outbox_dead_health_idx'
|
|
1586
|
+
AND index_row.indrelid = to_regclass($3)
|
|
1587
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1588
|
+
)
|
|
1589
|
+
AND EXISTS (
|
|
1590
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1591
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1592
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1593
|
+
WHERE index_namespace.nspname = $1
|
|
1594
|
+
AND index_class.relname = 'runtime_scheduler_outbox_lease_health_idx'
|
|
1595
|
+
AND index_row.indrelid = to_regclass($3)
|
|
1596
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1597
|
+
)
|
|
1598
|
+
AND EXISTS (
|
|
1599
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1600
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1601
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1602
|
+
WHERE index_namespace.nspname = $1
|
|
1603
|
+
AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_ready_health_idx'
|
|
1604
|
+
AND index_row.indrelid = to_regclass($7)
|
|
1605
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1606
|
+
)
|
|
1607
|
+
AND EXISTS (
|
|
1608
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1609
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1610
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1611
|
+
WHERE index_namespace.nspname = $1
|
|
1612
|
+
AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_lease_health_idx'
|
|
1613
|
+
AND index_row.indrelid = to_regclass($7)
|
|
1614
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1615
|
+
)
|
|
1616
|
+
AND EXISTS (
|
|
1617
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1618
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1619
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1620
|
+
WHERE index_namespace.nspname = $1
|
|
1621
|
+
AND index_class.relname = 'runtime_scheduler_compute_billing_ready_health_idx'
|
|
1622
|
+
AND index_row.indrelid = to_regclass($8)
|
|
1623
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1624
|
+
)
|
|
1625
|
+
AND EXISTS (
|
|
1626
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1627
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1628
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1629
|
+
WHERE index_namespace.nspname = $1
|
|
1630
|
+
AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx'
|
|
1631
|
+
AND index_row.indrelid = to_regclass($7)
|
|
1632
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1633
|
+
)
|
|
1634
|
+
AND EXISTS (
|
|
1635
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1636
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1637
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1638
|
+
WHERE index_namespace.nspname = $1
|
|
1639
|
+
AND index_class.relname = 'runtime_scheduler_compute_billing_lease_health_idx'
|
|
1640
|
+
AND index_row.indrelid = to_regclass($8)
|
|
1641
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1642
|
+
)
|
|
1643
|
+
AND EXISTS (
|
|
1644
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1645
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1646
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1647
|
+
WHERE index_namespace.nspname = $1
|
|
1648
|
+
AND index_class.relname = 'runtime_scheduler_compute_billing_blocked_health_idx'
|
|
1649
|
+
AND index_row.indrelid = to_regclass($8)
|
|
1650
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1651
|
+
)
|
|
1512
1652
|
AND (
|
|
1513
1653
|
$5::boolean = false
|
|
1514
1654
|
OR EXISTS (
|
|
@@ -1550,6 +1690,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1550
1690
|
`${quotedSchema}.${quoteIdent('runs')}`,
|
|
1551
1691
|
requirements.deliveredOutboxRetentionIndex !== false,
|
|
1552
1692
|
requirements.blockedOutboxIndex !== false,
|
|
1693
|
+
`${quotedSchema}.${quoteIdent('sandbox_cleanup_jobs_v2')}`,
|
|
1694
|
+
`${quotedSchema}.${quoteIdent('compute_billing_jobs_v2')}`,
|
|
1553
1695
|
],
|
|
1554
1696
|
);
|
|
1555
1697
|
return runtimeAdminIndexes.rows[0]?.ready === true;
|
|
@@ -1712,6 +1854,68 @@ async function ensurePostgresSchedulerOutboxWakeTrigger(
|
|
|
1712
1854
|
`);
|
|
1713
1855
|
}
|
|
1714
1856
|
|
|
1857
|
+
/**
|
|
1858
|
+
* Additive, independent schema for the cron successor owner. It intentionally
|
|
1859
|
+
* does not participate in the global scheduler schema marker: older app and
|
|
1860
|
+
* worker builds neither read nor write this table, and making ordinary Run
|
|
1861
|
+
* admission wait on this rollout would turn an additive timer deployment into
|
|
1862
|
+
* an availability dependency.
|
|
1863
|
+
*/
|
|
1864
|
+
async function ensurePostgresSchedulerCronScheduleSchema(
|
|
1865
|
+
client: PostgresSchedulerQueryClient,
|
|
1866
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
1867
|
+
): Promise<void> {
|
|
1868
|
+
await client.query(`
|
|
1869
|
+
CREATE TABLE IF NOT EXISTS ${tableName(
|
|
1870
|
+
POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE,
|
|
1871
|
+
options,
|
|
1872
|
+
)} (
|
|
1873
|
+
schedule_id text PRIMARY KEY,
|
|
1874
|
+
generation bigint NOT NULL CHECK (generation > 0),
|
|
1875
|
+
state text NOT NULL CHECK (state IN ('active', 'blocked', 'cancelled')),
|
|
1876
|
+
cron text NOT NULL,
|
|
1877
|
+
timezone text,
|
|
1878
|
+
current_due_at timestamptz NOT NULL,
|
|
1879
|
+
current_fire_task_id text,
|
|
1880
|
+
current_fire_generation bigint NOT NULL DEFAULT 1 CHECK (current_fire_generation > 0),
|
|
1881
|
+
current_fire_queue_name text,
|
|
1882
|
+
last_fired_at timestamptz,
|
|
1883
|
+
block_reason text,
|
|
1884
|
+
cancellation_reason text,
|
|
1885
|
+
source_pin_json jsonb,
|
|
1886
|
+
launch_json jsonb NOT NULL,
|
|
1887
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1888
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
1889
|
+
)
|
|
1890
|
+
`);
|
|
1891
|
+
await client.query(`
|
|
1892
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1893
|
+
ADD COLUMN IF NOT EXISTS block_reason text
|
|
1894
|
+
`);
|
|
1895
|
+
await client.query(`
|
|
1896
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1897
|
+
ADD COLUMN IF NOT EXISTS current_fire_generation bigint NOT NULL DEFAULT 1
|
|
1898
|
+
`);
|
|
1899
|
+
await client.query(`
|
|
1900
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1901
|
+
ADD COLUMN IF NOT EXISTS current_fire_queue_name text
|
|
1902
|
+
`);
|
|
1903
|
+
await client.query(`
|
|
1904
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1905
|
+
ADD COLUMN IF NOT EXISTS cancellation_reason text
|
|
1906
|
+
`);
|
|
1907
|
+
await client.query(`
|
|
1908
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1909
|
+
ADD COLUMN IF NOT EXISTS source_pin_json jsonb
|
|
1910
|
+
`);
|
|
1911
|
+
await client.query(`
|
|
1912
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_cron_schedules_due_idx
|
|
1913
|
+
ON ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1914
|
+
(current_due_at, schedule_id)
|
|
1915
|
+
WHERE state = 'active'
|
|
1916
|
+
`);
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1715
1919
|
export async function ensurePostgresSchedulerSchema(
|
|
1716
1920
|
client: PostgresSchedulerQueryClient,
|
|
1717
1921
|
options?: Pick<PostgresSchedulerOptions, 'schema'> & {
|
|
@@ -1817,7 +2021,7 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
1817
2021
|
// trigger as a narrow migration: replaying historical schema DDL in one
|
|
1818
2022
|
// transaction against a live worker fleet can invert table-lock order with
|
|
1819
2023
|
// terminal writers.
|
|
1820
|
-
const
|
|
2024
|
+
const schemaIsCurrentWithoutAdditiveFastPathObjects =
|
|
1821
2025
|
!schemaIsCurrent &&
|
|
1822
2026
|
(await hasCurrentPostgresSchedulerSchemaShape(client, options, {
|
|
1823
2027
|
openRecoveryIndex: !options?.deferOpenRecoveryIndex,
|
|
@@ -1825,17 +2029,10 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
1825
2029
|
admissionPriorityIndex: !options?.deferRuntimeAdminIndexes,
|
|
1826
2030
|
terminalComputeBillingOutboxTrigger: false,
|
|
1827
2031
|
outboxWakeTrigger: false,
|
|
1828
|
-
|
|
1829
|
-
const schemaIsCurrentWithoutTerminalComputeBillingOutboxTrigger =
|
|
1830
|
-
!schemaIsCurrent &&
|
|
1831
|
-
(await hasCurrentPostgresSchedulerSchemaShape(client, options, {
|
|
1832
|
-
openRecoveryIndex: !options?.deferOpenRecoveryIndex,
|
|
1833
|
-
runtimeAdminIndexes: !options?.deferRuntimeAdminIndexes,
|
|
1834
|
-
admissionPriorityIndex: !options?.deferRuntimeAdminIndexes,
|
|
1835
|
-
terminalComputeBillingOutboxTrigger: false,
|
|
2032
|
+
cronScheduleSchema: false,
|
|
1836
2033
|
}));
|
|
1837
2034
|
const schemaCoreIsCurrent =
|
|
1838
|
-
schemaIsCurrent ||
|
|
2035
|
+
schemaIsCurrent || schemaIsCurrentWithoutAdditiveFastPathObjects;
|
|
1839
2036
|
// Older workers can still create a release-only fence after the current
|
|
1840
2037
|
// schema marker has been written. The metadata is current, but the data
|
|
1841
2038
|
// must be copied before a new worker trusts the fast path.
|
|
@@ -1853,13 +2050,14 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
1853
2050
|
const hasLegacyAdmissionRows =
|
|
1854
2051
|
legacyAdmissionRows?.rows[0]?.present === true;
|
|
1855
2052
|
if (schemaCoreIsCurrent) {
|
|
1856
|
-
if (
|
|
2053
|
+
if (!schemaIsCurrent) {
|
|
1857
2054
|
await ensurePostgresSchedulerTerminalComputeBillingOutboxTrigger(
|
|
1858
2055
|
client,
|
|
1859
2056
|
options,
|
|
1860
2057
|
);
|
|
1861
2058
|
}
|
|
1862
2059
|
await ensurePostgresSchedulerOutboxWakeTrigger(client, options);
|
|
2060
|
+
await ensurePostgresSchedulerCronScheduleSchema(client, options);
|
|
1863
2061
|
if (hasLegacyAdmissionRows) {
|
|
1864
2062
|
await migrateLegacyReleaseAdmissionRows(client, options);
|
|
1865
2063
|
}
|
|
@@ -3020,6 +3218,30 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
3020
3218
|
(available_at, created_at, provider, sandbox_id)
|
|
3021
3219
|
WHERE status <> 'deleted'
|
|
3022
3220
|
`);
|
|
3221
|
+
if (!options?.deferRuntimeAdminIndexes) {
|
|
3222
|
+
await client.query(`
|
|
3223
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_ready_health_idx
|
|
3224
|
+
ON ${tableName('sandbox_cleanup_jobs_v2', options)}
|
|
3225
|
+
(available_at, created_at, provider, sandbox_id)
|
|
3226
|
+
WHERE status <> 'deleted'
|
|
3227
|
+
AND blocked_at IS NULL
|
|
3228
|
+
AND lease_owner IS NULL
|
|
3229
|
+
AND status <> 'deleting'
|
|
3230
|
+
`);
|
|
3231
|
+
await client.query(`
|
|
3232
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_lease_health_idx
|
|
3233
|
+
ON ${tableName('sandbox_cleanup_jobs_v2', options)}
|
|
3234
|
+
(lease_expires_at, created_at, provider, sandbox_id)
|
|
3235
|
+
WHERE status <> 'deleted' AND lease_owner IS NOT NULL
|
|
3236
|
+
`);
|
|
3237
|
+
await client.query(`
|
|
3238
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx
|
|
3239
|
+
ON ${tableName('sandbox_cleanup_jobs_v2', options)}
|
|
3240
|
+
(blocked_at, lease_expires_at, created_at, provider, sandbox_id)
|
|
3241
|
+
WHERE status <> 'deleted'
|
|
3242
|
+
AND (blocked_at IS NOT NULL OR status = 'deleting')
|
|
3243
|
+
`);
|
|
3244
|
+
}
|
|
3023
3245
|
await client.query(`
|
|
3024
3246
|
CREATE INDEX IF NOT EXISTS ${POSTGRES_SCHEDULER_COMPUTE_BILLING_PENDING_INDEX}
|
|
3025
3247
|
ON ${tableName('compute_billing_jobs_v2', options)}
|
|
@@ -3073,6 +3295,27 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
3073
3295
|
ON ${tableName('outbox', options)} (created_at, event_id)
|
|
3074
3296
|
WHERE delivered_at IS NULL AND failed_at IS NULL
|
|
3075
3297
|
`);
|
|
3298
|
+
await client.query(`
|
|
3299
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lease_health_idx
|
|
3300
|
+
ON ${tableName('outbox', options)}
|
|
3301
|
+
(lease_expires_at, created_at, event_id)
|
|
3302
|
+
WHERE delivered_at IS NULL
|
|
3303
|
+
AND failed_at IS NULL
|
|
3304
|
+
AND lease_owner IS NOT NULL
|
|
3305
|
+
`);
|
|
3306
|
+
await client.query(`
|
|
3307
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_ready_health_idx
|
|
3308
|
+
ON ${tableName('outbox', options)}
|
|
3309
|
+
(coalesce(next_attempt_at, created_at), created_at, event_id)
|
|
3310
|
+
WHERE delivered_at IS NULL
|
|
3311
|
+
AND failed_at IS NULL
|
|
3312
|
+
AND lease_owner IS NULL
|
|
3313
|
+
`);
|
|
3314
|
+
await client.query(`
|
|
3315
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_dead_health_idx
|
|
3316
|
+
ON ${tableName('outbox', options)} (failed_at, event_id)
|
|
3317
|
+
WHERE delivered_at IS NULL AND failed_at IS NOT NULL
|
|
3318
|
+
`);
|
|
3076
3319
|
await client.query(`
|
|
3077
3320
|
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_delivered_retention_idx
|
|
3078
3321
|
ON ${tableName('outbox', options)} (delivered_at, event_id)
|
|
@@ -3084,6 +3327,28 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
3084
3327
|
WHERE delivered_at IS NULL AND failed_at IS NOT NULL
|
|
3085
3328
|
`);
|
|
3086
3329
|
}
|
|
3330
|
+
if (!options?.deferRuntimeAdminIndexes) {
|
|
3331
|
+
await client.query(`
|
|
3332
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_ready_health_idx
|
|
3333
|
+
ON ${tableName('compute_billing_jobs_v2', options)}
|
|
3334
|
+
(available_at, created_at, run_id)
|
|
3335
|
+
WHERE status <> 'settled'
|
|
3336
|
+
AND blocked_at IS NULL
|
|
3337
|
+
AND lease_owner IS NULL
|
|
3338
|
+
`);
|
|
3339
|
+
await client.query(`
|
|
3340
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_lease_health_idx
|
|
3341
|
+
ON ${tableName('compute_billing_jobs_v2', options)}
|
|
3342
|
+
(lease_expires_at, created_at, run_id)
|
|
3343
|
+
WHERE status <> 'settled' AND lease_owner IS NOT NULL
|
|
3344
|
+
`);
|
|
3345
|
+
await client.query(`
|
|
3346
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_blocked_health_idx
|
|
3347
|
+
ON ${tableName('compute_billing_jobs_v2', options)}
|
|
3348
|
+
(blocked_at, lease_expires_at, created_at, run_id)
|
|
3349
|
+
WHERE status <> 'settled' AND blocked_at IS NOT NULL
|
|
3350
|
+
`);
|
|
3351
|
+
}
|
|
3087
3352
|
await client.query(`
|
|
3088
3353
|
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_run_created_idx
|
|
3089
3354
|
ON ${tableName('outbox', options)} (run_id, created_at, event_id)
|
|
@@ -3111,6 +3376,7 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
3111
3376
|
)} (version) VALUES ($1) ON CONFLICT (version) DO NOTHING`,
|
|
3112
3377
|
[POSTGRES_SCHEDULER_SCHEMA_VERSION],
|
|
3113
3378
|
);
|
|
3379
|
+
await ensurePostgresSchedulerCronScheduleSchema(client, options);
|
|
3114
3380
|
if (hadOutbox.rows[0]?.present !== true) {
|
|
3115
3381
|
await client.query(
|
|
3116
3382
|
`
|
|
@@ -3159,6 +3425,28 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
|
|
|
3159
3425
|
definition: `(created_at, event_id)
|
|
3160
3426
|
WHERE delivered_at IS NULL AND failed_at IS NULL`,
|
|
3161
3427
|
},
|
|
3428
|
+
{
|
|
3429
|
+
name: 'runtime_scheduler_outbox_lease_health_idx',
|
|
3430
|
+
table: 'outbox',
|
|
3431
|
+
definition: `(lease_expires_at, created_at, event_id)
|
|
3432
|
+
WHERE delivered_at IS NULL
|
|
3433
|
+
AND failed_at IS NULL
|
|
3434
|
+
AND lease_owner IS NOT NULL`,
|
|
3435
|
+
},
|
|
3436
|
+
{
|
|
3437
|
+
name: 'runtime_scheduler_outbox_ready_health_idx',
|
|
3438
|
+
table: 'outbox',
|
|
3439
|
+
definition: `(coalesce(next_attempt_at, created_at), created_at, event_id)
|
|
3440
|
+
WHERE delivered_at IS NULL
|
|
3441
|
+
AND failed_at IS NULL
|
|
3442
|
+
AND lease_owner IS NULL`,
|
|
3443
|
+
},
|
|
3444
|
+
{
|
|
3445
|
+
name: 'runtime_scheduler_outbox_dead_health_idx',
|
|
3446
|
+
table: 'outbox',
|
|
3447
|
+
definition: `(failed_at, event_id)
|
|
3448
|
+
WHERE delivered_at IS NULL AND failed_at IS NOT NULL`,
|
|
3449
|
+
},
|
|
3162
3450
|
{
|
|
3163
3451
|
name: 'runtime_scheduler_outbox_admission_pending_idx',
|
|
3164
3452
|
table: 'outbox',
|
|
@@ -3182,6 +3470,48 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
|
|
|
3182
3470
|
definition: `(created_at, event_id)
|
|
3183
3471
|
WHERE delivered_at IS NULL AND failed_at IS NOT NULL`,
|
|
3184
3472
|
},
|
|
3473
|
+
{
|
|
3474
|
+
name: 'runtime_scheduler_sandbox_cleanup_v2_ready_health_idx',
|
|
3475
|
+
table: 'sandbox_cleanup_jobs_v2',
|
|
3476
|
+
definition: `(available_at, created_at, provider, sandbox_id)
|
|
3477
|
+
WHERE status <> 'deleted'
|
|
3478
|
+
AND blocked_at IS NULL
|
|
3479
|
+
AND lease_owner IS NULL
|
|
3480
|
+
AND status <> 'deleting'`,
|
|
3481
|
+
},
|
|
3482
|
+
{
|
|
3483
|
+
name: 'runtime_scheduler_sandbox_cleanup_v2_lease_health_idx',
|
|
3484
|
+
table: 'sandbox_cleanup_jobs_v2',
|
|
3485
|
+
definition: `(lease_expires_at, created_at, provider, sandbox_id)
|
|
3486
|
+
WHERE status <> 'deleted' AND lease_owner IS NOT NULL`,
|
|
3487
|
+
},
|
|
3488
|
+
{
|
|
3489
|
+
name: 'runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx',
|
|
3490
|
+
table: 'sandbox_cleanup_jobs_v2',
|
|
3491
|
+
definition: `(blocked_at, lease_expires_at, created_at, provider, sandbox_id)
|
|
3492
|
+
WHERE status <> 'deleted'
|
|
3493
|
+
AND (blocked_at IS NOT NULL OR status = 'deleting')`,
|
|
3494
|
+
},
|
|
3495
|
+
{
|
|
3496
|
+
name: 'runtime_scheduler_compute_billing_ready_health_idx',
|
|
3497
|
+
table: 'compute_billing_jobs_v2',
|
|
3498
|
+
definition: `(available_at, created_at, run_id)
|
|
3499
|
+
WHERE status <> 'settled'
|
|
3500
|
+
AND blocked_at IS NULL
|
|
3501
|
+
AND lease_owner IS NULL`,
|
|
3502
|
+
},
|
|
3503
|
+
{
|
|
3504
|
+
name: 'runtime_scheduler_compute_billing_lease_health_idx',
|
|
3505
|
+
table: 'compute_billing_jobs_v2',
|
|
3506
|
+
definition: `(lease_expires_at, created_at, run_id)
|
|
3507
|
+
WHERE status <> 'settled' AND lease_owner IS NOT NULL`,
|
|
3508
|
+
},
|
|
3509
|
+
{
|
|
3510
|
+
name: 'runtime_scheduler_compute_billing_blocked_health_idx',
|
|
3511
|
+
table: 'compute_billing_jobs_v2',
|
|
3512
|
+
definition: `(blocked_at, lease_expires_at, created_at, run_id)
|
|
3513
|
+
WHERE status <> 'settled' AND blocked_at IS NOT NULL`,
|
|
3514
|
+
},
|
|
3185
3515
|
{
|
|
3186
3516
|
name: 'runtime_scheduler_runs_terminal_finished_idx',
|
|
3187
3517
|
table: 'runs',
|
|
@@ -3518,17 +3848,24 @@ export async function ensureAbsurdSchema(
|
|
|
3518
3848
|
client: PostgresSchedulerQueryClient,
|
|
3519
3849
|
): Promise<void> {
|
|
3520
3850
|
const [lockClassId, lockObjectId] = absurdSchemaInitAdvisoryLockKey();
|
|
3521
|
-
|
|
3851
|
+
let stage = 'begin transaction';
|
|
3852
|
+
let transactionStarted = false;
|
|
3522
3853
|
let committed = false;
|
|
3523
3854
|
try {
|
|
3855
|
+
await client.query('BEGIN');
|
|
3856
|
+
transactionStarted = true;
|
|
3857
|
+
|
|
3858
|
+
stage = 'set bootstrap statement timeout';
|
|
3524
3859
|
await client.query(
|
|
3525
3860
|
`SELECT set_config('statement_timeout', '${POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS}', true)`,
|
|
3526
3861
|
);
|
|
3862
|
+
stage = 'acquire Absurd schema bootstrap advisory lock';
|
|
3527
3863
|
await client.query(
|
|
3528
3864
|
`SELECT pg_advisory_xact_lock($1::integer, $2::integer)`,
|
|
3529
3865
|
[lockClassId, lockObjectId],
|
|
3530
3866
|
);
|
|
3531
3867
|
|
|
3868
|
+
stage = 'read Absurd schema catalog marker';
|
|
3532
3869
|
const present = await client.query<{ present: boolean }>(
|
|
3533
3870
|
`SELECT EXISTS (
|
|
3534
3871
|
SELECT 1
|
|
@@ -3540,11 +3877,13 @@ export async function ensureAbsurdSchema(
|
|
|
3540
3877
|
);
|
|
3541
3878
|
|
|
3542
3879
|
if (present.rows[0]?.present) {
|
|
3880
|
+
stage = 'read installed Absurd schema version';
|
|
3543
3881
|
const versionRow = await client.query<{ v: string | null }>(
|
|
3544
3882
|
`SELECT absurd.get_schema_version() AS v`,
|
|
3545
3883
|
);
|
|
3546
3884
|
const installed = versionRow.rows[0]?.v ?? null;
|
|
3547
3885
|
if (installed === ABSURD_SCHEMA_VERSION) {
|
|
3886
|
+
stage = 'read Absurd compatibility marker catalog entry';
|
|
3548
3887
|
const marker = await client.query<{ present: boolean }>(
|
|
3549
3888
|
`SELECT EXISTS (
|
|
3550
3889
|
SELECT 1
|
|
@@ -3554,6 +3893,7 @@ export async function ensureAbsurdSchema(
|
|
|
3554
3893
|
AND p.proname = 'deepline_no_scheduler_org_cap_compatibility_version'
|
|
3555
3894
|
) AS present`,
|
|
3556
3895
|
);
|
|
3896
|
+
stage = 'read Absurd compatibility marker version';
|
|
3557
3897
|
const markerVersion = marker.rows[0]?.present
|
|
3558
3898
|
? await client.query<{ v: string | null }>(
|
|
3559
3899
|
`SELECT absurd.deepline_no_scheduler_org_cap_compatibility_version() AS v`,
|
|
@@ -3563,13 +3903,16 @@ export async function ensureAbsurdSchema(
|
|
|
3563
3903
|
markerVersion?.rows[0]?.v !==
|
|
3564
3904
|
ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION
|
|
3565
3905
|
) {
|
|
3906
|
+
stage = 'apply Absurd no-org-cap compatibility function patch';
|
|
3566
3907
|
await client.query(
|
|
3567
3908
|
readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(),
|
|
3568
3909
|
);
|
|
3910
|
+
stage = 'apply Absurd no-org-cap compatibility marker patch';
|
|
3569
3911
|
await client.query(
|
|
3570
3912
|
readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(),
|
|
3571
3913
|
);
|
|
3572
3914
|
}
|
|
3915
|
+
stage = 'commit verified Absurd schema bootstrap';
|
|
3573
3916
|
await client.query('COMMIT');
|
|
3574
3917
|
committed = true;
|
|
3575
3918
|
return;
|
|
@@ -3579,32 +3922,42 @@ export async function ensureAbsurdSchema(
|
|
|
3579
3922
|
`"${ABSURD_SCHEMA_VERSION}". Continuing; the next SQL operation ` +
|
|
3580
3923
|
'will report any missing relation, column, or function.',
|
|
3581
3924
|
);
|
|
3925
|
+
stage = 'commit non-current Absurd schema marker';
|
|
3582
3926
|
await client.query('COMMIT');
|
|
3583
3927
|
committed = true;
|
|
3584
3928
|
return;
|
|
3585
3929
|
}
|
|
3586
3930
|
|
|
3931
|
+
stage = 'apply full Absurd schema';
|
|
3587
3932
|
await client.query(readAbsurdSchemaSql());
|
|
3933
|
+
stage = 'apply Absurd no-org-cap compatibility function patch';
|
|
3588
3934
|
await client.query(readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql());
|
|
3935
|
+
stage = 'apply Absurd no-org-cap compatibility marker patch';
|
|
3589
3936
|
await client.query(
|
|
3590
3937
|
readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(),
|
|
3591
3938
|
);
|
|
3939
|
+
stage = 'commit newly initialized Absurd schema';
|
|
3592
3940
|
await client.query('COMMIT');
|
|
3593
3941
|
committed = true;
|
|
3594
3942
|
} catch (error) {
|
|
3595
|
-
|
|
3943
|
+
const primaryError = new Error(
|
|
3944
|
+
`Failed to initialize Absurd schema during ${stage}. ` +
|
|
3945
|
+
`Primary error: ${errorMessage(error)}.`,
|
|
3946
|
+
{ cause: error },
|
|
3947
|
+
);
|
|
3948
|
+
if (transactionStarted && !committed) {
|
|
3596
3949
|
try {
|
|
3597
3950
|
await client.query('ROLLBACK');
|
|
3598
3951
|
} catch (rollbackError) {
|
|
3599
3952
|
throw new AggregateError(
|
|
3600
|
-
[
|
|
3953
|
+
[primaryError, rollbackError],
|
|
3601
3954
|
'Failed to initialize absurd schema and roll back the failed transaction. ' +
|
|
3602
|
-
|
|
3955
|
+
`${primaryError.message} ` +
|
|
3603
3956
|
`Rollback error: ${errorMessage(rollbackError)}.`,
|
|
3604
3957
|
);
|
|
3605
3958
|
}
|
|
3606
3959
|
}
|
|
3607
|
-
throw
|
|
3960
|
+
throw primaryError;
|
|
3608
3961
|
}
|
|
3609
3962
|
}
|
|
3610
3963
|
|
|
@@ -3629,7 +3982,12 @@ export async function submitPostgresAbsurdPlayRun(
|
|
|
3629
3982
|
taskRecoveryRequired: boolean;
|
|
3630
3983
|
}> {
|
|
3631
3984
|
const task = buildAbsurdPlayRunTask({
|
|
3632
|
-
|
|
3985
|
+
// This fence applies to admission only. A repaired historic task must
|
|
3986
|
+
// retain the persisted launch envelope, not the one-shot retry policy.
|
|
3987
|
+
launch: (() => {
|
|
3988
|
+
const { existingOnly: _existingOnly, ...launch } = input;
|
|
3989
|
+
return launch;
|
|
3990
|
+
})(),
|
|
3633
3991
|
schedulerSchema: options?.schema,
|
|
3634
3992
|
});
|
|
3635
3993
|
const submitted = await submitPostgresSchedulerRunTransaction(
|
|
@@ -3689,9 +4047,11 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
3689
4047
|
asyncParentRunAttempt: _asyncParentRunAttempt,
|
|
3690
4048
|
asyncParentRunLeaseToken: _asyncParentRunLeaseToken,
|
|
3691
4049
|
asyncParentAttemptRequired: _asyncParentAttemptRequired,
|
|
4050
|
+
existingOnly: requestedExistingOnly,
|
|
3692
4051
|
...durableInput
|
|
3693
4052
|
} = input;
|
|
3694
4053
|
input = durableInput;
|
|
4054
|
+
const existingOnlySql = requestedExistingOnly === true ? 'true' : 'false';
|
|
3695
4055
|
const releaseId = input.absurdReleaseId?.trim() || null;
|
|
3696
4056
|
const legacyPreviewRelease =
|
|
3697
4057
|
releaseId === 'dev' || releaseId?.startsWith('dev-') === true;
|
|
@@ -3734,6 +4094,7 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
3734
4094
|
NOT admission.existing_run
|
|
3735
4095
|
AND admission.accepting_new
|
|
3736
4096
|
AND admission.parent_accepting
|
|
4097
|
+
AND NOT ${existingOnlySql}
|
|
3737
4098
|
) OR (
|
|
3738
4099
|
${recoverExistingTaskSql}
|
|
3739
4100
|
AND admission.existing_run
|
|
@@ -3921,6 +4282,7 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
3921
4282
|
WHERE NOT existing_run
|
|
3922
4283
|
AND accepting_new
|
|
3923
4284
|
AND parent_accepting
|
|
4285
|
+
AND NOT ${existingOnlySql}
|
|
3924
4286
|
ON CONFLICT (run_id) DO NOTHING
|
|
3925
4287
|
RETURNING run_id, absurd_task_id
|
|
3926
4288
|
), run_recovery AS (
|
|
@@ -3948,153 +4310,1292 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
3948
4310
|
run_insert.run_id,
|
|
3949
4311
|
'run.created',
|
|
3950
4312
|
jsonb_strip_nulls(jsonb_build_object(
|
|
3951
|
-
'orgId', $2::text,
|
|
3952
|
-
'playId', $3::text,
|
|
3953
|
-
'playName', $4::text,
|
|
3954
|
-
-- Optional lineage makes a projected async child discoverable
|
|
3955
|
-
-- from its terminal parent. Older envelopes omit these keys.
|
|
3956
|
-
'parentRunId', $8::jsonb ->> 'parentRunId',
|
|
3957
|
-
'rootRunId', $8::jsonb ->> 'rootRunId',
|
|
3958
|
-
'asyncLaunchKey', $8::jsonb ->> 'asyncLaunchKey',
|
|
3959
|
-
'asyncInvocationKey', $8::jsonb ->> 'asyncInvocationKey',
|
|
3960
|
-
'asyncRequestFingerprint', $8::jsonb ->> 'asyncRequestFingerprint',
|
|
3961
|
-
'asyncAncestryPlayIds', $8::jsonb -> 'asyncAncestryPlayIds',
|
|
3962
|
-
'definitionName', $8::jsonb ->> 'definitionName',
|
|
3963
|
-
'playReference', $8::jsonb ->> 'playReference',
|
|
3964
|
-
'definitionScope', $8::jsonb ->> 'definitionScope',
|
|
3965
|
-
'revisionId', $8::jsonb ->> 'revisionId',
|
|
3966
|
-
'workflowFamilyKey', $8::jsonb ->> 'workflowFamilyKey',
|
|
3967
|
-
'artifactStorageKey', $7::text,
|
|
3968
|
-
'artifactHash', $8::jsonb ->> 'artifactHash',
|
|
3969
|
-
'graphHash', $5::text,
|
|
3970
|
-
'runtimeBackend', $6::text,
|
|
3971
|
-
'schedulerBackend', $8::jsonb ->> 'schedulerBackend',
|
|
3972
|
-
'schedulerSchema', $8::jsonb ->> 'schedulerSchema',
|
|
3973
|
-
'executionProfile', $8::jsonb ->> 'executionProfile',
|
|
3974
|
-
'runtimeReleaseId', $8::jsonb ->> 'runtimeReleaseId',
|
|
3975
|
-
'runtimeDeployVersion', $8::jsonb ->> 'runtimeDeployVersion',
|
|
3976
|
-
'runtimeCallbackBaseUrl', $8::jsonb ->> 'runtimeCallbackBaseUrl',
|
|
3977
|
-
'coordinatorUrl', $8::jsonb ->> 'coordinatorUrl',
|
|
3978
|
-
'coordinatorWorkerName', $8::jsonb ->> 'coordinatorWorkerName',
|
|
3979
|
-
'runtimeWorkflowName', $8::jsonb ->> 'runtimeWorkflowName',
|
|
3980
|
-
'runtimeHarnessWorkerName', $8::jsonb ->> 'runtimeHarnessWorkerName',
|
|
3981
|
-
'maxCreditsPerRun', $8::jsonb -> 'maxCreditsPerRun',
|
|
3982
|
-
'staticPipeline', $8::jsonb -> 'staticPipeline',
|
|
3983
|
-
'source', $8::jsonb ->> 'source',
|
|
3984
|
-
'triggerSource', $8::jsonb ->> 'triggerSource',
|
|
3985
|
-
'inputFileId', $8::jsonb ->> 'inputFileId',
|
|
3986
|
-
'inputBytes', $8::jsonb -> 'inputBytes',
|
|
3987
|
-
'inputSha256', $8::jsonb ->> 'inputSha256',
|
|
3988
|
-
'replayedFromRunId', $8::jsonb ->> 'replayedFromRunId',
|
|
3989
|
-
'secretRefs', $8::jsonb -> 'secretRefs',
|
|
3990
|
-
'inputSummary', $8::jsonb -> 'inputSummary'
|
|
4313
|
+
'orgId', $2::text,
|
|
4314
|
+
'playId', $3::text,
|
|
4315
|
+
'playName', $4::text,
|
|
4316
|
+
-- Optional lineage makes a projected async child discoverable
|
|
4317
|
+
-- from its terminal parent. Older envelopes omit these keys.
|
|
4318
|
+
'parentRunId', $8::jsonb ->> 'parentRunId',
|
|
4319
|
+
'rootRunId', $8::jsonb ->> 'rootRunId',
|
|
4320
|
+
'asyncLaunchKey', $8::jsonb ->> 'asyncLaunchKey',
|
|
4321
|
+
'asyncInvocationKey', $8::jsonb ->> 'asyncInvocationKey',
|
|
4322
|
+
'asyncRequestFingerprint', $8::jsonb ->> 'asyncRequestFingerprint',
|
|
4323
|
+
'asyncAncestryPlayIds', $8::jsonb -> 'asyncAncestryPlayIds',
|
|
4324
|
+
'definitionName', $8::jsonb ->> 'definitionName',
|
|
4325
|
+
'playReference', $8::jsonb ->> 'playReference',
|
|
4326
|
+
'definitionScope', $8::jsonb ->> 'definitionScope',
|
|
4327
|
+
'revisionId', $8::jsonb ->> 'revisionId',
|
|
4328
|
+
'workflowFamilyKey', $8::jsonb ->> 'workflowFamilyKey',
|
|
4329
|
+
'artifactStorageKey', $7::text,
|
|
4330
|
+
'artifactHash', $8::jsonb ->> 'artifactHash',
|
|
4331
|
+
'graphHash', $5::text,
|
|
4332
|
+
'runtimeBackend', $6::text,
|
|
4333
|
+
'schedulerBackend', $8::jsonb ->> 'schedulerBackend',
|
|
4334
|
+
'schedulerSchema', $8::jsonb ->> 'schedulerSchema',
|
|
4335
|
+
'executionProfile', $8::jsonb ->> 'executionProfile',
|
|
4336
|
+
'runtimeReleaseId', $8::jsonb ->> 'runtimeReleaseId',
|
|
4337
|
+
'runtimeDeployVersion', $8::jsonb ->> 'runtimeDeployVersion',
|
|
4338
|
+
'runtimeCallbackBaseUrl', $8::jsonb ->> 'runtimeCallbackBaseUrl',
|
|
4339
|
+
'coordinatorUrl', $8::jsonb ->> 'coordinatorUrl',
|
|
4340
|
+
'coordinatorWorkerName', $8::jsonb ->> 'coordinatorWorkerName',
|
|
4341
|
+
'runtimeWorkflowName', $8::jsonb ->> 'runtimeWorkflowName',
|
|
4342
|
+
'runtimeHarnessWorkerName', $8::jsonb ->> 'runtimeHarnessWorkerName',
|
|
4343
|
+
'maxCreditsPerRun', $8::jsonb -> 'maxCreditsPerRun',
|
|
4344
|
+
'staticPipeline', $8::jsonb -> 'staticPipeline',
|
|
4345
|
+
'source', $8::jsonb ->> 'source',
|
|
4346
|
+
'triggerSource', $8::jsonb ->> 'triggerSource',
|
|
4347
|
+
'inputFileId', $8::jsonb ->> 'inputFileId',
|
|
4348
|
+
'inputBytes', $8::jsonb -> 'inputBytes',
|
|
4349
|
+
'inputSha256', $8::jsonb ->> 'inputSha256',
|
|
4350
|
+
'replayedFromRunId', $8::jsonb ->> 'replayedFromRunId',
|
|
4351
|
+
'secretRefs', $8::jsonb -> 'secretRefs',
|
|
4352
|
+
'inputSummary', $8::jsonb -> 'inputSummary'
|
|
4353
|
+
)),
|
|
4354
|
+
${causalOutboxCreatedAt('run_insert.run_id', options)}
|
|
4355
|
+
FROM run_insert
|
|
4356
|
+
-- Old Vercel launchers do not carry the immutable admission envelope.
|
|
4357
|
+
-- During a Convex-first rollout they retain their established start
|
|
4358
|
+
-- projection; only the new launcher opts into queued projection.
|
|
4359
|
+
WHERE $8::jsonb ? 'playReference'
|
|
4360
|
+
ON CONFLICT (event_id) DO NOTHING
|
|
4361
|
+
RETURNING run_id
|
|
4362
|
+
)
|
|
4363
|
+
SELECT (
|
|
4364
|
+
(NOT admission.existing_run AND admission.accepting_new
|
|
4365
|
+
AND admission.parent_accepting AND NOT ${existingOnlySql})
|
|
4366
|
+
OR admission.matching_existing
|
|
4367
|
+
) AS accepted,
|
|
4368
|
+
admission.accepting_new,
|
|
4369
|
+
admission.parent_accepting,
|
|
4370
|
+
admission.parent_attempt_current,
|
|
4371
|
+
admission.parent_terminal,
|
|
4372
|
+
admission.existing_run,
|
|
4373
|
+
admission.matching_existing,
|
|
4374
|
+
coalesce(
|
|
4375
|
+
(SELECT task_id FROM task_spawn),
|
|
4376
|
+
(SELECT absurd_task_id FROM existing_run)
|
|
4377
|
+
) AS absurd_task_id,
|
|
4378
|
+
EXISTS (SELECT 1 FROM run_insert) AS run_created,
|
|
4379
|
+
(SELECT created FROM task_spawn) AS task_created,
|
|
4380
|
+
(
|
|
4381
|
+
admission.matching_existing
|
|
4382
|
+
AND NOT ${recoverExistingTaskSql}
|
|
4383
|
+
AND EXISTS (
|
|
4384
|
+
SELECT 1
|
|
4385
|
+
FROM existing_run
|
|
4386
|
+
WHERE existing_run.status = 'queued'
|
|
4387
|
+
AND existing_run.absurd_task_id IS NULL
|
|
4388
|
+
)
|
|
4389
|
+
) AS task_recovery_required,
|
|
4390
|
+
CASE WHEN EXISTS (SELECT 1 FROM run_insert)
|
|
4391
|
+
OR EXISTS (SELECT 1 FROM run_recovery)
|
|
4392
|
+
THEN pg_notify($10, $11)
|
|
4393
|
+
END
|
|
4394
|
+
FROM admission
|
|
4395
|
+
`,
|
|
4396
|
+
[
|
|
4397
|
+
input.runId,
|
|
4398
|
+
input.orgId,
|
|
4399
|
+
input.playId,
|
|
4400
|
+
input.playName,
|
|
4401
|
+
input.graphHash,
|
|
4402
|
+
input.runtimeBackend,
|
|
4403
|
+
input.artifactStorageKey,
|
|
4404
|
+
stringifyPostgresJson(input),
|
|
4405
|
+
stringifyPostgresJson(input.input ?? {}),
|
|
4406
|
+
postgresSchedulerNotifyChannel('queue', options),
|
|
4407
|
+
JSON.stringify({ runId: input.runId, playName: input.playName }),
|
|
4408
|
+
releaseAdmissionKey,
|
|
4409
|
+
releaseAdmissionLockKey,
|
|
4410
|
+
useLegacyAdmissionKey,
|
|
4411
|
+
releaseId,
|
|
4412
|
+
parentRunId,
|
|
4413
|
+
asyncRootRunId,
|
|
4414
|
+
asyncParentRunAttempt,
|
|
4415
|
+
asyncParentRunLeaseToken,
|
|
4416
|
+
asyncParentAttemptRequired,
|
|
4417
|
+
...(absurdTask
|
|
4418
|
+
? [
|
|
4419
|
+
absurdTask.queueName,
|
|
4420
|
+
absurdTask.taskName,
|
|
4421
|
+
stringifyPostgresJson(absurdTask.params),
|
|
4422
|
+
stringifyPostgresJson(absurdTask.options),
|
|
4423
|
+
]
|
|
4424
|
+
: []),
|
|
4425
|
+
],
|
|
4426
|
+
);
|
|
4427
|
+
const row = submitted.rows[0];
|
|
4428
|
+
if (
|
|
4429
|
+
row?.accepted === false &&
|
|
4430
|
+
!row.existing_run &&
|
|
4431
|
+
row.parent_accepting === false
|
|
4432
|
+
) {
|
|
4433
|
+
if (
|
|
4434
|
+
row?.parent_terminal === false &&
|
|
4435
|
+
row?.parent_attempt_current === false
|
|
4436
|
+
) {
|
|
4437
|
+
throw new Error(
|
|
4438
|
+
`CTX_RUN_PLAY_ASYNC_PARENT_ATTEMPT_STALE: parent Run "${parentRunId}" no longer has the executor attempt that requested this child.`,
|
|
4439
|
+
);
|
|
4440
|
+
}
|
|
4441
|
+
throw new Error(
|
|
4442
|
+
`CTX_RUN_PLAY_ASYNC_PARENT_TERMINAL: parent Run "${parentRunId}" is terminal, missing, belongs to another organization, or is pinned to a different release.`,
|
|
4443
|
+
);
|
|
4444
|
+
}
|
|
4445
|
+
if (row?.accepted === false && row.existing_run && !row.matching_existing) {
|
|
4446
|
+
throw new Error(
|
|
4447
|
+
`Play run "${input.runId}" already exists with a different launch.`,
|
|
4448
|
+
);
|
|
4449
|
+
}
|
|
4450
|
+
if (row?.accepted === false && !row?.existing_run && requestedExistingOnly) {
|
|
4451
|
+
throw new Error(
|
|
4452
|
+
`Play run "${input.runId}" does not exist for an existing-only admission.`,
|
|
4453
|
+
);
|
|
4454
|
+
}
|
|
4455
|
+
if (row?.accepted === false) {
|
|
4456
|
+
throw new Error(
|
|
4457
|
+
`Absurd runtime release "${releaseId}" no longer accepts new launches.`,
|
|
4458
|
+
);
|
|
4459
|
+
}
|
|
4460
|
+
return {
|
|
4461
|
+
absurdTaskId: row?.absurd_task_id ?? null,
|
|
4462
|
+
runCreated: row?.run_created === true,
|
|
4463
|
+
taskCreated: row?.task_created === true,
|
|
4464
|
+
taskRecoveryRequired: row?.task_recovery_required === true,
|
|
4465
|
+
};
|
|
4466
|
+
}
|
|
4467
|
+
|
|
4468
|
+
/** Immutable Runtime Postgres state for an Absurd-owned Play cron timer. */
|
|
4469
|
+
export type PostgresSchedulerCronSchedule = {
|
|
4470
|
+
scheduleId: string;
|
|
4471
|
+
generation: number;
|
|
4472
|
+
state: 'active' | 'blocked' | 'cancelled';
|
|
4473
|
+
cron: string;
|
|
4474
|
+
timezone: string | null;
|
|
4475
|
+
dueAtMs: number;
|
|
4476
|
+
currentFireTaskId: string | null;
|
|
4477
|
+
currentFireGeneration: number;
|
|
4478
|
+
currentFireQueueName: string | null;
|
|
4479
|
+
/** Immutable source identity only; no executor token or release lane. */
|
|
4480
|
+
sourcePin: PostgresSchedulerCronSourcePin;
|
|
4481
|
+
};
|
|
4482
|
+
|
|
4483
|
+
export type PostgresSchedulerCronFireTaskState =
|
|
4484
|
+
| 'pending'
|
|
4485
|
+
| 'running'
|
|
4486
|
+
| 'sleeping'
|
|
4487
|
+
| 'completed'
|
|
4488
|
+
| 'failed'
|
|
4489
|
+
| 'cancelled';
|
|
4490
|
+
|
|
4491
|
+
/**
|
|
4492
|
+
* Read-only health view for the Runtime-owned cron successor contract.
|
|
4493
|
+
*
|
|
4494
|
+
* A schedule is the source-of-truth cursor. Its current fire task is merely
|
|
4495
|
+
* the durable delivery vehicle for that cursor, so a missing or terminal task
|
|
4496
|
+
* on an active row is an operator-visible integrity failure rather than a
|
|
4497
|
+
* reason to silently advance or recreate an occurrence.
|
|
4498
|
+
*/
|
|
4499
|
+
export type PostgresSchedulerCronHealth = {
|
|
4500
|
+
observedAtMs: number;
|
|
4501
|
+
activeSchedules: number;
|
|
4502
|
+
blockedSchedules: number;
|
|
4503
|
+
cancelledSchedules: number;
|
|
4504
|
+
overdueSchedules: number;
|
|
4505
|
+
oldestOverdueDueAtMs: number | null;
|
|
4506
|
+
missingCurrentFireTasks: number;
|
|
4507
|
+
terminalCurrentFireTasks: number;
|
|
4508
|
+
issues: Array<{
|
|
4509
|
+
scheduleId: string;
|
|
4510
|
+
generation: number;
|
|
4511
|
+
state: 'active' | 'blocked' | 'cancelled';
|
|
4512
|
+
dueAtMs: number;
|
|
4513
|
+
currentFireGeneration: number;
|
|
4514
|
+
currentFireTaskId: string | null;
|
|
4515
|
+
currentFireQueueName: string | null;
|
|
4516
|
+
fireTaskState: PostgresSchedulerCronFireTaskState | null;
|
|
4517
|
+
overdue: boolean;
|
|
4518
|
+
missingCurrentFireTask: boolean;
|
|
4519
|
+
terminalCurrentFireTask: boolean;
|
|
4520
|
+
}>;
|
|
4521
|
+
};
|
|
4522
|
+
|
|
4523
|
+
type PostgresSchedulerCronHealthQueryRow = {
|
|
4524
|
+
schedule_id: string;
|
|
4525
|
+
generation: number | string;
|
|
4526
|
+
state: 'active' | 'blocked' | 'cancelled';
|
|
4527
|
+
due_at_ms: number | string;
|
|
4528
|
+
current_fire_generation: number | string;
|
|
4529
|
+
current_fire_task_id: string | null;
|
|
4530
|
+
current_fire_queue_name: string | null;
|
|
4531
|
+
fire_task_state: string | null;
|
|
4532
|
+
overdue: boolean;
|
|
4533
|
+
missing_current_fire_task: boolean;
|
|
4534
|
+
terminal_current_fire_task: boolean;
|
|
4535
|
+
};
|
|
4536
|
+
|
|
4537
|
+
const POSTGRES_SCHEDULER_CRON_FIRE_TASK_STATES =
|
|
4538
|
+
new Set<PostgresSchedulerCronFireTaskState>([
|
|
4539
|
+
'pending',
|
|
4540
|
+
'running',
|
|
4541
|
+
'sleeping',
|
|
4542
|
+
'completed',
|
|
4543
|
+
'failed',
|
|
4544
|
+
'cancelled',
|
|
4545
|
+
]);
|
|
4546
|
+
|
|
4547
|
+
function postgresSchedulerCronFireTaskState(
|
|
4548
|
+
value: string | null,
|
|
4549
|
+
): PostgresSchedulerCronFireTaskState | null {
|
|
4550
|
+
return value &&
|
|
4551
|
+
POSTGRES_SCHEDULER_CRON_FIRE_TASK_STATES.has(
|
|
4552
|
+
value as PostgresSchedulerCronFireTaskState,
|
|
4553
|
+
)
|
|
4554
|
+
? (value as PostgresSchedulerCronFireTaskState)
|
|
4555
|
+
: null;
|
|
4556
|
+
}
|
|
4557
|
+
|
|
4558
|
+
function postgresSchedulerCronHealthInteger(
|
|
4559
|
+
value: number | string | null | undefined,
|
|
4560
|
+
): number {
|
|
4561
|
+
const parsed = Number(value ?? 0);
|
|
4562
|
+
return Number.isSafeInteger(parsed) ? parsed : 0;
|
|
4563
|
+
}
|
|
4564
|
+
|
|
4565
|
+
function assertPostgresSchedulerCronHealthInput(input: {
|
|
4566
|
+
nowMs?: number;
|
|
4567
|
+
limit?: number;
|
|
4568
|
+
}): { nowMs: number; limit: number } {
|
|
4569
|
+
const nowMs = input.nowMs ?? Date.now();
|
|
4570
|
+
if (!Number.isSafeInteger(nowMs) || nowMs <= 0) {
|
|
4571
|
+
throw new Error('Cron health nowMs must be a positive safe integer.');
|
|
4572
|
+
}
|
|
4573
|
+
const limit = input.limit ?? 50;
|
|
4574
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
|
|
4575
|
+
throw new Error(
|
|
4576
|
+
'Cron health limit must be a safe integer from 1 through 100.',
|
|
4577
|
+
);
|
|
4578
|
+
}
|
|
4579
|
+
return { nowMs, limit };
|
|
4580
|
+
}
|
|
4581
|
+
|
|
4582
|
+
/**
|
|
4583
|
+
* Read a bounded, non-mutating cron integrity report.
|
|
4584
|
+
*
|
|
4585
|
+
* The query joins only registered Absurd queues before asking the engine for a
|
|
4586
|
+
* task. A retired/missing queue therefore becomes a `missingCurrentFireTask`
|
|
4587
|
+
* signal instead of causing this diagnostic endpoint to fail while it is most
|
|
4588
|
+
* needed. Repair remains an explicit, separately fenced operation.
|
|
4589
|
+
*/
|
|
4590
|
+
export async function readPostgresSchedulerCronHealth(
|
|
4591
|
+
client: PostgresSchedulerQueryClient,
|
|
4592
|
+
input: { nowMs?: number; limit?: number } = {},
|
|
4593
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
4594
|
+
): Promise<PostgresSchedulerCronHealth> {
|
|
4595
|
+
const { nowMs, limit } = assertPostgresSchedulerCronHealthInput(input);
|
|
4596
|
+
const schedules = tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options);
|
|
4597
|
+
const healthRowsSql = `
|
|
4598
|
+
WITH schedules AS MATERIALIZED (
|
|
4599
|
+
SELECT schedule_id,
|
|
4600
|
+
generation,
|
|
4601
|
+
state,
|
|
4602
|
+
floor(extract(epoch FROM current_due_at) * 1000)::bigint AS due_at_ms,
|
|
4603
|
+
current_fire_task_id,
|
|
4604
|
+
current_fire_generation,
|
|
4605
|
+
current_fire_queue_name
|
|
4606
|
+
FROM ${schedules}
|
|
4607
|
+
), valid_timer_references AS MATERIALIZED (
|
|
4608
|
+
SELECT schedule.schedule_id,
|
|
4609
|
+
schedule.current_fire_task_id,
|
|
4610
|
+
schedule.current_fire_queue_name
|
|
4611
|
+
FROM schedules AS schedule
|
|
4612
|
+
JOIN absurd.queues AS queue
|
|
4613
|
+
ON queue.queue_name = schedule.current_fire_queue_name
|
|
4614
|
+
WHERE schedule.current_fire_task_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
|
4615
|
+
), timer_states AS MATERIALIZED (
|
|
4616
|
+
SELECT reference.schedule_id,
|
|
4617
|
+
task.state AS fire_task_state
|
|
4618
|
+
FROM valid_timer_references AS reference
|
|
4619
|
+
CROSS JOIN LATERAL absurd.get_task_result(
|
|
4620
|
+
reference.current_fire_queue_name,
|
|
4621
|
+
reference.current_fire_task_id::uuid
|
|
4622
|
+
) AS task
|
|
4623
|
+
), health_rows AS MATERIALIZED (
|
|
4624
|
+
SELECT schedule.schedule_id,
|
|
4625
|
+
schedule.generation,
|
|
4626
|
+
schedule.state,
|
|
4627
|
+
schedule.due_at_ms,
|
|
4628
|
+
schedule.current_fire_generation,
|
|
4629
|
+
schedule.current_fire_task_id,
|
|
4630
|
+
schedule.current_fire_queue_name,
|
|
4631
|
+
timer.fire_task_state,
|
|
4632
|
+
(schedule.state = 'active' AND schedule.due_at_ms <= $1::bigint) AS overdue,
|
|
4633
|
+
(
|
|
4634
|
+
schedule.state = 'active'
|
|
4635
|
+
AND (
|
|
4636
|
+
schedule.current_fire_task_id IS NULL
|
|
4637
|
+
OR schedule.current_fire_queue_name IS NULL
|
|
4638
|
+
OR NOT COALESCE(
|
|
4639
|
+
schedule.current_fire_task_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$',
|
|
4640
|
+
false
|
|
4641
|
+
)
|
|
4642
|
+
OR timer.fire_task_state IS NULL
|
|
4643
|
+
)
|
|
4644
|
+
) AS missing_current_fire_task,
|
|
4645
|
+
(
|
|
4646
|
+
schedule.state = 'active'
|
|
4647
|
+
AND timer.fire_task_state IN ('completed', 'failed', 'cancelled')
|
|
4648
|
+
) AS terminal_current_fire_task
|
|
4649
|
+
FROM schedules AS schedule
|
|
4650
|
+
LEFT JOIN timer_states AS timer
|
|
4651
|
+
ON timer.schedule_id = schedule.schedule_id
|
|
4652
|
+
)`;
|
|
4653
|
+
const summary = await client.query<{
|
|
4654
|
+
active_schedules: number | string;
|
|
4655
|
+
blocked_schedules: number | string;
|
|
4656
|
+
cancelled_schedules: number | string;
|
|
4657
|
+
overdue_schedules: number | string;
|
|
4658
|
+
oldest_overdue_due_at_ms: number | string | null;
|
|
4659
|
+
missing_current_fire_tasks: number | string;
|
|
4660
|
+
terminal_current_fire_tasks: number | string;
|
|
4661
|
+
}>(
|
|
4662
|
+
`${healthRowsSql}
|
|
4663
|
+
SELECT count(*) FILTER (WHERE state = 'active')::text AS active_schedules,
|
|
4664
|
+
count(*) FILTER (WHERE state = 'blocked')::text AS blocked_schedules,
|
|
4665
|
+
count(*) FILTER (WHERE state = 'cancelled')::text AS cancelled_schedules,
|
|
4666
|
+
count(*) FILTER (WHERE overdue)::text AS overdue_schedules,
|
|
4667
|
+
min(due_at_ms) FILTER (WHERE overdue) AS oldest_overdue_due_at_ms,
|
|
4668
|
+
count(*) FILTER (WHERE missing_current_fire_task)::text AS missing_current_fire_tasks,
|
|
4669
|
+
count(*) FILTER (WHERE terminal_current_fire_task)::text AS terminal_current_fire_tasks
|
|
4670
|
+
FROM health_rows`,
|
|
4671
|
+
[nowMs],
|
|
4672
|
+
);
|
|
4673
|
+
const issueRows = await client.query<PostgresSchedulerCronHealthQueryRow>(
|
|
4674
|
+
`${healthRowsSql}
|
|
4675
|
+
SELECT schedule_id,
|
|
4676
|
+
generation,
|
|
4677
|
+
state,
|
|
4678
|
+
due_at_ms,
|
|
4679
|
+
current_fire_generation,
|
|
4680
|
+
current_fire_task_id,
|
|
4681
|
+
current_fire_queue_name,
|
|
4682
|
+
fire_task_state,
|
|
4683
|
+
overdue,
|
|
4684
|
+
missing_current_fire_task,
|
|
4685
|
+
terminal_current_fire_task
|
|
4686
|
+
FROM health_rows
|
|
4687
|
+
WHERE state = 'blocked'
|
|
4688
|
+
OR overdue
|
|
4689
|
+
OR missing_current_fire_task
|
|
4690
|
+
OR terminal_current_fire_task
|
|
4691
|
+
ORDER BY
|
|
4692
|
+
(state = 'blocked') DESC,
|
|
4693
|
+
missing_current_fire_task DESC,
|
|
4694
|
+
terminal_current_fire_task DESC,
|
|
4695
|
+
overdue DESC,
|
|
4696
|
+
due_at_ms ASC,
|
|
4697
|
+
schedule_id ASC
|
|
4698
|
+
LIMIT $2::int`,
|
|
4699
|
+
[nowMs, limit],
|
|
4700
|
+
);
|
|
4701
|
+
const row = summary.rows[0];
|
|
4702
|
+
return {
|
|
4703
|
+
observedAtMs: nowMs,
|
|
4704
|
+
activeSchedules: postgresSchedulerCronHealthInteger(row?.active_schedules),
|
|
4705
|
+
blockedSchedules: postgresSchedulerCronHealthInteger(
|
|
4706
|
+
row?.blocked_schedules,
|
|
4707
|
+
),
|
|
4708
|
+
cancelledSchedules: postgresSchedulerCronHealthInteger(
|
|
4709
|
+
row?.cancelled_schedules,
|
|
4710
|
+
),
|
|
4711
|
+
overdueSchedules: postgresSchedulerCronHealthInteger(
|
|
4712
|
+
row?.overdue_schedules,
|
|
4713
|
+
),
|
|
4714
|
+
oldestOverdueDueAtMs:
|
|
4715
|
+
row?.oldest_overdue_due_at_ms == null
|
|
4716
|
+
? null
|
|
4717
|
+
: postgresSchedulerCronHealthInteger(row.oldest_overdue_due_at_ms),
|
|
4718
|
+
missingCurrentFireTasks: postgresSchedulerCronHealthInteger(
|
|
4719
|
+
row?.missing_current_fire_tasks,
|
|
4720
|
+
),
|
|
4721
|
+
terminalCurrentFireTasks: postgresSchedulerCronHealthInteger(
|
|
4722
|
+
row?.terminal_current_fire_tasks,
|
|
4723
|
+
),
|
|
4724
|
+
issues: issueRows.rows.map((issue) => ({
|
|
4725
|
+
scheduleId: issue.schedule_id,
|
|
4726
|
+
generation: postgresSchedulerCronHealthInteger(issue.generation),
|
|
4727
|
+
state: issue.state,
|
|
4728
|
+
dueAtMs: postgresSchedulerCronHealthInteger(issue.due_at_ms),
|
|
4729
|
+
currentFireGeneration: postgresSchedulerCronHealthInteger(
|
|
4730
|
+
issue.current_fire_generation,
|
|
4731
|
+
),
|
|
4732
|
+
currentFireTaskId: issue.current_fire_task_id,
|
|
4733
|
+
currentFireQueueName: issue.current_fire_queue_name,
|
|
4734
|
+
fireTaskState: postgresSchedulerCronFireTaskState(issue.fire_task_state),
|
|
4735
|
+
overdue: issue.overdue === true,
|
|
4736
|
+
missingCurrentFireTask: issue.missing_current_fire_task === true,
|
|
4737
|
+
terminalCurrentFireTask: issue.terminal_current_fire_task === true,
|
|
4738
|
+
})),
|
|
4739
|
+
};
|
|
4740
|
+
}
|
|
4741
|
+
|
|
4742
|
+
/** A pinned schedule can never become valid by retrying its same fire task. */
|
|
4743
|
+
export class PostgresSchedulerCronScheduleConfigurationError extends Error {
|
|
4744
|
+
constructor(message: string) {
|
|
4745
|
+
super(message);
|
|
4746
|
+
this.name = 'PostgresSchedulerCronScheduleConfigurationError';
|
|
4747
|
+
}
|
|
4748
|
+
}
|
|
4749
|
+
|
|
4750
|
+
/**
|
|
4751
|
+
* The timer stores the facts needed to reconstruct a pinned admission, not a
|
|
4752
|
+
* previously prepared launch. Executor tokens, preloaded session keys,
|
|
4753
|
+
* callback URLs, and release lanes are all intentionally absent: Vercel
|
|
4754
|
+
* creates them at the fire instant.
|
|
4755
|
+
*/
|
|
4756
|
+
export type PostgresSchedulerCronSourcePin = {
|
|
4757
|
+
orgId: string;
|
|
4758
|
+
userId: string | null;
|
|
4759
|
+
playName: string;
|
|
4760
|
+
revisionId: string | null;
|
|
4761
|
+
artifactStorageKey: string;
|
|
4762
|
+
artifactKind: 'cjs_node20' | null;
|
|
4763
|
+
artifactHash: string | null;
|
|
4764
|
+
graphHash: string | null;
|
|
4765
|
+
staticPipeline: unknown | null;
|
|
4766
|
+
input: PlaySchedulerSubmitInput['input'];
|
|
4767
|
+
source: 'published' | 'draft';
|
|
4768
|
+
schedulerSchema: string | null;
|
|
4769
|
+
};
|
|
4770
|
+
|
|
4771
|
+
export function postgresSchedulerCronSourcePinFromLaunch(
|
|
4772
|
+
launch: PlaySchedulerSubmitInput,
|
|
4773
|
+
): PostgresSchedulerCronSourcePin {
|
|
4774
|
+
const artifactKind =
|
|
4775
|
+
launch.runtimeArtifact &&
|
|
4776
|
+
typeof launch.runtimeArtifact === 'object' &&
|
|
4777
|
+
(launch.runtimeArtifact as { artifactKind?: unknown }).artifactKind ===
|
|
4778
|
+
'cjs_node20'
|
|
4779
|
+
? 'cjs_node20'
|
|
4780
|
+
: null;
|
|
4781
|
+
return {
|
|
4782
|
+
orgId: launch.orgId,
|
|
4783
|
+
userId: launch.userId ?? null,
|
|
4784
|
+
playName: launch.playName,
|
|
4785
|
+
revisionId: launch.revisionId ?? null,
|
|
4786
|
+
artifactStorageKey: launch.artifactStorageKey,
|
|
4787
|
+
artifactKind,
|
|
4788
|
+
artifactHash: launch.artifactHash || null,
|
|
4789
|
+
graphHash: launch.graphHash || null,
|
|
4790
|
+
staticPipeline: launch.staticPipeline ?? null,
|
|
4791
|
+
input: launch.input,
|
|
4792
|
+
source: launch.source === 'draft' ? 'draft' : 'published',
|
|
4793
|
+
schedulerSchema: launch.schedulerSchema ?? null,
|
|
4794
|
+
};
|
|
4795
|
+
}
|
|
4796
|
+
|
|
4797
|
+
export type SeedPostgresSchedulerCronScheduleInput = {
|
|
4798
|
+
scheduleId: string;
|
|
4799
|
+
generation: number;
|
|
4800
|
+
dueAtMs: number;
|
|
4801
|
+
cron: string;
|
|
4802
|
+
timezone?: string | null;
|
|
4803
|
+
sourcePin: PostgresSchedulerCronSourcePin;
|
|
4804
|
+
queueName: string;
|
|
4805
|
+
/** Explicit release registration gate; absence is never treated as ready. */
|
|
4806
|
+
releaseId: string;
|
|
4807
|
+
releaseEnvironment: 'preview' | 'production';
|
|
4808
|
+
};
|
|
4809
|
+
|
|
4810
|
+
function assertPostgresSchedulerCronScheduleInput(input: {
|
|
4811
|
+
scheduleId: string;
|
|
4812
|
+
generation: number;
|
|
4813
|
+
fireGeneration?: number;
|
|
4814
|
+
dueAtMs: number;
|
|
4815
|
+
cron?: string;
|
|
4816
|
+
}): void {
|
|
4817
|
+
if (!input.scheduleId.trim()) {
|
|
4818
|
+
throw new Error('Cron schedule requires a non-empty scheduleId.');
|
|
4819
|
+
}
|
|
4820
|
+
if (!Number.isSafeInteger(input.generation) || input.generation <= 0) {
|
|
4821
|
+
throw new Error(
|
|
4822
|
+
'Cron schedule generation must be a positive safe integer.',
|
|
4823
|
+
);
|
|
4824
|
+
}
|
|
4825
|
+
if (
|
|
4826
|
+
input.fireGeneration !== undefined &&
|
|
4827
|
+
(!Number.isSafeInteger(input.fireGeneration) || input.fireGeneration <= 0)
|
|
4828
|
+
) {
|
|
4829
|
+
throw new Error(
|
|
4830
|
+
'Cron schedule fireGeneration must be a positive safe integer.',
|
|
4831
|
+
);
|
|
4832
|
+
}
|
|
4833
|
+
if (!Number.isSafeInteger(input.dueAtMs) || input.dueAtMs <= 0) {
|
|
4834
|
+
throw new Error('Cron schedule dueAtMs must be a positive safe integer.');
|
|
4835
|
+
}
|
|
4836
|
+
if (input.cron !== undefined && !input.cron.trim()) {
|
|
4837
|
+
throw new Error('Cron schedule requires a non-empty cron expression.');
|
|
4838
|
+
}
|
|
4839
|
+
}
|
|
4840
|
+
|
|
4841
|
+
/**
|
|
4842
|
+
* Persist (or generation-replace) one Runtime-owned cron cursor and its first
|
|
4843
|
+
* Absurd timer in a single statement. A caller must leave Convex as the owner
|
|
4844
|
+
* until this returns: the cross-store ownership flip is deliberately separate
|
|
4845
|
+
* from this durable timer primitive.
|
|
4846
|
+
*/
|
|
4847
|
+
export async function seedPostgresSchedulerCronSchedule(
|
|
4848
|
+
client: PostgresSchedulerQueryClient,
|
|
4849
|
+
input: SeedPostgresSchedulerCronScheduleInput,
|
|
4850
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
4851
|
+
): Promise<{
|
|
4852
|
+
created: boolean;
|
|
4853
|
+
scheduleId: string;
|
|
4854
|
+
fireTaskId: string | null;
|
|
4855
|
+
}> {
|
|
4856
|
+
assertPostgresSchedulerCronScheduleInput(input);
|
|
4857
|
+
if (!input.releaseId?.trim()) {
|
|
4858
|
+
throw new Error(
|
|
4859
|
+
'Cron schedule requires an explicitly registered releaseId.',
|
|
4860
|
+
);
|
|
4861
|
+
}
|
|
4862
|
+
const sourcePin = input.sourcePin;
|
|
4863
|
+
const task = buildAbsurdCronFireTask({
|
|
4864
|
+
scheduleId: input.scheduleId.trim(),
|
|
4865
|
+
generation: input.generation,
|
|
4866
|
+
fireGeneration: 1,
|
|
4867
|
+
dueAtMs: input.dueAtMs,
|
|
4868
|
+
schedulerSchema: options?.schema ?? sourcePin.schedulerSchema ?? null,
|
|
4869
|
+
orgId: sourcePin.orgId,
|
|
4870
|
+
});
|
|
4871
|
+
const releaseAdmissionKey = `${input.releaseEnvironment}:${input.releaseId.trim()}`;
|
|
4872
|
+
const result = await client.query<{
|
|
4873
|
+
accepted: boolean;
|
|
4874
|
+
created: boolean;
|
|
4875
|
+
existing_generation: number | null;
|
|
4876
|
+
existing_matches_input: boolean | null;
|
|
4877
|
+
fire_task_id: string | null;
|
|
4878
|
+
}>(
|
|
4879
|
+
`
|
|
4880
|
+
WITH schedule_lock AS MATERIALIZED (
|
|
4881
|
+
SELECT pg_advisory_xact_lock(
|
|
4882
|
+
${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
|
|
4883
|
+
${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
|
|
4884
|
+
'cron-schedule:' || $1::text
|
|
4885
|
+
)
|
|
4886
|
+
)
|
|
4887
|
+
), existing AS MATERIALIZED (
|
|
4888
|
+
SELECT schedule.generation,
|
|
4889
|
+
(
|
|
4890
|
+
schedule.cron = $4::text
|
|
4891
|
+
AND schedule.timezone IS NOT DISTINCT FROM $5::text
|
|
4892
|
+
AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
|
|
4893
|
+
AND schedule.source_pin_json = $6::jsonb
|
|
4894
|
+
) AS matches_input
|
|
4895
|
+
FROM schedule_lock
|
|
4896
|
+
JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
4897
|
+
ON schedule.schedule_id = $1::text
|
|
4898
|
+
FOR UPDATE
|
|
4899
|
+
), release_gate AS MATERIALIZED (
|
|
4900
|
+
SELECT accepting
|
|
4901
|
+
FROM ${tableName('runtime_release_admission', options)}
|
|
4902
|
+
WHERE release_id = $11::text
|
|
4903
|
+
), task_request AS (
|
|
4904
|
+
SELECT $7::text AS queue_name,
|
|
4905
|
+
$8::text AS task_name,
|
|
4906
|
+
$9::jsonb AS params,
|
|
4907
|
+
$10::jsonb AS options
|
|
4908
|
+
WHERE (
|
|
4909
|
+
NOT EXISTS (SELECT 1 FROM existing)
|
|
4910
|
+
OR (SELECT generation FROM existing) < $2::bigint
|
|
4911
|
+
)
|
|
4912
|
+
AND coalesce((SELECT accepting FROM release_gate), false)
|
|
4913
|
+
), task_spawn AS MATERIALIZED (
|
|
4914
|
+
SELECT spawned.task_id::text AS task_id
|
|
4915
|
+
FROM task_request
|
|
4916
|
+
CROSS JOIN LATERAL absurd.spawn_task(
|
|
4917
|
+
task_request.queue_name,
|
|
4918
|
+
task_request.task_name,
|
|
4919
|
+
task_request.params,
|
|
4920
|
+
task_request.options
|
|
4921
|
+
) AS spawned
|
|
4922
|
+
), schedule_upsert AS (
|
|
4923
|
+
INSERT INTO ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule (
|
|
4924
|
+
schedule_id, generation, state, cron, timezone, current_due_at,
|
|
4925
|
+
current_fire_task_id, current_fire_generation, current_fire_queue_name,
|
|
4926
|
+
source_pin_json, launch_json
|
|
4927
|
+
)
|
|
4928
|
+
SELECT $1::text, $2::bigint, 'active', $4::text, $5::text,
|
|
4929
|
+
to_timestamp($3::double precision / 1000.0),
|
|
4930
|
+
(SELECT task_id FROM task_spawn), 1, $7::text, $6::jsonb, $6::jsonb
|
|
4931
|
+
WHERE (
|
|
4932
|
+
NOT EXISTS (SELECT 1 FROM existing)
|
|
4933
|
+
OR (SELECT generation FROM existing) < $2::bigint
|
|
4934
|
+
)
|
|
4935
|
+
AND coalesce((SELECT accepting FROM release_gate), false)
|
|
4936
|
+
ON CONFLICT (schedule_id) DO UPDATE
|
|
4937
|
+
SET generation = excluded.generation,
|
|
4938
|
+
state = excluded.state,
|
|
4939
|
+
cron = excluded.cron,
|
|
4940
|
+
timezone = excluded.timezone,
|
|
4941
|
+
current_due_at = excluded.current_due_at,
|
|
4942
|
+
current_fire_task_id = excluded.current_fire_task_id,
|
|
4943
|
+
current_fire_generation = excluded.current_fire_generation,
|
|
4944
|
+
current_fire_queue_name = excluded.current_fire_queue_name,
|
|
4945
|
+
source_pin_json = excluded.source_pin_json,
|
|
4946
|
+
-- Keep the legacy non-null column free of credentials while old
|
|
4947
|
+
-- schedules drain. New readers use source_pin_json exclusively.
|
|
4948
|
+
launch_json = excluded.launch_json,
|
|
4949
|
+
last_fired_at = NULL,
|
|
4950
|
+
updated_at = now()
|
|
4951
|
+
WHERE schedule.generation < excluded.generation
|
|
4952
|
+
RETURNING schedule_id
|
|
4953
|
+
)
|
|
4954
|
+
SELECT
|
|
4955
|
+
(EXISTS (SELECT 1 FROM schedule_upsert) OR EXISTS (SELECT 1 FROM existing))
|
|
4956
|
+
AS accepted,
|
|
4957
|
+
EXISTS (SELECT 1 FROM schedule_upsert) AS created,
|
|
4958
|
+
(SELECT generation FROM existing) AS existing_generation,
|
|
4959
|
+
(SELECT matches_input FROM existing) AS existing_matches_input,
|
|
4960
|
+
coalesce((SELECT accepting FROM release_gate), false) AS release_accepting,
|
|
4961
|
+
(SELECT task_id FROM task_spawn) AS fire_task_id
|
|
4962
|
+
`,
|
|
4963
|
+
[
|
|
4964
|
+
input.scheduleId.trim(),
|
|
4965
|
+
input.generation,
|
|
4966
|
+
input.dueAtMs,
|
|
4967
|
+
input.cron.trim(),
|
|
4968
|
+
input.timezone?.trim() || null,
|
|
4969
|
+
stringifyPostgresJson(sourcePin),
|
|
4970
|
+
input.queueName,
|
|
4971
|
+
task.taskName,
|
|
4972
|
+
stringifyPostgresJson(task.params),
|
|
4973
|
+
stringifyPostgresJson(task.options),
|
|
4974
|
+
releaseAdmissionKey,
|
|
4975
|
+
],
|
|
4976
|
+
);
|
|
4977
|
+
const row = result.rows[0];
|
|
4978
|
+
if (!row?.accepted) {
|
|
4979
|
+
throw new Error(
|
|
4980
|
+
`Cron schedule "${input.scheduleId}" was not accepted for generation ${input.generation}.`,
|
|
4981
|
+
);
|
|
4982
|
+
}
|
|
4983
|
+
if (
|
|
4984
|
+
typeof row.existing_generation === 'number' &&
|
|
4985
|
+
row.existing_generation > input.generation
|
|
4986
|
+
) {
|
|
4987
|
+
throw new Error(
|
|
4988
|
+
`Cron schedule "${input.scheduleId}" is already owned by newer generation ${row.existing_generation}.`,
|
|
4989
|
+
);
|
|
4990
|
+
}
|
|
4991
|
+
if (
|
|
4992
|
+
row.existing_generation === input.generation &&
|
|
4993
|
+
row.existing_matches_input !== true
|
|
4994
|
+
) {
|
|
4995
|
+
throw new Error(
|
|
4996
|
+
`Cron schedule "${input.scheduleId}" generation ${input.generation} already exists with a different immutable configuration.`,
|
|
4997
|
+
);
|
|
4998
|
+
}
|
|
4999
|
+
return {
|
|
5000
|
+
created: row.created === true,
|
|
5001
|
+
scheduleId: input.scheduleId.trim(),
|
|
5002
|
+
fireTaskId: row.fire_task_id ?? null,
|
|
5003
|
+
};
|
|
5004
|
+
}
|
|
5005
|
+
|
|
5006
|
+
export async function readPostgresSchedulerCronSchedule(
|
|
5007
|
+
client: PostgresSchedulerQueryClient,
|
|
5008
|
+
scheduleId: string,
|
|
5009
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
5010
|
+
): Promise<PostgresSchedulerCronSchedule | null> {
|
|
5011
|
+
const result = await client.query<{
|
|
5012
|
+
schedule_id: string;
|
|
5013
|
+
generation: number;
|
|
5014
|
+
state: 'active' | 'blocked' | 'cancelled';
|
|
5015
|
+
cron: string;
|
|
5016
|
+
timezone: string | null;
|
|
5017
|
+
due_at_ms: number;
|
|
5018
|
+
current_fire_task_id: string | null;
|
|
5019
|
+
current_fire_generation: number;
|
|
5020
|
+
current_fire_queue_name: string | null;
|
|
5021
|
+
source_pin_json: unknown;
|
|
5022
|
+
launch_json: unknown;
|
|
5023
|
+
}>(
|
|
5024
|
+
`
|
|
5025
|
+
SELECT schedule_id,
|
|
5026
|
+
generation,
|
|
5027
|
+
state,
|
|
5028
|
+
cron,
|
|
5029
|
+
timezone,
|
|
5030
|
+
floor(extract(epoch FROM current_due_at) * 1000)::bigint AS due_at_ms,
|
|
5031
|
+
current_fire_task_id,
|
|
5032
|
+
current_fire_generation,
|
|
5033
|
+
current_fire_queue_name,
|
|
5034
|
+
source_pin_json,
|
|
5035
|
+
launch_json
|
|
5036
|
+
FROM ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
5037
|
+
WHERE schedule_id = $1::text
|
|
5038
|
+
`,
|
|
5039
|
+
[scheduleId.trim()],
|
|
5040
|
+
);
|
|
5041
|
+
const row = result.rows[0];
|
|
5042
|
+
if (!row) return null;
|
|
5043
|
+
let sourcePin: PostgresSchedulerCronSourcePin;
|
|
5044
|
+
try {
|
|
5045
|
+
sourcePin = row.source_pin_json
|
|
5046
|
+
? parsePostgresSchedulerCronSourcePin(row.source_pin_json)
|
|
5047
|
+
: postgresSchedulerCronSourcePinFromLaunch(
|
|
5048
|
+
parseLaunchJson(row.launch_json),
|
|
5049
|
+
);
|
|
5050
|
+
} catch (error) {
|
|
5051
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
5052
|
+
`Cron schedule "${row.schedule_id}" has malformed pinned source data: ${
|
|
5053
|
+
error instanceof Error ? error.message : String(error)
|
|
5054
|
+
}`,
|
|
5055
|
+
);
|
|
5056
|
+
}
|
|
5057
|
+
return {
|
|
5058
|
+
scheduleId: row.schedule_id,
|
|
5059
|
+
generation: Number(row.generation),
|
|
5060
|
+
state: row.state,
|
|
5061
|
+
cron: row.cron,
|
|
5062
|
+
timezone: row.timezone,
|
|
5063
|
+
dueAtMs: Number(row.due_at_ms),
|
|
5064
|
+
currentFireTaskId: row.current_fire_task_id ?? null,
|
|
5065
|
+
currentFireGeneration: Number(row.current_fire_generation),
|
|
5066
|
+
currentFireQueueName: row.current_fire_queue_name ?? null,
|
|
5067
|
+
sourcePin,
|
|
5068
|
+
};
|
|
5069
|
+
}
|
|
5070
|
+
|
|
5071
|
+
function cronOccurrenceLaunch(input: {
|
|
5072
|
+
schedule: PostgresSchedulerCronSchedule;
|
|
5073
|
+
launch: PlaySchedulerSubmitInput;
|
|
5074
|
+
schedulerSchema?: string | null;
|
|
5075
|
+
absurdReleaseId?: string | null;
|
|
5076
|
+
}): PlaySchedulerSubmitInput {
|
|
5077
|
+
const identity = buildCronFireRunIdentity({
|
|
5078
|
+
scheduleId: input.schedule.scheduleId,
|
|
5079
|
+
generation: input.schedule.generation,
|
|
5080
|
+
dueAtMs: input.schedule.dueAtMs,
|
|
5081
|
+
playName: input.schedule.sourcePin.playName,
|
|
5082
|
+
});
|
|
5083
|
+
const authority = input.launch.runtimeAuthority;
|
|
5084
|
+
if (
|
|
5085
|
+
!authority ||
|
|
5086
|
+
input.launch.runId !== identity.runId ||
|
|
5087
|
+
authority.workflowId !== identity.workflowId ||
|
|
5088
|
+
authority.runId !== identity.runId
|
|
5089
|
+
) {
|
|
5090
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
5091
|
+
`Cron schedule "${input.schedule.scheduleId}" did not receive a fresh authority for its pinned occurrence.`,
|
|
5092
|
+
);
|
|
5093
|
+
}
|
|
5094
|
+
return {
|
|
5095
|
+
...input.launch,
|
|
5096
|
+
schedulerSchema:
|
|
5097
|
+
input.schedulerSchema ?? input.launch.schedulerSchema ?? null,
|
|
5098
|
+
...(input.absurdReleaseId?.trim()
|
|
5099
|
+
? { absurdReleaseId: input.absurdReleaseId.trim() }
|
|
5100
|
+
: {}),
|
|
5101
|
+
runtimeAuthority: authority,
|
|
5102
|
+
runtimeProtocolVersion: protocolForNewPlayRuntimeLaunch(
|
|
5103
|
+
input.launch.runtimeProtocolVersion,
|
|
5104
|
+
),
|
|
5105
|
+
triggerSource: 'cron',
|
|
5106
|
+
requiresRunCreatedProjection: true,
|
|
5107
|
+
};
|
|
5108
|
+
}
|
|
5109
|
+
|
|
5110
|
+
export type FirePostgresSchedulerCronScheduleInput = {
|
|
5111
|
+
scheduleId: string;
|
|
5112
|
+
generation: number;
|
|
5113
|
+
/** Matches the durable timer-repair fence for this exact occurrence. */
|
|
5114
|
+
fireGeneration: number;
|
|
5115
|
+
dueAtMs: number;
|
|
5116
|
+
queueName: string;
|
|
5117
|
+
/** Built at the due instant by Vercel; never reuse the seed-time authority. */
|
|
5118
|
+
launch: PlaySchedulerSubmitInput;
|
|
5119
|
+
/** Current accepting lane, resolved by the waking worker rather than seed. */
|
|
5120
|
+
absurdReleaseId?: string | null;
|
|
5121
|
+
/** Environment scopes the release-admission fence. */
|
|
5122
|
+
releaseEnvironment?: 'preview' | 'production';
|
|
5123
|
+
};
|
|
5124
|
+
|
|
5125
|
+
/**
|
|
5126
|
+
* The cron OCC boundary. The matching schedule row is locked while this one
|
|
5127
|
+
* CTE inserts the immutable Run, its play-run task, run.created fact, next
|
|
5128
|
+
* cron-fire task, and next cursor. A task replay after commit sees a changed
|
|
5129
|
+
* cursor and is a harmless `superseded` no-op.
|
|
5130
|
+
*/
|
|
5131
|
+
export async function firePostgresSchedulerCronSchedule(
|
|
5132
|
+
client: PostgresSchedulerQueryClient,
|
|
5133
|
+
input: FirePostgresSchedulerCronScheduleInput,
|
|
5134
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
5135
|
+
): Promise<
|
|
5136
|
+
| { status: 'admitted'; runId: string; successorDueAtMs: number }
|
|
5137
|
+
| { status: 'superseded' }
|
|
5138
|
+
> {
|
|
5139
|
+
assertPostgresSchedulerCronScheduleInput(input);
|
|
5140
|
+
const schedule = await readPostgresSchedulerCronSchedule(
|
|
5141
|
+
client,
|
|
5142
|
+
input.scheduleId,
|
|
5143
|
+
options,
|
|
5144
|
+
);
|
|
5145
|
+
if (
|
|
5146
|
+
!schedule ||
|
|
5147
|
+
schedule.state !== 'active' ||
|
|
5148
|
+
schedule.generation !== input.generation ||
|
|
5149
|
+
schedule.dueAtMs !== input.dueAtMs ||
|
|
5150
|
+
schedule.currentFireGeneration !== input.fireGeneration
|
|
5151
|
+
) {
|
|
5152
|
+
return { status: 'superseded' };
|
|
5153
|
+
}
|
|
5154
|
+
let successorDueAtMs: number;
|
|
5155
|
+
try {
|
|
5156
|
+
successorDueAtMs = advanceWorkflowCronSchedule(
|
|
5157
|
+
schedule.cron,
|
|
5158
|
+
Math.max(Date.now(), schedule.dueAtMs),
|
|
5159
|
+
schedule.timezone,
|
|
5160
|
+
);
|
|
5161
|
+
} catch (error) {
|
|
5162
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
5163
|
+
`Cron schedule "${schedule.scheduleId}" has an invalid pinned expression or timezone: ${
|
|
5164
|
+
error instanceof Error ? error.message : String(error)
|
|
5165
|
+
}`,
|
|
5166
|
+
);
|
|
5167
|
+
}
|
|
5168
|
+
if (
|
|
5169
|
+
!Number.isSafeInteger(successorDueAtMs) ||
|
|
5170
|
+
successorDueAtMs <= schedule.dueAtMs
|
|
5171
|
+
) {
|
|
5172
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
5173
|
+
`Cron schedule "${schedule.scheduleId}" did not produce a future successor occurrence.`,
|
|
5174
|
+
);
|
|
5175
|
+
}
|
|
5176
|
+
const launch = cronOccurrenceLaunch({
|
|
5177
|
+
schedule,
|
|
5178
|
+
launch: input.launch,
|
|
5179
|
+
schedulerSchema: options?.schema,
|
|
5180
|
+
absurdReleaseId: input.absurdReleaseId,
|
|
5181
|
+
});
|
|
5182
|
+
const releaseId = launch.absurdReleaseId?.trim();
|
|
5183
|
+
if (!releaseId) {
|
|
5184
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
5185
|
+
`Cron schedule "${schedule.scheduleId}" has no fresh Absurd release lane.`,
|
|
5186
|
+
);
|
|
5187
|
+
}
|
|
5188
|
+
const releaseEnvironment =
|
|
5189
|
+
input.releaseEnvironment ?? launch.absurdReleaseEnvironment ?? 'production';
|
|
5190
|
+
const releaseAdmissionKey = `${releaseEnvironment}:${releaseId}`;
|
|
5191
|
+
const playTask = buildAbsurdPlayRunTask({
|
|
5192
|
+
launch,
|
|
5193
|
+
schedulerSchema: options?.schema,
|
|
5194
|
+
});
|
|
5195
|
+
const successorTask = buildAbsurdCronFireTask({
|
|
5196
|
+
scheduleId: schedule.scheduleId,
|
|
5197
|
+
generation: schedule.generation,
|
|
5198
|
+
fireGeneration: 1,
|
|
5199
|
+
dueAtMs: successorDueAtMs,
|
|
5200
|
+
schedulerSchema: options?.schema ?? launch.schedulerSchema ?? null,
|
|
5201
|
+
orgId: launch.orgId,
|
|
5202
|
+
});
|
|
5203
|
+
const result = await client.query<{
|
|
5204
|
+
admitted: boolean;
|
|
5205
|
+
release_accepting: boolean;
|
|
5206
|
+
}>(
|
|
5207
|
+
`
|
|
5208
|
+
WITH schedule_lock AS MATERIALIZED (
|
|
5209
|
+
SELECT pg_advisory_xact_lock(
|
|
5210
|
+
${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
|
|
5211
|
+
${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
|
|
5212
|
+
'cron-schedule:' || $1::text
|
|
5213
|
+
)
|
|
5214
|
+
)
|
|
5215
|
+
), matching_schedule AS MATERIALIZED (
|
|
5216
|
+
SELECT schedule.schedule_id
|
|
5217
|
+
FROM schedule_lock
|
|
5218
|
+
JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5219
|
+
ON schedule.schedule_id = $1::text
|
|
5220
|
+
JOIN ${tableName('runtime_release_admission', options)} AS release_gate
|
|
5221
|
+
ON release_gate.release_id = $16::text
|
|
5222
|
+
WHERE schedule.generation = $2::bigint
|
|
5223
|
+
AND schedule.state = 'active'
|
|
5224
|
+
AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
|
|
5225
|
+
AND schedule.current_fire_generation = $15::bigint
|
|
5226
|
+
AND release_gate.accepting
|
|
5227
|
+
FOR UPDATE
|
|
5228
|
+
), existing_run AS MATERIALIZED (
|
|
5229
|
+
SELECT runs.run_id
|
|
5230
|
+
FROM matching_schedule
|
|
5231
|
+
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = $5::text
|
|
5232
|
+
), play_task_request AS (
|
|
5233
|
+
SELECT $7::text AS queue_name,
|
|
5234
|
+
$8::text AS task_name,
|
|
5235
|
+
$9::jsonb AS params,
|
|
5236
|
+
$10::jsonb AS options
|
|
5237
|
+
WHERE EXISTS (SELECT 1 FROM matching_schedule)
|
|
5238
|
+
AND NOT EXISTS (SELECT 1 FROM existing_run)
|
|
5239
|
+
), play_task_spawn AS MATERIALIZED (
|
|
5240
|
+
SELECT spawned.task_id::text AS task_id
|
|
5241
|
+
FROM play_task_request
|
|
5242
|
+
CROSS JOIN LATERAL absurd.spawn_task(
|
|
5243
|
+
play_task_request.queue_name,
|
|
5244
|
+
play_task_request.task_name,
|
|
5245
|
+
play_task_request.params,
|
|
5246
|
+
play_task_request.options
|
|
5247
|
+
) AS spawned
|
|
5248
|
+
), run_insert AS (
|
|
5249
|
+
INSERT INTO ${tableName('runs', options)} (
|
|
5250
|
+
run_id, org_id, play_id, play_name, graph_hash, runtime_profile,
|
|
5251
|
+
artifact_storage_key, launch_json, input_json, status,
|
|
5252
|
+
absurd_task_id, admission_json
|
|
5253
|
+
)
|
|
5254
|
+
SELECT $5::text,
|
|
5255
|
+
$6::jsonb ->> 'orgId',
|
|
5256
|
+
NULLIF($6::jsonb ->> 'playId', ''),
|
|
5257
|
+
$6::jsonb ->> 'playName',
|
|
5258
|
+
$6::jsonb ->> 'graphHash',
|
|
5259
|
+
$6::jsonb ->> 'runtimeBackend',
|
|
5260
|
+
$6::jsonb ->> 'artifactStorageKey',
|
|
5261
|
+
$6::jsonb,
|
|
5262
|
+
coalesce($6::jsonb -> 'input', '{}'::jsonb),
|
|
5263
|
+
'queued',
|
|
5264
|
+
(SELECT task_id FROM play_task_spawn),
|
|
5265
|
+
$6::jsonb - ARRAY[
|
|
5266
|
+
'executorToken',
|
|
5267
|
+
'preloadedDbSessionUnwrapKey',
|
|
5268
|
+
'preloadedDbSessions',
|
|
5269
|
+
'submittedAtMs',
|
|
5270
|
+
'runtimeDeployVersion',
|
|
5271
|
+
'coordinatorInternalToken',
|
|
5272
|
+
'vercelProtectionBypassToken'
|
|
5273
|
+
]::text[]
|
|
5274
|
+
WHERE EXISTS (SELECT 1 FROM matching_schedule)
|
|
5275
|
+
AND NOT EXISTS (SELECT 1 FROM existing_run)
|
|
5276
|
+
RETURNING run_id
|
|
5277
|
+
), successor_task_request AS (
|
|
5278
|
+
SELECT $11::text AS queue_name,
|
|
5279
|
+
$12::text AS task_name,
|
|
5280
|
+
$13::jsonb AS params,
|
|
5281
|
+
$14::jsonb AS options
|
|
5282
|
+
WHERE EXISTS (SELECT 1 FROM run_insert)
|
|
5283
|
+
), successor_task_spawn AS MATERIALIZED (
|
|
5284
|
+
SELECT spawned.task_id::text AS task_id
|
|
5285
|
+
FROM successor_task_request
|
|
5286
|
+
CROSS JOIN LATERAL absurd.spawn_task(
|
|
5287
|
+
successor_task_request.queue_name,
|
|
5288
|
+
successor_task_request.task_name,
|
|
5289
|
+
successor_task_request.params,
|
|
5290
|
+
successor_task_request.options
|
|
5291
|
+
) AS spawned
|
|
5292
|
+
), outbox_insert AS (
|
|
5293
|
+
INSERT INTO ${tableName('outbox', options)} (
|
|
5294
|
+
event_id, run_id, event_type, payload_json, created_at
|
|
5295
|
+
)
|
|
5296
|
+
SELECT
|
|
5297
|
+
'run-created:' || run_insert.run_id,
|
|
5298
|
+
run_insert.run_id,
|
|
5299
|
+
'run.created',
|
|
5300
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
5301
|
+
'orgId', $6::jsonb ->> 'orgId',
|
|
5302
|
+
'playId', $6::jsonb ->> 'playId',
|
|
5303
|
+
'playName', $6::jsonb ->> 'playName',
|
|
5304
|
+
'definitionName', $6::jsonb ->> 'definitionName',
|
|
5305
|
+
'playReference', $6::jsonb ->> 'playReference',
|
|
5306
|
+
'definitionScope', $6::jsonb ->> 'definitionScope',
|
|
5307
|
+
'revisionId', $6::jsonb ->> 'revisionId',
|
|
5308
|
+
'workflowFamilyKey', $6::jsonb ->> 'workflowFamilyKey',
|
|
5309
|
+
'artifactStorageKey', $6::jsonb ->> 'artifactStorageKey',
|
|
5310
|
+
'artifactHash', $6::jsonb ->> 'artifactHash',
|
|
5311
|
+
'graphHash', $6::jsonb ->> 'graphHash',
|
|
5312
|
+
'runtimeBackend', $6::jsonb ->> 'runtimeBackend',
|
|
5313
|
+
'schedulerBackend', $6::jsonb ->> 'schedulerBackend',
|
|
5314
|
+
'schedulerSchema', $6::jsonb ->> 'schedulerSchema',
|
|
5315
|
+
'executionProfile', $6::jsonb ->> 'executionProfile',
|
|
5316
|
+
'runtimeReleaseId', $6::jsonb ->> 'runtimeReleaseId',
|
|
5317
|
+
'runtimeDeployVersion', $6::jsonb ->> 'runtimeDeployVersion',
|
|
5318
|
+
'runtimeCallbackBaseUrl', $6::jsonb ->> 'runtimeCallbackBaseUrl',
|
|
5319
|
+
'coordinatorUrl', $6::jsonb ->> 'coordinatorUrl',
|
|
5320
|
+
'coordinatorWorkerName', $6::jsonb ->> 'coordinatorWorkerName',
|
|
5321
|
+
'runtimeWorkflowName', $6::jsonb ->> 'runtimeWorkflowName',
|
|
5322
|
+
'runtimeHarnessWorkerName', $6::jsonb ->> 'runtimeHarnessWorkerName',
|
|
5323
|
+
'maxCreditsPerRun', $6::jsonb -> 'maxCreditsPerRun',
|
|
5324
|
+
'triggerSource', $6::jsonb ->> 'triggerSource',
|
|
5325
|
+
'source', $6::jsonb ->> 'source',
|
|
5326
|
+
'staticPipeline', $6::jsonb -> 'staticPipeline',
|
|
5327
|
+
'inputFileId', $6::jsonb ->> 'inputFileId',
|
|
5328
|
+
'inputBytes', $6::jsonb -> 'inputBytes',
|
|
5329
|
+
'inputSha256', $6::jsonb ->> 'inputSha256',
|
|
5330
|
+
'replayedFromRunId', $6::jsonb ->> 'replayedFromRunId',
|
|
5331
|
+
'parentRunId', $6::jsonb ->> 'parentRunId',
|
|
5332
|
+
'rootRunId', $6::jsonb ->> 'rootRunId',
|
|
5333
|
+
'asyncLaunchKey', $6::jsonb ->> 'asyncLaunchKey',
|
|
5334
|
+
'asyncInvocationKey', $6::jsonb ->> 'asyncInvocationKey',
|
|
5335
|
+
'asyncRequestFingerprint', $6::jsonb ->> 'asyncRequestFingerprint',
|
|
5336
|
+
'asyncAncestryPlayIds', $6::jsonb -> 'asyncAncestryPlayIds',
|
|
5337
|
+
'secretRefs', $6::jsonb -> 'secretRefs',
|
|
5338
|
+
'inputSummary', $6::jsonb -> 'inputSummary'
|
|
3991
5339
|
)),
|
|
3992
5340
|
${causalOutboxCreatedAt('run_insert.run_id', options)}
|
|
3993
5341
|
FROM run_insert
|
|
3994
|
-
-- Old Vercel launchers do not carry the immutable admission envelope.
|
|
3995
|
-
-- During a Convex-first rollout they retain their established start
|
|
3996
|
-
-- projection; only the new launcher opts into queued projection.
|
|
3997
|
-
WHERE $8::jsonb ? 'playReference'
|
|
3998
5342
|
ON CONFLICT (event_id) DO NOTHING
|
|
3999
|
-
|
|
5343
|
+
), schedule_advance AS (
|
|
5344
|
+
UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5345
|
+
SET current_due_at = to_timestamp($4::double precision / 1000.0),
|
|
5346
|
+
current_fire_task_id = (SELECT task_id FROM successor_task_spawn),
|
|
5347
|
+
current_fire_generation = 1,
|
|
5348
|
+
current_fire_queue_name = $11::text,
|
|
5349
|
+
last_fired_at = clock_timestamp(),
|
|
5350
|
+
updated_at = now()
|
|
5351
|
+
WHERE schedule.schedule_id = $1::text
|
|
5352
|
+
AND schedule.generation = $2::bigint
|
|
5353
|
+
AND schedule.state = 'active'
|
|
5354
|
+
AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
|
|
5355
|
+
AND schedule.current_fire_generation = $15::bigint
|
|
5356
|
+
AND EXISTS (SELECT 1 FROM run_insert)
|
|
5357
|
+
RETURNING schedule_id
|
|
4000
5358
|
)
|
|
4001
|
-
SELECT (
|
|
4002
|
-
(NOT admission.existing_run AND admission.accepting_new
|
|
4003
|
-
AND admission.parent_accepting)
|
|
4004
|
-
OR admission.matching_existing
|
|
4005
|
-
) AS accepted,
|
|
4006
|
-
admission.accepting_new,
|
|
4007
|
-
admission.parent_accepting,
|
|
4008
|
-
admission.parent_attempt_current,
|
|
4009
|
-
admission.parent_terminal,
|
|
4010
|
-
admission.existing_run,
|
|
4011
|
-
admission.matching_existing,
|
|
5359
|
+
SELECT EXISTS (SELECT 1 FROM schedule_advance) AS admitted,
|
|
4012
5360
|
coalesce(
|
|
4013
|
-
(SELECT
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
(
|
|
4019
|
-
admission.matching_existing
|
|
4020
|
-
AND NOT ${recoverExistingTaskSql}
|
|
4021
|
-
AND EXISTS (
|
|
4022
|
-
SELECT 1
|
|
4023
|
-
FROM existing_run
|
|
4024
|
-
WHERE existing_run.status = 'queued'
|
|
4025
|
-
AND existing_run.absurd_task_id IS NULL
|
|
4026
|
-
)
|
|
4027
|
-
) AS task_recovery_required,
|
|
4028
|
-
CASE WHEN EXISTS (SELECT 1 FROM run_insert)
|
|
4029
|
-
OR EXISTS (SELECT 1 FROM run_recovery)
|
|
4030
|
-
THEN pg_notify($10, $11)
|
|
4031
|
-
END
|
|
4032
|
-
FROM admission
|
|
5361
|
+
(SELECT accepting
|
|
5362
|
+
FROM ${tableName('runtime_release_admission', options)}
|
|
5363
|
+
WHERE release_id = $16::text),
|
|
5364
|
+
false
|
|
5365
|
+
) AS release_accepting
|
|
4033
5366
|
`,
|
|
4034
5367
|
[
|
|
4035
|
-
|
|
4036
|
-
|
|
4037
|
-
|
|
4038
|
-
|
|
4039
|
-
|
|
4040
|
-
|
|
4041
|
-
input.
|
|
4042
|
-
|
|
4043
|
-
stringifyPostgresJson(
|
|
4044
|
-
|
|
4045
|
-
|
|
5368
|
+
schedule.scheduleId,
|
|
5369
|
+
schedule.generation,
|
|
5370
|
+
schedule.dueAtMs,
|
|
5371
|
+
successorDueAtMs,
|
|
5372
|
+
launch.runId,
|
|
5373
|
+
stringifyPostgresJson(launch),
|
|
5374
|
+
input.queueName,
|
|
5375
|
+
playTask.taskName,
|
|
5376
|
+
stringifyPostgresJson(playTask.params),
|
|
5377
|
+
stringifyPostgresJson(playTask.options),
|
|
5378
|
+
input.queueName,
|
|
5379
|
+
successorTask.taskName,
|
|
5380
|
+
stringifyPostgresJson(successorTask.params),
|
|
5381
|
+
stringifyPostgresJson(successorTask.options),
|
|
5382
|
+
schedule.currentFireGeneration,
|
|
4046
5383
|
releaseAdmissionKey,
|
|
4047
|
-
releaseAdmissionLockKey,
|
|
4048
|
-
useLegacyAdmissionKey,
|
|
4049
|
-
releaseId,
|
|
4050
|
-
parentRunId,
|
|
4051
|
-
asyncRootRunId,
|
|
4052
|
-
asyncParentRunAttempt,
|
|
4053
|
-
asyncParentRunLeaseToken,
|
|
4054
|
-
asyncParentAttemptRequired,
|
|
4055
|
-
...(absurdTask
|
|
4056
|
-
? [
|
|
4057
|
-
absurdTask.queueName,
|
|
4058
|
-
absurdTask.taskName,
|
|
4059
|
-
stringifyPostgresJson(absurdTask.params),
|
|
4060
|
-
stringifyPostgresJson(absurdTask.options),
|
|
4061
|
-
]
|
|
4062
|
-
: []),
|
|
4063
5384
|
],
|
|
4064
5385
|
);
|
|
4065
|
-
|
|
4066
|
-
if (
|
|
4067
|
-
row?.accepted === false &&
|
|
4068
|
-
!row.existing_run &&
|
|
4069
|
-
row.parent_accepting === false
|
|
4070
|
-
) {
|
|
4071
|
-
if (
|
|
4072
|
-
row?.parent_terminal === false &&
|
|
4073
|
-
row?.parent_attempt_current === false
|
|
4074
|
-
) {
|
|
4075
|
-
throw new Error(
|
|
4076
|
-
`CTX_RUN_PLAY_ASYNC_PARENT_ATTEMPT_STALE: parent Run "${parentRunId}" no longer has the executor attempt that requested this child.`,
|
|
4077
|
-
);
|
|
4078
|
-
}
|
|
5386
|
+
if (result.rows[0]?.release_accepting !== true) {
|
|
4079
5387
|
throw new Error(
|
|
4080
|
-
`
|
|
5388
|
+
`Cron schedule "${schedule.scheduleId}" cannot fire because Absurd release "${releaseAdmissionKey}" is not accepting admissions.`,
|
|
4081
5389
|
);
|
|
4082
5390
|
}
|
|
4083
|
-
if (
|
|
5391
|
+
if (result.rows[0]?.admitted !== true) {
|
|
5392
|
+
return { status: 'superseded' };
|
|
5393
|
+
}
|
|
5394
|
+
return {
|
|
5395
|
+
status: 'admitted',
|
|
5396
|
+
runId: launch.runId,
|
|
5397
|
+
successorDueAtMs,
|
|
5398
|
+
};
|
|
5399
|
+
}
|
|
5400
|
+
|
|
5401
|
+
/** Fence a malformed/invalid timer so it cannot repeatedly consume workers. */
|
|
5402
|
+
export async function blockPostgresSchedulerCronSchedule(
|
|
5403
|
+
client: PostgresSchedulerQueryClient,
|
|
5404
|
+
input: Pick<
|
|
5405
|
+
FirePostgresSchedulerCronScheduleInput,
|
|
5406
|
+
'scheduleId' | 'generation' | 'fireGeneration' | 'dueAtMs'
|
|
5407
|
+
> & {
|
|
5408
|
+
reason: string;
|
|
5409
|
+
},
|
|
5410
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
5411
|
+
): Promise<boolean> {
|
|
5412
|
+
const result = await client.query<{ schedule_id: string }>(
|
|
5413
|
+
`
|
|
5414
|
+
UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
5415
|
+
SET state = 'blocked',
|
|
5416
|
+
block_reason = left($4::text, 1_000),
|
|
5417
|
+
updated_at = now()
|
|
5418
|
+
WHERE schedule_id = $1::text
|
|
5419
|
+
AND generation = $2::bigint
|
|
5420
|
+
AND state = 'active'
|
|
5421
|
+
AND current_due_at = to_timestamp($3::double precision / 1000.0)
|
|
5422
|
+
AND current_fire_generation = $5::bigint
|
|
5423
|
+
RETURNING schedule_id
|
|
5424
|
+
`,
|
|
5425
|
+
[
|
|
5426
|
+
input.scheduleId,
|
|
5427
|
+
input.generation,
|
|
5428
|
+
input.dueAtMs,
|
|
5429
|
+
input.reason,
|
|
5430
|
+
input.fireGeneration,
|
|
5431
|
+
],
|
|
5432
|
+
);
|
|
5433
|
+
return result.rows.length > 0;
|
|
5434
|
+
}
|
|
5435
|
+
|
|
5436
|
+
/**
|
|
5437
|
+
* Fence a Runtime-owned schedule before its source binding is disabled,
|
|
5438
|
+
* archived, suspended, or replaced. The advisory lock is shared with fire and
|
|
5439
|
+
* seed, so an already-sleeping task either commits before the lifecycle write
|
|
5440
|
+
* (a run valid at that instant) or observes `cancelled` and becomes a
|
|
5441
|
+
* superseded no-op. It can never admit after this mutation commits.
|
|
5442
|
+
*/
|
|
5443
|
+
export async function cancelPostgresSchedulerCronSchedule(
|
|
5444
|
+
client: PostgresSchedulerQueryClient,
|
|
5445
|
+
input: {
|
|
5446
|
+
scheduleId: string;
|
|
5447
|
+
generation: number;
|
|
5448
|
+
reason: string;
|
|
5449
|
+
},
|
|
5450
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
5451
|
+
): Promise<boolean> {
|
|
5452
|
+
if (!input.scheduleId.trim()) {
|
|
4084
5453
|
throw new Error(
|
|
4085
|
-
|
|
5454
|
+
'Cron schedule cancellation requires a non-empty scheduleId.',
|
|
4086
5455
|
);
|
|
4087
5456
|
}
|
|
4088
|
-
if (
|
|
5457
|
+
if (!Number.isSafeInteger(input.generation) || input.generation <= 0) {
|
|
4089
5458
|
throw new Error(
|
|
4090
|
-
|
|
5459
|
+
'Cron schedule cancellation requires a positive generation.',
|
|
4091
5460
|
);
|
|
4092
5461
|
}
|
|
5462
|
+
const result = await client.query<{ schedule_id: string }>(
|
|
5463
|
+
`
|
|
5464
|
+
WITH schedule_lock AS MATERIALIZED (
|
|
5465
|
+
SELECT pg_advisory_xact_lock(
|
|
5466
|
+
${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
|
|
5467
|
+
${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
|
|
5468
|
+
'cron-schedule:' || $1::text
|
|
5469
|
+
)
|
|
5470
|
+
)
|
|
5471
|
+
)
|
|
5472
|
+
UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5473
|
+
SET state = 'cancelled',
|
|
5474
|
+
cancellation_reason = left($3::text, 1_000),
|
|
5475
|
+
updated_at = now()
|
|
5476
|
+
FROM schedule_lock
|
|
5477
|
+
WHERE schedule.schedule_id = $1::text
|
|
5478
|
+
AND schedule.generation = $2::bigint
|
|
5479
|
+
AND schedule.state = 'active'
|
|
5480
|
+
RETURNING schedule.schedule_id
|
|
5481
|
+
`,
|
|
5482
|
+
[input.scheduleId.trim(), input.generation, input.reason],
|
|
5483
|
+
);
|
|
5484
|
+
return result.rows.length > 0;
|
|
5485
|
+
}
|
|
5486
|
+
|
|
5487
|
+
/**
|
|
5488
|
+
* Replace an observed missing, cancelled, or exhausted timer without changing
|
|
5489
|
+
* its scheduled occurrence. Callers must first inspect the engine task and
|
|
5490
|
+
* pass the observed `fireGeneration`; the CTE then advances that fence and
|
|
5491
|
+
* spawns the replacement in one transaction. A late old timer is superseded
|
|
5492
|
+
* by the generation comparison in `firePostgresSchedulerCronSchedule`.
|
|
5493
|
+
*/
|
|
5494
|
+
export async function repairPostgresSchedulerCronFire(
|
|
5495
|
+
client: PostgresSchedulerQueryClient,
|
|
5496
|
+
input: {
|
|
5497
|
+
scheduleId: string;
|
|
5498
|
+
generation: number;
|
|
5499
|
+
fireGeneration: number;
|
|
5500
|
+
dueAtMs: number;
|
|
5501
|
+
queueName: string;
|
|
5502
|
+
},
|
|
5503
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
5504
|
+
): Promise<{
|
|
5505
|
+
repaired: boolean;
|
|
5506
|
+
fireTaskId: string | null;
|
|
5507
|
+
fireGeneration: number | null;
|
|
5508
|
+
}> {
|
|
5509
|
+
assertPostgresSchedulerCronScheduleInput(input);
|
|
5510
|
+
const schedule = await readPostgresSchedulerCronSchedule(
|
|
5511
|
+
client,
|
|
5512
|
+
input.scheduleId,
|
|
5513
|
+
options,
|
|
5514
|
+
);
|
|
5515
|
+
if (
|
|
5516
|
+
!schedule ||
|
|
5517
|
+
schedule.state !== 'active' ||
|
|
5518
|
+
schedule.generation !== input.generation ||
|
|
5519
|
+
schedule.dueAtMs !== input.dueAtMs ||
|
|
5520
|
+
schedule.currentFireGeneration !== input.fireGeneration
|
|
5521
|
+
) {
|
|
5522
|
+
return { repaired: false, fireTaskId: null, fireGeneration: null };
|
|
5523
|
+
}
|
|
5524
|
+
const replacementGeneration = schedule.currentFireGeneration + 1;
|
|
5525
|
+
const task = buildAbsurdCronFireTask({
|
|
5526
|
+
scheduleId: schedule.scheduleId,
|
|
5527
|
+
generation: schedule.generation,
|
|
5528
|
+
fireGeneration: replacementGeneration,
|
|
5529
|
+
dueAtMs: schedule.dueAtMs,
|
|
5530
|
+
schedulerSchema:
|
|
5531
|
+
options?.schema ?? schedule.sourcePin.schedulerSchema ?? null,
|
|
5532
|
+
orgId: schedule.sourcePin.orgId,
|
|
5533
|
+
});
|
|
5534
|
+
const result = await client.query<{
|
|
5535
|
+
repaired: boolean;
|
|
5536
|
+
fire_task_id: string | null;
|
|
5537
|
+
}>(
|
|
5538
|
+
`
|
|
5539
|
+
WITH schedule_lock AS MATERIALIZED (
|
|
5540
|
+
SELECT pg_advisory_xact_lock(
|
|
5541
|
+
${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
|
|
5542
|
+
${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
|
|
5543
|
+
'cron-schedule:' || $1::text
|
|
5544
|
+
)
|
|
5545
|
+
)
|
|
5546
|
+
), matching_schedule AS MATERIALIZED (
|
|
5547
|
+
SELECT schedule.schedule_id
|
|
5548
|
+
FROM schedule_lock
|
|
5549
|
+
JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5550
|
+
ON schedule.schedule_id = $1::text
|
|
5551
|
+
WHERE schedule.generation = $2::bigint
|
|
5552
|
+
AND schedule.state = 'active'
|
|
5553
|
+
AND schedule.current_due_at = to_timestamp($4::double precision / 1000.0)
|
|
5554
|
+
AND schedule.current_fire_generation = $3::bigint
|
|
5555
|
+
FOR UPDATE
|
|
5556
|
+
), task_spawn AS MATERIALIZED (
|
|
5557
|
+
SELECT spawned.task_id::text AS task_id
|
|
5558
|
+
FROM matching_schedule
|
|
5559
|
+
CROSS JOIN LATERAL absurd.spawn_task(
|
|
5560
|
+
$5::text,
|
|
5561
|
+
$6::text,
|
|
5562
|
+
$7::jsonb,
|
|
5563
|
+
$8::jsonb
|
|
5564
|
+
) AS spawned
|
|
5565
|
+
), repaired_schedule AS (
|
|
5566
|
+
UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5567
|
+
SET current_fire_task_id = (SELECT task_id FROM task_spawn),
|
|
5568
|
+
current_fire_generation = $9::bigint,
|
|
5569
|
+
current_fire_queue_name = $5::text,
|
|
5570
|
+
updated_at = now()
|
|
5571
|
+
WHERE schedule.schedule_id = $1::text
|
|
5572
|
+
AND schedule.generation = $2::bigint
|
|
5573
|
+
AND schedule.state = 'active'
|
|
5574
|
+
AND schedule.current_due_at = to_timestamp($4::double precision / 1000.0)
|
|
5575
|
+
AND schedule.current_fire_generation = $3::bigint
|
|
5576
|
+
AND EXISTS (SELECT 1 FROM task_spawn)
|
|
5577
|
+
RETURNING schedule.schedule_id
|
|
5578
|
+
)
|
|
5579
|
+
SELECT EXISTS (SELECT 1 FROM repaired_schedule) AS repaired,
|
|
5580
|
+
(SELECT task_id FROM task_spawn) AS fire_task_id
|
|
5581
|
+
`,
|
|
5582
|
+
[
|
|
5583
|
+
schedule.scheduleId,
|
|
5584
|
+
schedule.generation,
|
|
5585
|
+
schedule.currentFireGeneration,
|
|
5586
|
+
schedule.dueAtMs,
|
|
5587
|
+
input.queueName,
|
|
5588
|
+
task.taskName,
|
|
5589
|
+
stringifyPostgresJson(task.params),
|
|
5590
|
+
stringifyPostgresJson(task.options),
|
|
5591
|
+
replacementGeneration,
|
|
5592
|
+
],
|
|
5593
|
+
);
|
|
5594
|
+
const row = result.rows[0];
|
|
4093
5595
|
return {
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
taskRecoveryRequired: row?.task_recovery_required === true,
|
|
5596
|
+
repaired: row?.repaired === true,
|
|
5597
|
+
fireTaskId: row?.fire_task_id ?? null,
|
|
5598
|
+
fireGeneration: row?.repaired === true ? replacementGeneration : null,
|
|
4098
5599
|
};
|
|
4099
5600
|
}
|
|
4100
5601
|
|
|
@@ -5923,6 +7424,58 @@ export async function readPostgresSchedulerLaunch(
|
|
|
5923
7424
|
}
|
|
5924
7425
|
}
|
|
5925
7426
|
|
|
7427
|
+
/**
|
|
7428
|
+
* One round-trip replay inspection for externally retried admissions. The
|
|
7429
|
+
* caller uses this only to return an already-committed Run without rebuilding
|
|
7430
|
+
* mutable launch dependencies; it is never an admission authority.
|
|
7431
|
+
*/
|
|
7432
|
+
export type PostgresSchedulerAdmissionSnapshot = {
|
|
7433
|
+
runId: string;
|
|
7434
|
+
/** Empty only for historic rows that predate an owning organization. */
|
|
7435
|
+
orgId: string;
|
|
7436
|
+
status: string;
|
|
7437
|
+
absurdTaskId: string | null;
|
|
7438
|
+
launch: PlaySchedulerSubmitInput;
|
|
7439
|
+
};
|
|
7440
|
+
|
|
7441
|
+
export async function readPostgresSchedulerAdmission(
|
|
7442
|
+
client: PostgresSchedulerQueryClient,
|
|
7443
|
+
runId: string,
|
|
7444
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
7445
|
+
): Promise<PostgresSchedulerAdmissionSnapshot | null> {
|
|
7446
|
+
const result = await client.query<{
|
|
7447
|
+
run_id: string;
|
|
7448
|
+
org_id: string | null;
|
|
7449
|
+
status: string;
|
|
7450
|
+
absurd_task_id: string | null;
|
|
7451
|
+
launch_json: unknown;
|
|
7452
|
+
}>(
|
|
7453
|
+
`
|
|
7454
|
+
SELECT runs.run_id,
|
|
7455
|
+
runs.org_id,
|
|
7456
|
+
runs.status,
|
|
7457
|
+
runs.absurd_task_id,
|
|
7458
|
+
runs.launch_json || coalesce(runs.admission_json, '{}'::jsonb)
|
|
7459
|
+
AS launch_json
|
|
7460
|
+
FROM ${tableName('runs', options)} AS runs
|
|
7461
|
+
WHERE runs.run_id = $1
|
|
7462
|
+
`,
|
|
7463
|
+
[runId],
|
|
7464
|
+
);
|
|
7465
|
+
const row = result.rows[0] ?? null;
|
|
7466
|
+
if (!row) return null;
|
|
7467
|
+
return {
|
|
7468
|
+
runId: row.run_id,
|
|
7469
|
+
orgId: row.org_id ?? '',
|
|
7470
|
+
status: row.status,
|
|
7471
|
+
absurdTaskId:
|
|
7472
|
+
typeof row.absurd_task_id === 'string' && row.absurd_task_id.trim()
|
|
7473
|
+
? row.absurd_task_id
|
|
7474
|
+
: null,
|
|
7475
|
+
launch: parseLaunchJson(row.launch_json),
|
|
7476
|
+
};
|
|
7477
|
+
}
|
|
7478
|
+
|
|
5926
7479
|
/**
|
|
5927
7480
|
* Resolve the immutable launch only when the caller proves ownership of the
|
|
5928
7481
|
* currently live scheduler attempt. Used by the Vercel token issuer so a Fly
|
|
@@ -14411,6 +15964,40 @@ function parseLaunchJson(value: unknown): PlaySchedulerSubmitInput {
|
|
|
14411
15964
|
return launch as PlaySchedulerSubmitInput;
|
|
14412
15965
|
}
|
|
14413
15966
|
|
|
15967
|
+
function parsePostgresSchedulerCronSourcePin(
|
|
15968
|
+
value: unknown,
|
|
15969
|
+
): PostgresSchedulerCronSourcePin {
|
|
15970
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
15971
|
+
throw new Error('Cron source pin is not an object.');
|
|
15972
|
+
}
|
|
15973
|
+
const pin = value as Partial<PostgresSchedulerCronSourcePin>;
|
|
15974
|
+
if (
|
|
15975
|
+
typeof pin.orgId !== 'string' ||
|
|
15976
|
+
typeof pin.playName !== 'string' ||
|
|
15977
|
+
typeof pin.artifactStorageKey !== 'string' ||
|
|
15978
|
+
!('input' in pin) ||
|
|
15979
|
+
(pin.source !== 'published' && pin.source !== 'draft')
|
|
15980
|
+
) {
|
|
15981
|
+
throw new Error('Cron source pin is missing required immutable fields.');
|
|
15982
|
+
}
|
|
15983
|
+
return {
|
|
15984
|
+
orgId: pin.orgId,
|
|
15985
|
+
userId: typeof pin.userId === 'string' ? pin.userId : null,
|
|
15986
|
+
playName: pin.playName,
|
|
15987
|
+
revisionId: typeof pin.revisionId === 'string' ? pin.revisionId : null,
|
|
15988
|
+
artifactStorageKey: pin.artifactStorageKey,
|
|
15989
|
+
artifactKind: pin.artifactKind === 'cjs_node20' ? 'cjs_node20' : null,
|
|
15990
|
+
artifactHash:
|
|
15991
|
+
typeof pin.artifactHash === 'string' ? pin.artifactHash : null,
|
|
15992
|
+
graphHash: typeof pin.graphHash === 'string' ? pin.graphHash : null,
|
|
15993
|
+
staticPipeline: pin.staticPipeline ?? null,
|
|
15994
|
+
input: pin.input as PlaySchedulerSubmitInput['input'],
|
|
15995
|
+
source: pin.source,
|
|
15996
|
+
schedulerSchema:
|
|
15997
|
+
typeof pin.schedulerSchema === 'string' ? pin.schedulerSchema : null,
|
|
15998
|
+
};
|
|
15999
|
+
}
|
|
16000
|
+
|
|
14414
16001
|
function errorMessage(error: unknown): string {
|
|
14415
16002
|
if (error && typeof error === 'object') {
|
|
14416
16003
|
const message = (error as { message?: unknown }).message;
|