deepline 0.3.113 → 0.3.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +3 -1
- package/dist/bundling-sources/sdk/src/compat.ts +167 -11
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/skills-version.ts +27 -1
- package/dist/bundling-sources/shared_libs/monitors/inbound-queue.ts +6 -2
- package/dist/bundling-sources/shared_libs/observability/queue-health.ts +291 -6
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +24 -24
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +2 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +29 -1
- package/dist/bundling-sources/shared_libs/play-runtime/admission-transport-policy.ts +91 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/direct-admission-deadline.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +325 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-admission-outcome.ts +134 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-webhook-admission-probe.ts +32 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-webhook-identity.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-cron-fire-task.ts +100 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +313 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +111 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +67 -8
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1394 -22
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +74 -13
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts +79 -15
- package/dist/bundling-sources/shared_libs/play-runtime/webhook-receipt-retry.ts +21 -21
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +4 -1
- package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +64 -0
- package/dist/bundling-sources/shared_libs/product-notifications/default-policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/product-notifications/events.ts +20 -1
- package/dist/bundling-sources/shared_libs/security/receipt-admission-token.ts +224 -0
- package/dist/cli/index.js +574 -114
- package/dist/cli/index.mjs +592 -133
- package/dist/index.d.mts +54 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.js +8 -3
- package/dist/index.mjs +8 -3
- package/dist/install-integrity.json +7 -0
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -159,6 +159,11 @@ import {
|
|
|
159
159
|
} from './scheduler-launch-read-observability';
|
|
160
160
|
import { emitPlayTelemetry } from '../runtime-telemetry';
|
|
161
161
|
import { buildAbsurdPlayRunTask } from './absurd-play-run-task';
|
|
162
|
+
import {
|
|
163
|
+
buildAbsurdCronFireTask,
|
|
164
|
+
buildCronFireRunIdentity,
|
|
165
|
+
} from './absurd-cron-fire-task';
|
|
166
|
+
import { advanceWorkflowCronSchedule } from '@deepline/workflows/scheduler';
|
|
162
167
|
|
|
163
168
|
type QueryResult<
|
|
164
169
|
Row extends Record<string, unknown> = Record<string, unknown>,
|
|
@@ -212,16 +217,23 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
|
|
|
212
217
|
* interrupted deployment cannot claim a newer version without its required
|
|
213
218
|
* objects.
|
|
214
219
|
*/
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
|
|
220
|
+
// v37 adds the runtime-owned cron schedule cursor and its due-time index. The
|
|
221
|
+
// table is additive: v36 admission and outbox projection remain valid while a
|
|
222
|
+
// deploy expands the shared schema, then the marker records the completed
|
|
223
|
+
// runtime-cron capability for deployment tooling.
|
|
224
|
+
export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 37;
|
|
219
225
|
const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
|
|
220
226
|
'scheduler_schema_migrations';
|
|
221
227
|
export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
|
|
222
228
|
'runtime_scheduler_runs_open_recovery_idx';
|
|
223
229
|
const POSTGRES_SCHEDULER_DEPENDENCY_CIRCUITS_TABLE =
|
|
224
230
|
'runtime_dependency_circuits';
|
|
231
|
+
/**
|
|
232
|
+
* Runtime-owned cursor for a scheduled Play. Convex remains the authored
|
|
233
|
+
* control-plane projection; this row is the execution owner once a binding is
|
|
234
|
+
* cut over, so a Run and its successor can commit together.
|
|
235
|
+
*/
|
|
236
|
+
const POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE = 'cron_schedules';
|
|
225
237
|
const RECOVERY_CIRCUIT_OPEN_MS =
|
|
226
238
|
RUNTIME_RELIABILITY_POLICY.recovery.circuitOpenMs;
|
|
227
239
|
const RECOVERY_CIRCUIT_PROBE_LEASE_MS =
|
|
@@ -992,6 +1004,7 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
|
|
|
992
1004
|
admissionPriorityIndex?: boolean;
|
|
993
1005
|
terminalComputeBillingOutboxTrigger?: boolean;
|
|
994
1006
|
outboxWakeTrigger?: boolean;
|
|
1007
|
+
cronScheduleSchema?: boolean;
|
|
995
1008
|
}>;
|
|
996
1009
|
|
|
997
1010
|
export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
@@ -1240,6 +1253,49 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1240
1253
|
AND to_regclass($20) IS NOT NULL
|
|
1241
1254
|
AND to_regclass($21) IS NOT NULL
|
|
1242
1255
|
AND to_regclass($22) IS NOT NULL
|
|
1256
|
+
AND (
|
|
1257
|
+
$25::boolean = false
|
|
1258
|
+
OR (
|
|
1259
|
+
to_regclass($26) IS NOT NULL
|
|
1260
|
+
AND (
|
|
1261
|
+
SELECT count(DISTINCT column_name)
|
|
1262
|
+
FROM information_schema.columns
|
|
1263
|
+
WHERE table_schema = $1
|
|
1264
|
+
AND table_name = '${POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE}'
|
|
1265
|
+
AND column_name IN (
|
|
1266
|
+
'schedule_id',
|
|
1267
|
+
'generation',
|
|
1268
|
+
'state',
|
|
1269
|
+
'cron',
|
|
1270
|
+
'timezone',
|
|
1271
|
+
'current_due_at',
|
|
1272
|
+
'current_fire_task_id',
|
|
1273
|
+
'current_fire_generation',
|
|
1274
|
+
'current_fire_queue_name',
|
|
1275
|
+
'last_fired_at',
|
|
1276
|
+
'block_reason',
|
|
1277
|
+
'cancellation_reason',
|
|
1278
|
+
'source_pin_json',
|
|
1279
|
+
'launch_json',
|
|
1280
|
+
'created_at',
|
|
1281
|
+
'updated_at'
|
|
1282
|
+
)
|
|
1283
|
+
) = 16
|
|
1284
|
+
AND EXISTS (
|
|
1285
|
+
SELECT 1
|
|
1286
|
+
FROM pg_index AS index_row
|
|
1287
|
+
JOIN pg_class AS index_class
|
|
1288
|
+
ON index_class.oid = index_row.indexrelid
|
|
1289
|
+
JOIN pg_namespace AS index_namespace
|
|
1290
|
+
ON index_namespace.oid = index_class.relnamespace
|
|
1291
|
+
WHERE index_namespace.nspname = $1
|
|
1292
|
+
AND index_class.relname = 'runtime_scheduler_cron_schedules_due_idx'
|
|
1293
|
+
AND index_row.indrelid = to_regclass($26)
|
|
1294
|
+
AND index_row.indisready
|
|
1295
|
+
AND index_row.indisvalid
|
|
1296
|
+
)
|
|
1297
|
+
)
|
|
1298
|
+
)
|
|
1243
1299
|
-- Canonical Integration Operation outcomes are additive to the
|
|
1244
1300
|
-- transport receipt. Old byte-only rows remain readable by the
|
|
1245
1301
|
-- gateway, but can never be mistaken for a canonical operation.
|
|
@@ -1483,6 +1539,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1483
1539
|
`${quotedSchema}.${quoteIdent('outbox_repair_operations')}`,
|
|
1484
1540
|
requirements.outboxWakeTrigger !== false,
|
|
1485
1541
|
requirements.admissionPriorityIndex !== false,
|
|
1542
|
+
requirements.cronScheduleSchema !== false,
|
|
1543
|
+
`${quotedSchema}.${quoteIdent(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE)}`,
|
|
1486
1544
|
],
|
|
1487
1545
|
);
|
|
1488
1546
|
if (currentShape.rows[0]?.ready !== true) return false;
|
|
@@ -1712,6 +1770,68 @@ async function ensurePostgresSchedulerOutboxWakeTrigger(
|
|
|
1712
1770
|
`);
|
|
1713
1771
|
}
|
|
1714
1772
|
|
|
1773
|
+
/**
|
|
1774
|
+
* Additive, independent schema for the cron successor owner. It intentionally
|
|
1775
|
+
* does not participate in the global scheduler schema marker: older app and
|
|
1776
|
+
* worker builds neither read nor write this table, and making ordinary Run
|
|
1777
|
+
* admission wait on this rollout would turn an additive timer deployment into
|
|
1778
|
+
* an availability dependency.
|
|
1779
|
+
*/
|
|
1780
|
+
async function ensurePostgresSchedulerCronScheduleSchema(
|
|
1781
|
+
client: PostgresSchedulerQueryClient,
|
|
1782
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
1783
|
+
): Promise<void> {
|
|
1784
|
+
await client.query(`
|
|
1785
|
+
CREATE TABLE IF NOT EXISTS ${tableName(
|
|
1786
|
+
POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE,
|
|
1787
|
+
options,
|
|
1788
|
+
)} (
|
|
1789
|
+
schedule_id text PRIMARY KEY,
|
|
1790
|
+
generation bigint NOT NULL CHECK (generation > 0),
|
|
1791
|
+
state text NOT NULL CHECK (state IN ('active', 'blocked', 'cancelled')),
|
|
1792
|
+
cron text NOT NULL,
|
|
1793
|
+
timezone text,
|
|
1794
|
+
current_due_at timestamptz NOT NULL,
|
|
1795
|
+
current_fire_task_id text,
|
|
1796
|
+
current_fire_generation bigint NOT NULL DEFAULT 1 CHECK (current_fire_generation > 0),
|
|
1797
|
+
current_fire_queue_name text,
|
|
1798
|
+
last_fired_at timestamptz,
|
|
1799
|
+
block_reason text,
|
|
1800
|
+
cancellation_reason text,
|
|
1801
|
+
source_pin_json jsonb,
|
|
1802
|
+
launch_json jsonb NOT NULL,
|
|
1803
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
1804
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
1805
|
+
)
|
|
1806
|
+
`);
|
|
1807
|
+
await client.query(`
|
|
1808
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1809
|
+
ADD COLUMN IF NOT EXISTS block_reason text
|
|
1810
|
+
`);
|
|
1811
|
+
await client.query(`
|
|
1812
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1813
|
+
ADD COLUMN IF NOT EXISTS current_fire_generation bigint NOT NULL DEFAULT 1
|
|
1814
|
+
`);
|
|
1815
|
+
await client.query(`
|
|
1816
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1817
|
+
ADD COLUMN IF NOT EXISTS current_fire_queue_name text
|
|
1818
|
+
`);
|
|
1819
|
+
await client.query(`
|
|
1820
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1821
|
+
ADD COLUMN IF NOT EXISTS cancellation_reason text
|
|
1822
|
+
`);
|
|
1823
|
+
await client.query(`
|
|
1824
|
+
ALTER TABLE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1825
|
+
ADD COLUMN IF NOT EXISTS source_pin_json jsonb
|
|
1826
|
+
`);
|
|
1827
|
+
await client.query(`
|
|
1828
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_cron_schedules_due_idx
|
|
1829
|
+
ON ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
1830
|
+
(current_due_at, schedule_id)
|
|
1831
|
+
WHERE state = 'active'
|
|
1832
|
+
`);
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1715
1835
|
export async function ensurePostgresSchedulerSchema(
|
|
1716
1836
|
client: PostgresSchedulerQueryClient,
|
|
1717
1837
|
options?: Pick<PostgresSchedulerOptions, 'schema'> & {
|
|
@@ -1817,7 +1937,7 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
1817
1937
|
// trigger as a narrow migration: replaying historical schema DDL in one
|
|
1818
1938
|
// transaction against a live worker fleet can invert table-lock order with
|
|
1819
1939
|
// terminal writers.
|
|
1820
|
-
const
|
|
1940
|
+
const schemaIsCurrentWithoutAdditiveFastPathObjects =
|
|
1821
1941
|
!schemaIsCurrent &&
|
|
1822
1942
|
(await hasCurrentPostgresSchedulerSchemaShape(client, options, {
|
|
1823
1943
|
openRecoveryIndex: !options?.deferOpenRecoveryIndex,
|
|
@@ -1825,17 +1945,10 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
1825
1945
|
admissionPriorityIndex: !options?.deferRuntimeAdminIndexes,
|
|
1826
1946
|
terminalComputeBillingOutboxTrigger: false,
|
|
1827
1947
|
outboxWakeTrigger: false,
|
|
1828
|
-
|
|
1829
|
-
const schemaIsCurrentWithoutTerminalComputeBillingOutboxTrigger =
|
|
1830
|
-
!schemaIsCurrent &&
|
|
1831
|
-
(await hasCurrentPostgresSchedulerSchemaShape(client, options, {
|
|
1832
|
-
openRecoveryIndex: !options?.deferOpenRecoveryIndex,
|
|
1833
|
-
runtimeAdminIndexes: !options?.deferRuntimeAdminIndexes,
|
|
1834
|
-
admissionPriorityIndex: !options?.deferRuntimeAdminIndexes,
|
|
1835
|
-
terminalComputeBillingOutboxTrigger: false,
|
|
1948
|
+
cronScheduleSchema: false,
|
|
1836
1949
|
}));
|
|
1837
1950
|
const schemaCoreIsCurrent =
|
|
1838
|
-
schemaIsCurrent ||
|
|
1951
|
+
schemaIsCurrent || schemaIsCurrentWithoutAdditiveFastPathObjects;
|
|
1839
1952
|
// Older workers can still create a release-only fence after the current
|
|
1840
1953
|
// schema marker has been written. The metadata is current, but the data
|
|
1841
1954
|
// must be copied before a new worker trusts the fast path.
|
|
@@ -1853,13 +1966,14 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
1853
1966
|
const hasLegacyAdmissionRows =
|
|
1854
1967
|
legacyAdmissionRows?.rows[0]?.present === true;
|
|
1855
1968
|
if (schemaCoreIsCurrent) {
|
|
1856
|
-
if (
|
|
1969
|
+
if (!schemaIsCurrent) {
|
|
1857
1970
|
await ensurePostgresSchedulerTerminalComputeBillingOutboxTrigger(
|
|
1858
1971
|
client,
|
|
1859
1972
|
options,
|
|
1860
1973
|
);
|
|
1861
1974
|
}
|
|
1862
1975
|
await ensurePostgresSchedulerOutboxWakeTrigger(client, options);
|
|
1976
|
+
await ensurePostgresSchedulerCronScheduleSchema(client, options);
|
|
1863
1977
|
if (hasLegacyAdmissionRows) {
|
|
1864
1978
|
await migrateLegacyReleaseAdmissionRows(client, options);
|
|
1865
1979
|
}
|
|
@@ -3111,6 +3225,7 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
3111
3225
|
)} (version) VALUES ($1) ON CONFLICT (version) DO NOTHING`,
|
|
3112
3226
|
[POSTGRES_SCHEDULER_SCHEMA_VERSION],
|
|
3113
3227
|
);
|
|
3228
|
+
await ensurePostgresSchedulerCronScheduleSchema(client, options);
|
|
3114
3229
|
if (hadOutbox.rows[0]?.present !== true) {
|
|
3115
3230
|
await client.query(
|
|
3116
3231
|
`
|
|
@@ -3518,17 +3633,24 @@ export async function ensureAbsurdSchema(
|
|
|
3518
3633
|
client: PostgresSchedulerQueryClient,
|
|
3519
3634
|
): Promise<void> {
|
|
3520
3635
|
const [lockClassId, lockObjectId] = absurdSchemaInitAdvisoryLockKey();
|
|
3521
|
-
|
|
3636
|
+
let stage = 'begin transaction';
|
|
3637
|
+
let transactionStarted = false;
|
|
3522
3638
|
let committed = false;
|
|
3523
3639
|
try {
|
|
3640
|
+
await client.query('BEGIN');
|
|
3641
|
+
transactionStarted = true;
|
|
3642
|
+
|
|
3643
|
+
stage = 'set bootstrap statement timeout';
|
|
3524
3644
|
await client.query(
|
|
3525
3645
|
`SELECT set_config('statement_timeout', '${POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS}', true)`,
|
|
3526
3646
|
);
|
|
3647
|
+
stage = 'acquire Absurd schema bootstrap advisory lock';
|
|
3527
3648
|
await client.query(
|
|
3528
3649
|
`SELECT pg_advisory_xact_lock($1::integer, $2::integer)`,
|
|
3529
3650
|
[lockClassId, lockObjectId],
|
|
3530
3651
|
);
|
|
3531
3652
|
|
|
3653
|
+
stage = 'read Absurd schema catalog marker';
|
|
3532
3654
|
const present = await client.query<{ present: boolean }>(
|
|
3533
3655
|
`SELECT EXISTS (
|
|
3534
3656
|
SELECT 1
|
|
@@ -3540,11 +3662,13 @@ export async function ensureAbsurdSchema(
|
|
|
3540
3662
|
);
|
|
3541
3663
|
|
|
3542
3664
|
if (present.rows[0]?.present) {
|
|
3665
|
+
stage = 'read installed Absurd schema version';
|
|
3543
3666
|
const versionRow = await client.query<{ v: string | null }>(
|
|
3544
3667
|
`SELECT absurd.get_schema_version() AS v`,
|
|
3545
3668
|
);
|
|
3546
3669
|
const installed = versionRow.rows[0]?.v ?? null;
|
|
3547
3670
|
if (installed === ABSURD_SCHEMA_VERSION) {
|
|
3671
|
+
stage = 'read Absurd compatibility marker catalog entry';
|
|
3548
3672
|
const marker = await client.query<{ present: boolean }>(
|
|
3549
3673
|
`SELECT EXISTS (
|
|
3550
3674
|
SELECT 1
|
|
@@ -3554,6 +3678,7 @@ export async function ensureAbsurdSchema(
|
|
|
3554
3678
|
AND p.proname = 'deepline_no_scheduler_org_cap_compatibility_version'
|
|
3555
3679
|
) AS present`,
|
|
3556
3680
|
);
|
|
3681
|
+
stage = 'read Absurd compatibility marker version';
|
|
3557
3682
|
const markerVersion = marker.rows[0]?.present
|
|
3558
3683
|
? await client.query<{ v: string | null }>(
|
|
3559
3684
|
`SELECT absurd.deepline_no_scheduler_org_cap_compatibility_version() AS v`,
|
|
@@ -3563,13 +3688,16 @@ export async function ensureAbsurdSchema(
|
|
|
3563
3688
|
markerVersion?.rows[0]?.v !==
|
|
3564
3689
|
ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION
|
|
3565
3690
|
) {
|
|
3691
|
+
stage = 'apply Absurd no-org-cap compatibility function patch';
|
|
3566
3692
|
await client.query(
|
|
3567
3693
|
readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(),
|
|
3568
3694
|
);
|
|
3695
|
+
stage = 'apply Absurd no-org-cap compatibility marker patch';
|
|
3569
3696
|
await client.query(
|
|
3570
3697
|
readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(),
|
|
3571
3698
|
);
|
|
3572
3699
|
}
|
|
3700
|
+
stage = 'commit verified Absurd schema bootstrap';
|
|
3573
3701
|
await client.query('COMMIT');
|
|
3574
3702
|
committed = true;
|
|
3575
3703
|
return;
|
|
@@ -3579,32 +3707,42 @@ export async function ensureAbsurdSchema(
|
|
|
3579
3707
|
`"${ABSURD_SCHEMA_VERSION}". Continuing; the next SQL operation ` +
|
|
3580
3708
|
'will report any missing relation, column, or function.',
|
|
3581
3709
|
);
|
|
3710
|
+
stage = 'commit non-current Absurd schema marker';
|
|
3582
3711
|
await client.query('COMMIT');
|
|
3583
3712
|
committed = true;
|
|
3584
3713
|
return;
|
|
3585
3714
|
}
|
|
3586
3715
|
|
|
3716
|
+
stage = 'apply full Absurd schema';
|
|
3587
3717
|
await client.query(readAbsurdSchemaSql());
|
|
3718
|
+
stage = 'apply Absurd no-org-cap compatibility function patch';
|
|
3588
3719
|
await client.query(readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql());
|
|
3720
|
+
stage = 'apply Absurd no-org-cap compatibility marker patch';
|
|
3589
3721
|
await client.query(
|
|
3590
3722
|
readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(),
|
|
3591
3723
|
);
|
|
3724
|
+
stage = 'commit newly initialized Absurd schema';
|
|
3592
3725
|
await client.query('COMMIT');
|
|
3593
3726
|
committed = true;
|
|
3594
3727
|
} catch (error) {
|
|
3595
|
-
|
|
3728
|
+
const primaryError = new Error(
|
|
3729
|
+
`Failed to initialize Absurd schema during ${stage}. ` +
|
|
3730
|
+
`Primary error: ${errorMessage(error)}.`,
|
|
3731
|
+
{ cause: error },
|
|
3732
|
+
);
|
|
3733
|
+
if (transactionStarted && !committed) {
|
|
3596
3734
|
try {
|
|
3597
3735
|
await client.query('ROLLBACK');
|
|
3598
3736
|
} catch (rollbackError) {
|
|
3599
3737
|
throw new AggregateError(
|
|
3600
|
-
[
|
|
3738
|
+
[primaryError, rollbackError],
|
|
3601
3739
|
'Failed to initialize absurd schema and roll back the failed transaction. ' +
|
|
3602
|
-
|
|
3740
|
+
`${primaryError.message} ` +
|
|
3603
3741
|
`Rollback error: ${errorMessage(rollbackError)}.`,
|
|
3604
3742
|
);
|
|
3605
3743
|
}
|
|
3606
3744
|
}
|
|
3607
|
-
throw
|
|
3745
|
+
throw primaryError;
|
|
3608
3746
|
}
|
|
3609
3747
|
}
|
|
3610
3748
|
|
|
@@ -3629,7 +3767,12 @@ export async function submitPostgresAbsurdPlayRun(
|
|
|
3629
3767
|
taskRecoveryRequired: boolean;
|
|
3630
3768
|
}> {
|
|
3631
3769
|
const task = buildAbsurdPlayRunTask({
|
|
3632
|
-
|
|
3770
|
+
// This fence applies to admission only. A repaired historic task must
|
|
3771
|
+
// retain the persisted launch envelope, not the one-shot retry policy.
|
|
3772
|
+
launch: (() => {
|
|
3773
|
+
const { existingOnly: _existingOnly, ...launch } = input;
|
|
3774
|
+
return launch;
|
|
3775
|
+
})(),
|
|
3633
3776
|
schedulerSchema: options?.schema,
|
|
3634
3777
|
});
|
|
3635
3778
|
const submitted = await submitPostgresSchedulerRunTransaction(
|
|
@@ -3689,9 +3832,11 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
3689
3832
|
asyncParentRunAttempt: _asyncParentRunAttempt,
|
|
3690
3833
|
asyncParentRunLeaseToken: _asyncParentRunLeaseToken,
|
|
3691
3834
|
asyncParentAttemptRequired: _asyncParentAttemptRequired,
|
|
3835
|
+
existingOnly: requestedExistingOnly,
|
|
3692
3836
|
...durableInput
|
|
3693
3837
|
} = input;
|
|
3694
3838
|
input = durableInput;
|
|
3839
|
+
const existingOnlySql = requestedExistingOnly === true ? 'true' : 'false';
|
|
3695
3840
|
const releaseId = input.absurdReleaseId?.trim() || null;
|
|
3696
3841
|
const legacyPreviewRelease =
|
|
3697
3842
|
releaseId === 'dev' || releaseId?.startsWith('dev-') === true;
|
|
@@ -3734,6 +3879,7 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
3734
3879
|
NOT admission.existing_run
|
|
3735
3880
|
AND admission.accepting_new
|
|
3736
3881
|
AND admission.parent_accepting
|
|
3882
|
+
AND NOT ${existingOnlySql}
|
|
3737
3883
|
) OR (
|
|
3738
3884
|
${recoverExistingTaskSql}
|
|
3739
3885
|
AND admission.existing_run
|
|
@@ -3921,6 +4067,7 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
3921
4067
|
WHERE NOT existing_run
|
|
3922
4068
|
AND accepting_new
|
|
3923
4069
|
AND parent_accepting
|
|
4070
|
+
AND NOT ${existingOnlySql}
|
|
3924
4071
|
ON CONFLICT (run_id) DO NOTHING
|
|
3925
4072
|
RETURNING run_id, absurd_task_id
|
|
3926
4073
|
), run_recovery AS (
|
|
@@ -4000,7 +4147,7 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
4000
4147
|
)
|
|
4001
4148
|
SELECT (
|
|
4002
4149
|
(NOT admission.existing_run AND admission.accepting_new
|
|
4003
|
-
AND admission.parent_accepting)
|
|
4150
|
+
AND admission.parent_accepting AND NOT ${existingOnlySql})
|
|
4004
4151
|
OR admission.matching_existing
|
|
4005
4152
|
) AS accepted,
|
|
4006
4153
|
admission.accepting_new,
|
|
@@ -4085,6 +4232,11 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
4085
4232
|
`Play run "${input.runId}" already exists with a different launch.`,
|
|
4086
4233
|
);
|
|
4087
4234
|
}
|
|
4235
|
+
if (row?.accepted === false && !row?.existing_run && requestedExistingOnly) {
|
|
4236
|
+
throw new Error(
|
|
4237
|
+
`Play run "${input.runId}" does not exist for an existing-only admission.`,
|
|
4238
|
+
);
|
|
4239
|
+
}
|
|
4088
4240
|
if (row?.accepted === false) {
|
|
4089
4241
|
throw new Error(
|
|
4090
4242
|
`Absurd runtime release "${releaseId}" no longer accepts new launches.`,
|
|
@@ -4098,6 +4250,1140 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
4098
4250
|
};
|
|
4099
4251
|
}
|
|
4100
4252
|
|
|
4253
|
+
/** Immutable Runtime Postgres state for an Absurd-owned Play cron timer. */
|
|
4254
|
+
export type PostgresSchedulerCronSchedule = {
|
|
4255
|
+
scheduleId: string;
|
|
4256
|
+
generation: number;
|
|
4257
|
+
state: 'active' | 'blocked' | 'cancelled';
|
|
4258
|
+
cron: string;
|
|
4259
|
+
timezone: string | null;
|
|
4260
|
+
dueAtMs: number;
|
|
4261
|
+
currentFireTaskId: string | null;
|
|
4262
|
+
currentFireGeneration: number;
|
|
4263
|
+
currentFireQueueName: string | null;
|
|
4264
|
+
/** Immutable source identity only; no executor token or release lane. */
|
|
4265
|
+
sourcePin: PostgresSchedulerCronSourcePin;
|
|
4266
|
+
};
|
|
4267
|
+
|
|
4268
|
+
export type PostgresSchedulerCronFireTaskState =
|
|
4269
|
+
| 'pending'
|
|
4270
|
+
| 'running'
|
|
4271
|
+
| 'sleeping'
|
|
4272
|
+
| 'completed'
|
|
4273
|
+
| 'failed'
|
|
4274
|
+
| 'cancelled';
|
|
4275
|
+
|
|
4276
|
+
/**
|
|
4277
|
+
* Read-only health view for the Runtime-owned cron successor contract.
|
|
4278
|
+
*
|
|
4279
|
+
* A schedule is the source-of-truth cursor. Its current fire task is merely
|
|
4280
|
+
* the durable delivery vehicle for that cursor, so a missing or terminal task
|
|
4281
|
+
* on an active row is an operator-visible integrity failure rather than a
|
|
4282
|
+
* reason to silently advance or recreate an occurrence.
|
|
4283
|
+
*/
|
|
4284
|
+
export type PostgresSchedulerCronHealth = {
|
|
4285
|
+
observedAtMs: number;
|
|
4286
|
+
activeSchedules: number;
|
|
4287
|
+
blockedSchedules: number;
|
|
4288
|
+
cancelledSchedules: number;
|
|
4289
|
+
overdueSchedules: number;
|
|
4290
|
+
oldestOverdueDueAtMs: number | null;
|
|
4291
|
+
missingCurrentFireTasks: number;
|
|
4292
|
+
terminalCurrentFireTasks: number;
|
|
4293
|
+
issues: Array<{
|
|
4294
|
+
scheduleId: string;
|
|
4295
|
+
generation: number;
|
|
4296
|
+
state: 'active' | 'blocked' | 'cancelled';
|
|
4297
|
+
dueAtMs: number;
|
|
4298
|
+
currentFireGeneration: number;
|
|
4299
|
+
currentFireTaskId: string | null;
|
|
4300
|
+
currentFireQueueName: string | null;
|
|
4301
|
+
fireTaskState: PostgresSchedulerCronFireTaskState | null;
|
|
4302
|
+
overdue: boolean;
|
|
4303
|
+
missingCurrentFireTask: boolean;
|
|
4304
|
+
terminalCurrentFireTask: boolean;
|
|
4305
|
+
}>;
|
|
4306
|
+
};
|
|
4307
|
+
|
|
4308
|
+
type PostgresSchedulerCronHealthQueryRow = {
|
|
4309
|
+
schedule_id: string;
|
|
4310
|
+
generation: number | string;
|
|
4311
|
+
state: 'active' | 'blocked' | 'cancelled';
|
|
4312
|
+
due_at_ms: number | string;
|
|
4313
|
+
current_fire_generation: number | string;
|
|
4314
|
+
current_fire_task_id: string | null;
|
|
4315
|
+
current_fire_queue_name: string | null;
|
|
4316
|
+
fire_task_state: string | null;
|
|
4317
|
+
overdue: boolean;
|
|
4318
|
+
missing_current_fire_task: boolean;
|
|
4319
|
+
terminal_current_fire_task: boolean;
|
|
4320
|
+
};
|
|
4321
|
+
|
|
4322
|
+
const POSTGRES_SCHEDULER_CRON_FIRE_TASK_STATES =
|
|
4323
|
+
new Set<PostgresSchedulerCronFireTaskState>([
|
|
4324
|
+
'pending',
|
|
4325
|
+
'running',
|
|
4326
|
+
'sleeping',
|
|
4327
|
+
'completed',
|
|
4328
|
+
'failed',
|
|
4329
|
+
'cancelled',
|
|
4330
|
+
]);
|
|
4331
|
+
|
|
4332
|
+
function postgresSchedulerCronFireTaskState(
|
|
4333
|
+
value: string | null,
|
|
4334
|
+
): PostgresSchedulerCronFireTaskState | null {
|
|
4335
|
+
return value &&
|
|
4336
|
+
POSTGRES_SCHEDULER_CRON_FIRE_TASK_STATES.has(
|
|
4337
|
+
value as PostgresSchedulerCronFireTaskState,
|
|
4338
|
+
)
|
|
4339
|
+
? (value as PostgresSchedulerCronFireTaskState)
|
|
4340
|
+
: null;
|
|
4341
|
+
}
|
|
4342
|
+
|
|
4343
|
+
function postgresSchedulerCronHealthInteger(
|
|
4344
|
+
value: number | string | null | undefined,
|
|
4345
|
+
): number {
|
|
4346
|
+
const parsed = Number(value ?? 0);
|
|
4347
|
+
return Number.isSafeInteger(parsed) ? parsed : 0;
|
|
4348
|
+
}
|
|
4349
|
+
|
|
4350
|
+
function assertPostgresSchedulerCronHealthInput(input: {
|
|
4351
|
+
nowMs?: number;
|
|
4352
|
+
limit?: number;
|
|
4353
|
+
}): { nowMs: number; limit: number } {
|
|
4354
|
+
const nowMs = input.nowMs ?? Date.now();
|
|
4355
|
+
if (!Number.isSafeInteger(nowMs) || nowMs <= 0) {
|
|
4356
|
+
throw new Error('Cron health nowMs must be a positive safe integer.');
|
|
4357
|
+
}
|
|
4358
|
+
const limit = input.limit ?? 50;
|
|
4359
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
|
|
4360
|
+
throw new Error(
|
|
4361
|
+
'Cron health limit must be a safe integer from 1 through 100.',
|
|
4362
|
+
);
|
|
4363
|
+
}
|
|
4364
|
+
return { nowMs, limit };
|
|
4365
|
+
}
|
|
4366
|
+
|
|
4367
|
+
/**
|
|
4368
|
+
* Read a bounded, non-mutating cron integrity report.
|
|
4369
|
+
*
|
|
4370
|
+
* The query joins only registered Absurd queues before asking the engine for a
|
|
4371
|
+
* task. A retired/missing queue therefore becomes a `missingCurrentFireTask`
|
|
4372
|
+
* signal instead of causing this diagnostic endpoint to fail while it is most
|
|
4373
|
+
* needed. Repair remains an explicit, separately fenced operation.
|
|
4374
|
+
*/
|
|
4375
|
+
export async function readPostgresSchedulerCronHealth(
|
|
4376
|
+
client: PostgresSchedulerQueryClient,
|
|
4377
|
+
input: { nowMs?: number; limit?: number } = {},
|
|
4378
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
4379
|
+
): Promise<PostgresSchedulerCronHealth> {
|
|
4380
|
+
const { nowMs, limit } = assertPostgresSchedulerCronHealthInput(input);
|
|
4381
|
+
const schedules = tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options);
|
|
4382
|
+
const healthRowsSql = `
|
|
4383
|
+
WITH schedules AS MATERIALIZED (
|
|
4384
|
+
SELECT schedule_id,
|
|
4385
|
+
generation,
|
|
4386
|
+
state,
|
|
4387
|
+
floor(extract(epoch FROM current_due_at) * 1000)::bigint AS due_at_ms,
|
|
4388
|
+
current_fire_task_id,
|
|
4389
|
+
current_fire_generation,
|
|
4390
|
+
current_fire_queue_name
|
|
4391
|
+
FROM ${schedules}
|
|
4392
|
+
), valid_timer_references AS MATERIALIZED (
|
|
4393
|
+
SELECT schedule.schedule_id,
|
|
4394
|
+
schedule.current_fire_task_id,
|
|
4395
|
+
schedule.current_fire_queue_name
|
|
4396
|
+
FROM schedules AS schedule
|
|
4397
|
+
JOIN absurd.queues AS queue
|
|
4398
|
+
ON queue.queue_name = schedule.current_fire_queue_name
|
|
4399
|
+
WHERE schedule.current_fire_task_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
|
4400
|
+
), timer_states AS MATERIALIZED (
|
|
4401
|
+
SELECT reference.schedule_id,
|
|
4402
|
+
task.state AS fire_task_state
|
|
4403
|
+
FROM valid_timer_references AS reference
|
|
4404
|
+
CROSS JOIN LATERAL absurd.get_task_result(
|
|
4405
|
+
reference.current_fire_queue_name,
|
|
4406
|
+
reference.current_fire_task_id::uuid
|
|
4407
|
+
) AS task
|
|
4408
|
+
), health_rows AS MATERIALIZED (
|
|
4409
|
+
SELECT schedule.schedule_id,
|
|
4410
|
+
schedule.generation,
|
|
4411
|
+
schedule.state,
|
|
4412
|
+
schedule.due_at_ms,
|
|
4413
|
+
schedule.current_fire_generation,
|
|
4414
|
+
schedule.current_fire_task_id,
|
|
4415
|
+
schedule.current_fire_queue_name,
|
|
4416
|
+
timer.fire_task_state,
|
|
4417
|
+
(schedule.state = 'active' AND schedule.due_at_ms <= $1::bigint) AS overdue,
|
|
4418
|
+
(
|
|
4419
|
+
schedule.state = 'active'
|
|
4420
|
+
AND (
|
|
4421
|
+
schedule.current_fire_task_id IS NULL
|
|
4422
|
+
OR schedule.current_fire_queue_name IS NULL
|
|
4423
|
+
OR NOT COALESCE(
|
|
4424
|
+
schedule.current_fire_task_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$',
|
|
4425
|
+
false
|
|
4426
|
+
)
|
|
4427
|
+
OR timer.fire_task_state IS NULL
|
|
4428
|
+
)
|
|
4429
|
+
) AS missing_current_fire_task,
|
|
4430
|
+
(
|
|
4431
|
+
schedule.state = 'active'
|
|
4432
|
+
AND timer.fire_task_state IN ('completed', 'failed', 'cancelled')
|
|
4433
|
+
) AS terminal_current_fire_task
|
|
4434
|
+
FROM schedules AS schedule
|
|
4435
|
+
LEFT JOIN timer_states AS timer
|
|
4436
|
+
ON timer.schedule_id = schedule.schedule_id
|
|
4437
|
+
)`;
|
|
4438
|
+
const summary = await client.query<{
|
|
4439
|
+
active_schedules: number | string;
|
|
4440
|
+
blocked_schedules: number | string;
|
|
4441
|
+
cancelled_schedules: number | string;
|
|
4442
|
+
overdue_schedules: number | string;
|
|
4443
|
+
oldest_overdue_due_at_ms: number | string | null;
|
|
4444
|
+
missing_current_fire_tasks: number | string;
|
|
4445
|
+
terminal_current_fire_tasks: number | string;
|
|
4446
|
+
}>(
|
|
4447
|
+
`${healthRowsSql}
|
|
4448
|
+
SELECT count(*) FILTER (WHERE state = 'active')::text AS active_schedules,
|
|
4449
|
+
count(*) FILTER (WHERE state = 'blocked')::text AS blocked_schedules,
|
|
4450
|
+
count(*) FILTER (WHERE state = 'cancelled')::text AS cancelled_schedules,
|
|
4451
|
+
count(*) FILTER (WHERE overdue)::text AS overdue_schedules,
|
|
4452
|
+
min(due_at_ms) FILTER (WHERE overdue) AS oldest_overdue_due_at_ms,
|
|
4453
|
+
count(*) FILTER (WHERE missing_current_fire_task)::text AS missing_current_fire_tasks,
|
|
4454
|
+
count(*) FILTER (WHERE terminal_current_fire_task)::text AS terminal_current_fire_tasks
|
|
4455
|
+
FROM health_rows`,
|
|
4456
|
+
[nowMs],
|
|
4457
|
+
);
|
|
4458
|
+
const issueRows = await client.query<PostgresSchedulerCronHealthQueryRow>(
|
|
4459
|
+
`${healthRowsSql}
|
|
4460
|
+
SELECT schedule_id,
|
|
4461
|
+
generation,
|
|
4462
|
+
state,
|
|
4463
|
+
due_at_ms,
|
|
4464
|
+
current_fire_generation,
|
|
4465
|
+
current_fire_task_id,
|
|
4466
|
+
current_fire_queue_name,
|
|
4467
|
+
fire_task_state,
|
|
4468
|
+
overdue,
|
|
4469
|
+
missing_current_fire_task,
|
|
4470
|
+
terminal_current_fire_task
|
|
4471
|
+
FROM health_rows
|
|
4472
|
+
WHERE state = 'blocked'
|
|
4473
|
+
OR overdue
|
|
4474
|
+
OR missing_current_fire_task
|
|
4475
|
+
OR terminal_current_fire_task
|
|
4476
|
+
ORDER BY
|
|
4477
|
+
(state = 'blocked') DESC,
|
|
4478
|
+
missing_current_fire_task DESC,
|
|
4479
|
+
terminal_current_fire_task DESC,
|
|
4480
|
+
overdue DESC,
|
|
4481
|
+
due_at_ms ASC,
|
|
4482
|
+
schedule_id ASC
|
|
4483
|
+
LIMIT $2::int`,
|
|
4484
|
+
[nowMs, limit],
|
|
4485
|
+
);
|
|
4486
|
+
const row = summary.rows[0];
|
|
4487
|
+
return {
|
|
4488
|
+
observedAtMs: nowMs,
|
|
4489
|
+
activeSchedules: postgresSchedulerCronHealthInteger(row?.active_schedules),
|
|
4490
|
+
blockedSchedules: postgresSchedulerCronHealthInteger(
|
|
4491
|
+
row?.blocked_schedules,
|
|
4492
|
+
),
|
|
4493
|
+
cancelledSchedules: postgresSchedulerCronHealthInteger(
|
|
4494
|
+
row?.cancelled_schedules,
|
|
4495
|
+
),
|
|
4496
|
+
overdueSchedules: postgresSchedulerCronHealthInteger(
|
|
4497
|
+
row?.overdue_schedules,
|
|
4498
|
+
),
|
|
4499
|
+
oldestOverdueDueAtMs:
|
|
4500
|
+
row?.oldest_overdue_due_at_ms == null
|
|
4501
|
+
? null
|
|
4502
|
+
: postgresSchedulerCronHealthInteger(row.oldest_overdue_due_at_ms),
|
|
4503
|
+
missingCurrentFireTasks: postgresSchedulerCronHealthInteger(
|
|
4504
|
+
row?.missing_current_fire_tasks,
|
|
4505
|
+
),
|
|
4506
|
+
terminalCurrentFireTasks: postgresSchedulerCronHealthInteger(
|
|
4507
|
+
row?.terminal_current_fire_tasks,
|
|
4508
|
+
),
|
|
4509
|
+
issues: issueRows.rows.map((issue) => ({
|
|
4510
|
+
scheduleId: issue.schedule_id,
|
|
4511
|
+
generation: postgresSchedulerCronHealthInteger(issue.generation),
|
|
4512
|
+
state: issue.state,
|
|
4513
|
+
dueAtMs: postgresSchedulerCronHealthInteger(issue.due_at_ms),
|
|
4514
|
+
currentFireGeneration: postgresSchedulerCronHealthInteger(
|
|
4515
|
+
issue.current_fire_generation,
|
|
4516
|
+
),
|
|
4517
|
+
currentFireTaskId: issue.current_fire_task_id,
|
|
4518
|
+
currentFireQueueName: issue.current_fire_queue_name,
|
|
4519
|
+
fireTaskState: postgresSchedulerCronFireTaskState(issue.fire_task_state),
|
|
4520
|
+
overdue: issue.overdue === true,
|
|
4521
|
+
missingCurrentFireTask: issue.missing_current_fire_task === true,
|
|
4522
|
+
terminalCurrentFireTask: issue.terminal_current_fire_task === true,
|
|
4523
|
+
})),
|
|
4524
|
+
};
|
|
4525
|
+
}
|
|
4526
|
+
|
|
4527
|
+
/** A pinned schedule can never become valid by retrying its same fire task. */
|
|
4528
|
+
export class PostgresSchedulerCronScheduleConfigurationError extends Error {
|
|
4529
|
+
constructor(message: string) {
|
|
4530
|
+
super(message);
|
|
4531
|
+
this.name = 'PostgresSchedulerCronScheduleConfigurationError';
|
|
4532
|
+
}
|
|
4533
|
+
}
|
|
4534
|
+
|
|
4535
|
+
/**
|
|
4536
|
+
* The timer stores the facts needed to reconstruct a pinned admission, not a
|
|
4537
|
+
* previously prepared launch. Executor tokens, preloaded session keys,
|
|
4538
|
+
* callback URLs, and release lanes are all intentionally absent: Vercel
|
|
4539
|
+
* creates them at the fire instant.
|
|
4540
|
+
*/
|
|
4541
|
+
export type PostgresSchedulerCronSourcePin = {
|
|
4542
|
+
orgId: string;
|
|
4543
|
+
userId: string | null;
|
|
4544
|
+
playName: string;
|
|
4545
|
+
revisionId: string | null;
|
|
4546
|
+
artifactStorageKey: string;
|
|
4547
|
+
artifactKind: 'cjs_node20' | null;
|
|
4548
|
+
artifactHash: string | null;
|
|
4549
|
+
graphHash: string | null;
|
|
4550
|
+
staticPipeline: unknown | null;
|
|
4551
|
+
input: PlaySchedulerSubmitInput['input'];
|
|
4552
|
+
source: 'published' | 'draft';
|
|
4553
|
+
schedulerSchema: string | null;
|
|
4554
|
+
};
|
|
4555
|
+
|
|
4556
|
+
export function postgresSchedulerCronSourcePinFromLaunch(
|
|
4557
|
+
launch: PlaySchedulerSubmitInput,
|
|
4558
|
+
): PostgresSchedulerCronSourcePin {
|
|
4559
|
+
const artifactKind =
|
|
4560
|
+
launch.runtimeArtifact &&
|
|
4561
|
+
typeof launch.runtimeArtifact === 'object' &&
|
|
4562
|
+
(launch.runtimeArtifact as { artifactKind?: unknown }).artifactKind ===
|
|
4563
|
+
'cjs_node20'
|
|
4564
|
+
? 'cjs_node20'
|
|
4565
|
+
: null;
|
|
4566
|
+
return {
|
|
4567
|
+
orgId: launch.orgId,
|
|
4568
|
+
userId: launch.userId ?? null,
|
|
4569
|
+
playName: launch.playName,
|
|
4570
|
+
revisionId: launch.revisionId ?? null,
|
|
4571
|
+
artifactStorageKey: launch.artifactStorageKey,
|
|
4572
|
+
artifactKind,
|
|
4573
|
+
artifactHash: launch.artifactHash || null,
|
|
4574
|
+
graphHash: launch.graphHash || null,
|
|
4575
|
+
staticPipeline: launch.staticPipeline ?? null,
|
|
4576
|
+
input: launch.input,
|
|
4577
|
+
source: launch.source === 'draft' ? 'draft' : 'published',
|
|
4578
|
+
schedulerSchema: launch.schedulerSchema ?? null,
|
|
4579
|
+
};
|
|
4580
|
+
}
|
|
4581
|
+
|
|
4582
|
+
export type SeedPostgresSchedulerCronScheduleInput = {
|
|
4583
|
+
scheduleId: string;
|
|
4584
|
+
generation: number;
|
|
4585
|
+
dueAtMs: number;
|
|
4586
|
+
cron: string;
|
|
4587
|
+
timezone?: string | null;
|
|
4588
|
+
sourcePin: PostgresSchedulerCronSourcePin;
|
|
4589
|
+
queueName: string;
|
|
4590
|
+
/** Explicit release registration gate; absence is never treated as ready. */
|
|
4591
|
+
releaseId: string;
|
|
4592
|
+
releaseEnvironment: 'preview' | 'production';
|
|
4593
|
+
};
|
|
4594
|
+
|
|
4595
|
+
function assertPostgresSchedulerCronScheduleInput(input: {
|
|
4596
|
+
scheduleId: string;
|
|
4597
|
+
generation: number;
|
|
4598
|
+
fireGeneration?: number;
|
|
4599
|
+
dueAtMs: number;
|
|
4600
|
+
cron?: string;
|
|
4601
|
+
}): void {
|
|
4602
|
+
if (!input.scheduleId.trim()) {
|
|
4603
|
+
throw new Error('Cron schedule requires a non-empty scheduleId.');
|
|
4604
|
+
}
|
|
4605
|
+
if (!Number.isSafeInteger(input.generation) || input.generation <= 0) {
|
|
4606
|
+
throw new Error(
|
|
4607
|
+
'Cron schedule generation must be a positive safe integer.',
|
|
4608
|
+
);
|
|
4609
|
+
}
|
|
4610
|
+
if (
|
|
4611
|
+
input.fireGeneration !== undefined &&
|
|
4612
|
+
(!Number.isSafeInteger(input.fireGeneration) || input.fireGeneration <= 0)
|
|
4613
|
+
) {
|
|
4614
|
+
throw new Error(
|
|
4615
|
+
'Cron schedule fireGeneration must be a positive safe integer.',
|
|
4616
|
+
);
|
|
4617
|
+
}
|
|
4618
|
+
if (!Number.isSafeInteger(input.dueAtMs) || input.dueAtMs <= 0) {
|
|
4619
|
+
throw new Error('Cron schedule dueAtMs must be a positive safe integer.');
|
|
4620
|
+
}
|
|
4621
|
+
if (input.cron !== undefined && !input.cron.trim()) {
|
|
4622
|
+
throw new Error('Cron schedule requires a non-empty cron expression.');
|
|
4623
|
+
}
|
|
4624
|
+
}
|
|
4625
|
+
|
|
4626
|
+
/**
|
|
4627
|
+
* Persist (or generation-replace) one Runtime-owned cron cursor and its first
|
|
4628
|
+
* Absurd timer in a single statement. A caller must leave Convex as the owner
|
|
4629
|
+
* until this returns: the cross-store ownership flip is deliberately separate
|
|
4630
|
+
* from this durable timer primitive.
|
|
4631
|
+
*/
|
|
4632
|
+
export async function seedPostgresSchedulerCronSchedule(
|
|
4633
|
+
client: PostgresSchedulerQueryClient,
|
|
4634
|
+
input: SeedPostgresSchedulerCronScheduleInput,
|
|
4635
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
4636
|
+
): Promise<{
|
|
4637
|
+
created: boolean;
|
|
4638
|
+
scheduleId: string;
|
|
4639
|
+
fireTaskId: string | null;
|
|
4640
|
+
}> {
|
|
4641
|
+
assertPostgresSchedulerCronScheduleInput(input);
|
|
4642
|
+
if (!input.releaseId?.trim()) {
|
|
4643
|
+
throw new Error(
|
|
4644
|
+
'Cron schedule requires an explicitly registered releaseId.',
|
|
4645
|
+
);
|
|
4646
|
+
}
|
|
4647
|
+
const sourcePin = input.sourcePin;
|
|
4648
|
+
const task = buildAbsurdCronFireTask({
|
|
4649
|
+
scheduleId: input.scheduleId.trim(),
|
|
4650
|
+
generation: input.generation,
|
|
4651
|
+
fireGeneration: 1,
|
|
4652
|
+
dueAtMs: input.dueAtMs,
|
|
4653
|
+
schedulerSchema: options?.schema ?? sourcePin.schedulerSchema ?? null,
|
|
4654
|
+
orgId: sourcePin.orgId,
|
|
4655
|
+
});
|
|
4656
|
+
const releaseAdmissionKey = `${input.releaseEnvironment}:${input.releaseId.trim()}`;
|
|
4657
|
+
const result = await client.query<{
|
|
4658
|
+
accepted: boolean;
|
|
4659
|
+
created: boolean;
|
|
4660
|
+
existing_generation: number | null;
|
|
4661
|
+
existing_matches_input: boolean | null;
|
|
4662
|
+
fire_task_id: string | null;
|
|
4663
|
+
}>(
|
|
4664
|
+
`
|
|
4665
|
+
WITH schedule_lock AS MATERIALIZED (
|
|
4666
|
+
SELECT pg_advisory_xact_lock(
|
|
4667
|
+
${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
|
|
4668
|
+
${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
|
|
4669
|
+
'cron-schedule:' || $1::text
|
|
4670
|
+
)
|
|
4671
|
+
)
|
|
4672
|
+
), existing AS MATERIALIZED (
|
|
4673
|
+
SELECT schedule.generation,
|
|
4674
|
+
(
|
|
4675
|
+
schedule.cron = $4::text
|
|
4676
|
+
AND schedule.timezone IS NOT DISTINCT FROM $5::text
|
|
4677
|
+
AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
|
|
4678
|
+
AND schedule.source_pin_json = $6::jsonb
|
|
4679
|
+
) AS matches_input
|
|
4680
|
+
FROM schedule_lock
|
|
4681
|
+
JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
4682
|
+
ON schedule.schedule_id = $1::text
|
|
4683
|
+
FOR UPDATE
|
|
4684
|
+
), release_gate AS MATERIALIZED (
|
|
4685
|
+
SELECT accepting
|
|
4686
|
+
FROM ${tableName('runtime_release_admission', options)}
|
|
4687
|
+
WHERE release_id = $11::text
|
|
4688
|
+
), task_request AS (
|
|
4689
|
+
SELECT $7::text AS queue_name,
|
|
4690
|
+
$8::text AS task_name,
|
|
4691
|
+
$9::jsonb AS params,
|
|
4692
|
+
$10::jsonb AS options
|
|
4693
|
+
WHERE (
|
|
4694
|
+
NOT EXISTS (SELECT 1 FROM existing)
|
|
4695
|
+
OR (SELECT generation FROM existing) < $2::bigint
|
|
4696
|
+
)
|
|
4697
|
+
AND coalesce((SELECT accepting FROM release_gate), false)
|
|
4698
|
+
), task_spawn AS MATERIALIZED (
|
|
4699
|
+
SELECT spawned.task_id::text AS task_id
|
|
4700
|
+
FROM task_request
|
|
4701
|
+
CROSS JOIN LATERAL absurd.spawn_task(
|
|
4702
|
+
task_request.queue_name,
|
|
4703
|
+
task_request.task_name,
|
|
4704
|
+
task_request.params,
|
|
4705
|
+
task_request.options
|
|
4706
|
+
) AS spawned
|
|
4707
|
+
), schedule_upsert AS (
|
|
4708
|
+
INSERT INTO ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule (
|
|
4709
|
+
schedule_id, generation, state, cron, timezone, current_due_at,
|
|
4710
|
+
current_fire_task_id, current_fire_generation, current_fire_queue_name,
|
|
4711
|
+
source_pin_json, launch_json
|
|
4712
|
+
)
|
|
4713
|
+
SELECT $1::text, $2::bigint, 'active', $4::text, $5::text,
|
|
4714
|
+
to_timestamp($3::double precision / 1000.0),
|
|
4715
|
+
(SELECT task_id FROM task_spawn), 1, $7::text, $6::jsonb, $6::jsonb
|
|
4716
|
+
WHERE (
|
|
4717
|
+
NOT EXISTS (SELECT 1 FROM existing)
|
|
4718
|
+
OR (SELECT generation FROM existing) < $2::bigint
|
|
4719
|
+
)
|
|
4720
|
+
AND coalesce((SELECT accepting FROM release_gate), false)
|
|
4721
|
+
ON CONFLICT (schedule_id) DO UPDATE
|
|
4722
|
+
SET generation = excluded.generation,
|
|
4723
|
+
state = excluded.state,
|
|
4724
|
+
cron = excluded.cron,
|
|
4725
|
+
timezone = excluded.timezone,
|
|
4726
|
+
current_due_at = excluded.current_due_at,
|
|
4727
|
+
current_fire_task_id = excluded.current_fire_task_id,
|
|
4728
|
+
current_fire_generation = excluded.current_fire_generation,
|
|
4729
|
+
current_fire_queue_name = excluded.current_fire_queue_name,
|
|
4730
|
+
source_pin_json = excluded.source_pin_json,
|
|
4731
|
+
-- Keep the legacy non-null column free of credentials while old
|
|
4732
|
+
-- schedules drain. New readers use source_pin_json exclusively.
|
|
4733
|
+
launch_json = excluded.launch_json,
|
|
4734
|
+
last_fired_at = NULL,
|
|
4735
|
+
updated_at = now()
|
|
4736
|
+
WHERE schedule.generation < excluded.generation
|
|
4737
|
+
RETURNING schedule_id
|
|
4738
|
+
)
|
|
4739
|
+
SELECT
|
|
4740
|
+
(EXISTS (SELECT 1 FROM schedule_upsert) OR EXISTS (SELECT 1 FROM existing))
|
|
4741
|
+
AS accepted,
|
|
4742
|
+
EXISTS (SELECT 1 FROM schedule_upsert) AS created,
|
|
4743
|
+
(SELECT generation FROM existing) AS existing_generation,
|
|
4744
|
+
(SELECT matches_input FROM existing) AS existing_matches_input,
|
|
4745
|
+
coalesce((SELECT accepting FROM release_gate), false) AS release_accepting,
|
|
4746
|
+
(SELECT task_id FROM task_spawn) AS fire_task_id
|
|
4747
|
+
`,
|
|
4748
|
+
[
|
|
4749
|
+
input.scheduleId.trim(),
|
|
4750
|
+
input.generation,
|
|
4751
|
+
input.dueAtMs,
|
|
4752
|
+
input.cron.trim(),
|
|
4753
|
+
input.timezone?.trim() || null,
|
|
4754
|
+
stringifyPostgresJson(sourcePin),
|
|
4755
|
+
input.queueName,
|
|
4756
|
+
task.taskName,
|
|
4757
|
+
stringifyPostgresJson(task.params),
|
|
4758
|
+
stringifyPostgresJson(task.options),
|
|
4759
|
+
releaseAdmissionKey,
|
|
4760
|
+
],
|
|
4761
|
+
);
|
|
4762
|
+
const row = result.rows[0];
|
|
4763
|
+
if (!row?.accepted) {
|
|
4764
|
+
throw new Error(
|
|
4765
|
+
`Cron schedule "${input.scheduleId}" was not accepted for generation ${input.generation}.`,
|
|
4766
|
+
);
|
|
4767
|
+
}
|
|
4768
|
+
if (
|
|
4769
|
+
typeof row.existing_generation === 'number' &&
|
|
4770
|
+
row.existing_generation > input.generation
|
|
4771
|
+
) {
|
|
4772
|
+
throw new Error(
|
|
4773
|
+
`Cron schedule "${input.scheduleId}" is already owned by newer generation ${row.existing_generation}.`,
|
|
4774
|
+
);
|
|
4775
|
+
}
|
|
4776
|
+
if (
|
|
4777
|
+
row.existing_generation === input.generation &&
|
|
4778
|
+
row.existing_matches_input !== true
|
|
4779
|
+
) {
|
|
4780
|
+
throw new Error(
|
|
4781
|
+
`Cron schedule "${input.scheduleId}" generation ${input.generation} already exists with a different immutable configuration.`,
|
|
4782
|
+
);
|
|
4783
|
+
}
|
|
4784
|
+
return {
|
|
4785
|
+
created: row.created === true,
|
|
4786
|
+
scheduleId: input.scheduleId.trim(),
|
|
4787
|
+
fireTaskId: row.fire_task_id ?? null,
|
|
4788
|
+
};
|
|
4789
|
+
}
|
|
4790
|
+
|
|
4791
|
+
export async function readPostgresSchedulerCronSchedule(
|
|
4792
|
+
client: PostgresSchedulerQueryClient,
|
|
4793
|
+
scheduleId: string,
|
|
4794
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
4795
|
+
): Promise<PostgresSchedulerCronSchedule | null> {
|
|
4796
|
+
const result = await client.query<{
|
|
4797
|
+
schedule_id: string;
|
|
4798
|
+
generation: number;
|
|
4799
|
+
state: 'active' | 'blocked' | 'cancelled';
|
|
4800
|
+
cron: string;
|
|
4801
|
+
timezone: string | null;
|
|
4802
|
+
due_at_ms: number;
|
|
4803
|
+
current_fire_task_id: string | null;
|
|
4804
|
+
current_fire_generation: number;
|
|
4805
|
+
current_fire_queue_name: string | null;
|
|
4806
|
+
source_pin_json: unknown;
|
|
4807
|
+
launch_json: unknown;
|
|
4808
|
+
}>(
|
|
4809
|
+
`
|
|
4810
|
+
SELECT schedule_id,
|
|
4811
|
+
generation,
|
|
4812
|
+
state,
|
|
4813
|
+
cron,
|
|
4814
|
+
timezone,
|
|
4815
|
+
floor(extract(epoch FROM current_due_at) * 1000)::bigint AS due_at_ms,
|
|
4816
|
+
current_fire_task_id,
|
|
4817
|
+
current_fire_generation,
|
|
4818
|
+
current_fire_queue_name,
|
|
4819
|
+
source_pin_json,
|
|
4820
|
+
launch_json
|
|
4821
|
+
FROM ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
4822
|
+
WHERE schedule_id = $1::text
|
|
4823
|
+
`,
|
|
4824
|
+
[scheduleId.trim()],
|
|
4825
|
+
);
|
|
4826
|
+
const row = result.rows[0];
|
|
4827
|
+
if (!row) return null;
|
|
4828
|
+
let sourcePin: PostgresSchedulerCronSourcePin;
|
|
4829
|
+
try {
|
|
4830
|
+
sourcePin = row.source_pin_json
|
|
4831
|
+
? parsePostgresSchedulerCronSourcePin(row.source_pin_json)
|
|
4832
|
+
: postgresSchedulerCronSourcePinFromLaunch(
|
|
4833
|
+
parseLaunchJson(row.launch_json),
|
|
4834
|
+
);
|
|
4835
|
+
} catch (error) {
|
|
4836
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
4837
|
+
`Cron schedule "${row.schedule_id}" has malformed pinned source data: ${
|
|
4838
|
+
error instanceof Error ? error.message : String(error)
|
|
4839
|
+
}`,
|
|
4840
|
+
);
|
|
4841
|
+
}
|
|
4842
|
+
return {
|
|
4843
|
+
scheduleId: row.schedule_id,
|
|
4844
|
+
generation: Number(row.generation),
|
|
4845
|
+
state: row.state,
|
|
4846
|
+
cron: row.cron,
|
|
4847
|
+
timezone: row.timezone,
|
|
4848
|
+
dueAtMs: Number(row.due_at_ms),
|
|
4849
|
+
currentFireTaskId: row.current_fire_task_id ?? null,
|
|
4850
|
+
currentFireGeneration: Number(row.current_fire_generation),
|
|
4851
|
+
currentFireQueueName: row.current_fire_queue_name ?? null,
|
|
4852
|
+
sourcePin,
|
|
4853
|
+
};
|
|
4854
|
+
}
|
|
4855
|
+
|
|
4856
|
+
function cronOccurrenceLaunch(input: {
|
|
4857
|
+
schedule: PostgresSchedulerCronSchedule;
|
|
4858
|
+
launch: PlaySchedulerSubmitInput;
|
|
4859
|
+
schedulerSchema?: string | null;
|
|
4860
|
+
absurdReleaseId?: string | null;
|
|
4861
|
+
}): PlaySchedulerSubmitInput {
|
|
4862
|
+
const identity = buildCronFireRunIdentity({
|
|
4863
|
+
scheduleId: input.schedule.scheduleId,
|
|
4864
|
+
generation: input.schedule.generation,
|
|
4865
|
+
dueAtMs: input.schedule.dueAtMs,
|
|
4866
|
+
playName: input.schedule.sourcePin.playName,
|
|
4867
|
+
});
|
|
4868
|
+
const authority = input.launch.runtimeAuthority;
|
|
4869
|
+
if (
|
|
4870
|
+
!authority ||
|
|
4871
|
+
input.launch.runId !== identity.runId ||
|
|
4872
|
+
authority.workflowId !== identity.workflowId ||
|
|
4873
|
+
authority.runId !== identity.runId
|
|
4874
|
+
) {
|
|
4875
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
4876
|
+
`Cron schedule "${input.schedule.scheduleId}" did not receive a fresh authority for its pinned occurrence.`,
|
|
4877
|
+
);
|
|
4878
|
+
}
|
|
4879
|
+
return {
|
|
4880
|
+
...input.launch,
|
|
4881
|
+
schedulerSchema:
|
|
4882
|
+
input.schedulerSchema ?? input.launch.schedulerSchema ?? null,
|
|
4883
|
+
...(input.absurdReleaseId?.trim()
|
|
4884
|
+
? { absurdReleaseId: input.absurdReleaseId.trim() }
|
|
4885
|
+
: {}),
|
|
4886
|
+
runtimeAuthority: authority,
|
|
4887
|
+
runtimeProtocolVersion: protocolForNewPlayRuntimeLaunch(
|
|
4888
|
+
input.launch.runtimeProtocolVersion,
|
|
4889
|
+
),
|
|
4890
|
+
triggerSource: 'cron',
|
|
4891
|
+
requiresRunCreatedProjection: true,
|
|
4892
|
+
};
|
|
4893
|
+
}
|
|
4894
|
+
|
|
4895
|
+
export type FirePostgresSchedulerCronScheduleInput = {
|
|
4896
|
+
scheduleId: string;
|
|
4897
|
+
generation: number;
|
|
4898
|
+
/** Matches the durable timer-repair fence for this exact occurrence. */
|
|
4899
|
+
fireGeneration: number;
|
|
4900
|
+
dueAtMs: number;
|
|
4901
|
+
queueName: string;
|
|
4902
|
+
/** Built at the due instant by Vercel; never reuse the seed-time authority. */
|
|
4903
|
+
launch: PlaySchedulerSubmitInput;
|
|
4904
|
+
/** Current accepting lane, resolved by the waking worker rather than seed. */
|
|
4905
|
+
absurdReleaseId?: string | null;
|
|
4906
|
+
/** Environment scopes the release-admission fence. */
|
|
4907
|
+
releaseEnvironment?: 'preview' | 'production';
|
|
4908
|
+
};
|
|
4909
|
+
|
|
4910
|
+
/**
|
|
4911
|
+
* The cron OCC boundary. The matching schedule row is locked while this one
|
|
4912
|
+
* CTE inserts the immutable Run, its play-run task, run.created fact, next
|
|
4913
|
+
* cron-fire task, and next cursor. A task replay after commit sees a changed
|
|
4914
|
+
* cursor and is a harmless `superseded` no-op.
|
|
4915
|
+
*/
|
|
4916
|
+
export async function firePostgresSchedulerCronSchedule(
|
|
4917
|
+
client: PostgresSchedulerQueryClient,
|
|
4918
|
+
input: FirePostgresSchedulerCronScheduleInput,
|
|
4919
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
4920
|
+
): Promise<
|
|
4921
|
+
| { status: 'admitted'; runId: string; successorDueAtMs: number }
|
|
4922
|
+
| { status: 'superseded' }
|
|
4923
|
+
> {
|
|
4924
|
+
assertPostgresSchedulerCronScheduleInput(input);
|
|
4925
|
+
const schedule = await readPostgresSchedulerCronSchedule(
|
|
4926
|
+
client,
|
|
4927
|
+
input.scheduleId,
|
|
4928
|
+
options,
|
|
4929
|
+
);
|
|
4930
|
+
if (
|
|
4931
|
+
!schedule ||
|
|
4932
|
+
schedule.state !== 'active' ||
|
|
4933
|
+
schedule.generation !== input.generation ||
|
|
4934
|
+
schedule.dueAtMs !== input.dueAtMs ||
|
|
4935
|
+
schedule.currentFireGeneration !== input.fireGeneration
|
|
4936
|
+
) {
|
|
4937
|
+
return { status: 'superseded' };
|
|
4938
|
+
}
|
|
4939
|
+
let successorDueAtMs: number;
|
|
4940
|
+
try {
|
|
4941
|
+
successorDueAtMs = advanceWorkflowCronSchedule(
|
|
4942
|
+
schedule.cron,
|
|
4943
|
+
Math.max(Date.now(), schedule.dueAtMs),
|
|
4944
|
+
schedule.timezone,
|
|
4945
|
+
);
|
|
4946
|
+
} catch (error) {
|
|
4947
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
4948
|
+
`Cron schedule "${schedule.scheduleId}" has an invalid pinned expression or timezone: ${
|
|
4949
|
+
error instanceof Error ? error.message : String(error)
|
|
4950
|
+
}`,
|
|
4951
|
+
);
|
|
4952
|
+
}
|
|
4953
|
+
if (
|
|
4954
|
+
!Number.isSafeInteger(successorDueAtMs) ||
|
|
4955
|
+
successorDueAtMs <= schedule.dueAtMs
|
|
4956
|
+
) {
|
|
4957
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
4958
|
+
`Cron schedule "${schedule.scheduleId}" did not produce a future successor occurrence.`,
|
|
4959
|
+
);
|
|
4960
|
+
}
|
|
4961
|
+
const launch = cronOccurrenceLaunch({
|
|
4962
|
+
schedule,
|
|
4963
|
+
launch: input.launch,
|
|
4964
|
+
schedulerSchema: options?.schema,
|
|
4965
|
+
absurdReleaseId: input.absurdReleaseId,
|
|
4966
|
+
});
|
|
4967
|
+
const releaseId = launch.absurdReleaseId?.trim();
|
|
4968
|
+
if (!releaseId) {
|
|
4969
|
+
throw new PostgresSchedulerCronScheduleConfigurationError(
|
|
4970
|
+
`Cron schedule "${schedule.scheduleId}" has no fresh Absurd release lane.`,
|
|
4971
|
+
);
|
|
4972
|
+
}
|
|
4973
|
+
const releaseEnvironment =
|
|
4974
|
+
input.releaseEnvironment ?? launch.absurdReleaseEnvironment ?? 'production';
|
|
4975
|
+
const releaseAdmissionKey = `${releaseEnvironment}:${releaseId}`;
|
|
4976
|
+
const playTask = buildAbsurdPlayRunTask({
|
|
4977
|
+
launch,
|
|
4978
|
+
schedulerSchema: options?.schema,
|
|
4979
|
+
});
|
|
4980
|
+
const successorTask = buildAbsurdCronFireTask({
|
|
4981
|
+
scheduleId: schedule.scheduleId,
|
|
4982
|
+
generation: schedule.generation,
|
|
4983
|
+
fireGeneration: 1,
|
|
4984
|
+
dueAtMs: successorDueAtMs,
|
|
4985
|
+
schedulerSchema: options?.schema ?? launch.schedulerSchema ?? null,
|
|
4986
|
+
orgId: launch.orgId,
|
|
4987
|
+
});
|
|
4988
|
+
const result = await client.query<{
|
|
4989
|
+
admitted: boolean;
|
|
4990
|
+
release_accepting: boolean;
|
|
4991
|
+
}>(
|
|
4992
|
+
`
|
|
4993
|
+
WITH schedule_lock AS MATERIALIZED (
|
|
4994
|
+
SELECT pg_advisory_xact_lock(
|
|
4995
|
+
${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
|
|
4996
|
+
${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
|
|
4997
|
+
'cron-schedule:' || $1::text
|
|
4998
|
+
)
|
|
4999
|
+
)
|
|
5000
|
+
), matching_schedule AS MATERIALIZED (
|
|
5001
|
+
SELECT schedule.schedule_id
|
|
5002
|
+
FROM schedule_lock
|
|
5003
|
+
JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5004
|
+
ON schedule.schedule_id = $1::text
|
|
5005
|
+
JOIN ${tableName('runtime_release_admission', options)} AS release_gate
|
|
5006
|
+
ON release_gate.release_id = $16::text
|
|
5007
|
+
WHERE schedule.generation = $2::bigint
|
|
5008
|
+
AND schedule.state = 'active'
|
|
5009
|
+
AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
|
|
5010
|
+
AND schedule.current_fire_generation = $15::bigint
|
|
5011
|
+
AND release_gate.accepting
|
|
5012
|
+
FOR UPDATE
|
|
5013
|
+
), existing_run AS MATERIALIZED (
|
|
5014
|
+
SELECT runs.run_id
|
|
5015
|
+
FROM matching_schedule
|
|
5016
|
+
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = $5::text
|
|
5017
|
+
), play_task_request AS (
|
|
5018
|
+
SELECT $7::text AS queue_name,
|
|
5019
|
+
$8::text AS task_name,
|
|
5020
|
+
$9::jsonb AS params,
|
|
5021
|
+
$10::jsonb AS options
|
|
5022
|
+
WHERE EXISTS (SELECT 1 FROM matching_schedule)
|
|
5023
|
+
AND NOT EXISTS (SELECT 1 FROM existing_run)
|
|
5024
|
+
), play_task_spawn AS MATERIALIZED (
|
|
5025
|
+
SELECT spawned.task_id::text AS task_id
|
|
5026
|
+
FROM play_task_request
|
|
5027
|
+
CROSS JOIN LATERAL absurd.spawn_task(
|
|
5028
|
+
play_task_request.queue_name,
|
|
5029
|
+
play_task_request.task_name,
|
|
5030
|
+
play_task_request.params,
|
|
5031
|
+
play_task_request.options
|
|
5032
|
+
) AS spawned
|
|
5033
|
+
), run_insert AS (
|
|
5034
|
+
INSERT INTO ${tableName('runs', options)} (
|
|
5035
|
+
run_id, org_id, play_id, play_name, graph_hash, runtime_profile,
|
|
5036
|
+
artifact_storage_key, launch_json, input_json, status,
|
|
5037
|
+
absurd_task_id, admission_json
|
|
5038
|
+
)
|
|
5039
|
+
SELECT $5::text,
|
|
5040
|
+
$6::jsonb ->> 'orgId',
|
|
5041
|
+
NULLIF($6::jsonb ->> 'playId', ''),
|
|
5042
|
+
$6::jsonb ->> 'playName',
|
|
5043
|
+
$6::jsonb ->> 'graphHash',
|
|
5044
|
+
$6::jsonb ->> 'runtimeBackend',
|
|
5045
|
+
$6::jsonb ->> 'artifactStorageKey',
|
|
5046
|
+
$6::jsonb,
|
|
5047
|
+
coalesce($6::jsonb -> 'input', '{}'::jsonb),
|
|
5048
|
+
'queued',
|
|
5049
|
+
(SELECT task_id FROM play_task_spawn),
|
|
5050
|
+
$6::jsonb - ARRAY[
|
|
5051
|
+
'executorToken',
|
|
5052
|
+
'preloadedDbSessionUnwrapKey',
|
|
5053
|
+
'preloadedDbSessions',
|
|
5054
|
+
'submittedAtMs',
|
|
5055
|
+
'runtimeDeployVersion',
|
|
5056
|
+
'coordinatorInternalToken',
|
|
5057
|
+
'vercelProtectionBypassToken'
|
|
5058
|
+
]::text[]
|
|
5059
|
+
WHERE EXISTS (SELECT 1 FROM matching_schedule)
|
|
5060
|
+
AND NOT EXISTS (SELECT 1 FROM existing_run)
|
|
5061
|
+
RETURNING run_id
|
|
5062
|
+
), successor_task_request AS (
|
|
5063
|
+
SELECT $11::text AS queue_name,
|
|
5064
|
+
$12::text AS task_name,
|
|
5065
|
+
$13::jsonb AS params,
|
|
5066
|
+
$14::jsonb AS options
|
|
5067
|
+
WHERE EXISTS (SELECT 1 FROM run_insert)
|
|
5068
|
+
), successor_task_spawn AS MATERIALIZED (
|
|
5069
|
+
SELECT spawned.task_id::text AS task_id
|
|
5070
|
+
FROM successor_task_request
|
|
5071
|
+
CROSS JOIN LATERAL absurd.spawn_task(
|
|
5072
|
+
successor_task_request.queue_name,
|
|
5073
|
+
successor_task_request.task_name,
|
|
5074
|
+
successor_task_request.params,
|
|
5075
|
+
successor_task_request.options
|
|
5076
|
+
) AS spawned
|
|
5077
|
+
), outbox_insert AS (
|
|
5078
|
+
INSERT INTO ${tableName('outbox', options)} (
|
|
5079
|
+
event_id, run_id, event_type, payload_json, created_at
|
|
5080
|
+
)
|
|
5081
|
+
SELECT
|
|
5082
|
+
'run-created:' || run_insert.run_id,
|
|
5083
|
+
run_insert.run_id,
|
|
5084
|
+
'run.created',
|
|
5085
|
+
jsonb_strip_nulls(jsonb_build_object(
|
|
5086
|
+
'orgId', $6::jsonb ->> 'orgId',
|
|
5087
|
+
'playId', $6::jsonb ->> 'playId',
|
|
5088
|
+
'playName', $6::jsonb ->> 'playName',
|
|
5089
|
+
'definitionName', $6::jsonb ->> 'definitionName',
|
|
5090
|
+
'playReference', $6::jsonb ->> 'playReference',
|
|
5091
|
+
'definitionScope', $6::jsonb ->> 'definitionScope',
|
|
5092
|
+
'revisionId', $6::jsonb ->> 'revisionId',
|
|
5093
|
+
'workflowFamilyKey', $6::jsonb ->> 'workflowFamilyKey',
|
|
5094
|
+
'artifactStorageKey', $6::jsonb ->> 'artifactStorageKey',
|
|
5095
|
+
'artifactHash', $6::jsonb ->> 'artifactHash',
|
|
5096
|
+
'graphHash', $6::jsonb ->> 'graphHash',
|
|
5097
|
+
'runtimeBackend', $6::jsonb ->> 'runtimeBackend',
|
|
5098
|
+
'schedulerBackend', $6::jsonb ->> 'schedulerBackend',
|
|
5099
|
+
'schedulerSchema', $6::jsonb ->> 'schedulerSchema',
|
|
5100
|
+
'executionProfile', $6::jsonb ->> 'executionProfile',
|
|
5101
|
+
'runtimeReleaseId', $6::jsonb ->> 'runtimeReleaseId',
|
|
5102
|
+
'runtimeDeployVersion', $6::jsonb ->> 'runtimeDeployVersion',
|
|
5103
|
+
'runtimeCallbackBaseUrl', $6::jsonb ->> 'runtimeCallbackBaseUrl',
|
|
5104
|
+
'coordinatorUrl', $6::jsonb ->> 'coordinatorUrl',
|
|
5105
|
+
'coordinatorWorkerName', $6::jsonb ->> 'coordinatorWorkerName',
|
|
5106
|
+
'runtimeWorkflowName', $6::jsonb ->> 'runtimeWorkflowName',
|
|
5107
|
+
'runtimeHarnessWorkerName', $6::jsonb ->> 'runtimeHarnessWorkerName',
|
|
5108
|
+
'maxCreditsPerRun', $6::jsonb -> 'maxCreditsPerRun',
|
|
5109
|
+
'triggerSource', $6::jsonb ->> 'triggerSource',
|
|
5110
|
+
'source', $6::jsonb ->> 'source',
|
|
5111
|
+
'staticPipeline', $6::jsonb -> 'staticPipeline',
|
|
5112
|
+
'inputFileId', $6::jsonb ->> 'inputFileId',
|
|
5113
|
+
'inputBytes', $6::jsonb -> 'inputBytes',
|
|
5114
|
+
'inputSha256', $6::jsonb ->> 'inputSha256',
|
|
5115
|
+
'replayedFromRunId', $6::jsonb ->> 'replayedFromRunId',
|
|
5116
|
+
'parentRunId', $6::jsonb ->> 'parentRunId',
|
|
5117
|
+
'rootRunId', $6::jsonb ->> 'rootRunId',
|
|
5118
|
+
'asyncLaunchKey', $6::jsonb ->> 'asyncLaunchKey',
|
|
5119
|
+
'asyncInvocationKey', $6::jsonb ->> 'asyncInvocationKey',
|
|
5120
|
+
'asyncRequestFingerprint', $6::jsonb ->> 'asyncRequestFingerprint',
|
|
5121
|
+
'asyncAncestryPlayIds', $6::jsonb -> 'asyncAncestryPlayIds',
|
|
5122
|
+
'secretRefs', $6::jsonb -> 'secretRefs',
|
|
5123
|
+
'inputSummary', $6::jsonb -> 'inputSummary'
|
|
5124
|
+
)),
|
|
5125
|
+
${causalOutboxCreatedAt('run_insert.run_id', options)}
|
|
5126
|
+
FROM run_insert
|
|
5127
|
+
ON CONFLICT (event_id) DO NOTHING
|
|
5128
|
+
), schedule_advance AS (
|
|
5129
|
+
UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5130
|
+
SET current_due_at = to_timestamp($4::double precision / 1000.0),
|
|
5131
|
+
current_fire_task_id = (SELECT task_id FROM successor_task_spawn),
|
|
5132
|
+
current_fire_generation = 1,
|
|
5133
|
+
current_fire_queue_name = $11::text,
|
|
5134
|
+
last_fired_at = clock_timestamp(),
|
|
5135
|
+
updated_at = now()
|
|
5136
|
+
WHERE schedule.schedule_id = $1::text
|
|
5137
|
+
AND schedule.generation = $2::bigint
|
|
5138
|
+
AND schedule.state = 'active'
|
|
5139
|
+
AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
|
|
5140
|
+
AND schedule.current_fire_generation = $15::bigint
|
|
5141
|
+
AND EXISTS (SELECT 1 FROM run_insert)
|
|
5142
|
+
RETURNING schedule_id
|
|
5143
|
+
)
|
|
5144
|
+
SELECT EXISTS (SELECT 1 FROM schedule_advance) AS admitted,
|
|
5145
|
+
coalesce(
|
|
5146
|
+
(SELECT accepting
|
|
5147
|
+
FROM ${tableName('runtime_release_admission', options)}
|
|
5148
|
+
WHERE release_id = $16::text),
|
|
5149
|
+
false
|
|
5150
|
+
) AS release_accepting
|
|
5151
|
+
`,
|
|
5152
|
+
[
|
|
5153
|
+
schedule.scheduleId,
|
|
5154
|
+
schedule.generation,
|
|
5155
|
+
schedule.dueAtMs,
|
|
5156
|
+
successorDueAtMs,
|
|
5157
|
+
launch.runId,
|
|
5158
|
+
stringifyPostgresJson(launch),
|
|
5159
|
+
input.queueName,
|
|
5160
|
+
playTask.taskName,
|
|
5161
|
+
stringifyPostgresJson(playTask.params),
|
|
5162
|
+
stringifyPostgresJson(playTask.options),
|
|
5163
|
+
input.queueName,
|
|
5164
|
+
successorTask.taskName,
|
|
5165
|
+
stringifyPostgresJson(successorTask.params),
|
|
5166
|
+
stringifyPostgresJson(successorTask.options),
|
|
5167
|
+
schedule.currentFireGeneration,
|
|
5168
|
+
releaseAdmissionKey,
|
|
5169
|
+
],
|
|
5170
|
+
);
|
|
5171
|
+
if (result.rows[0]?.release_accepting !== true) {
|
|
5172
|
+
throw new Error(
|
|
5173
|
+
`Cron schedule "${schedule.scheduleId}" cannot fire because Absurd release "${releaseAdmissionKey}" is not accepting admissions.`,
|
|
5174
|
+
);
|
|
5175
|
+
}
|
|
5176
|
+
if (result.rows[0]?.admitted !== true) {
|
|
5177
|
+
return { status: 'superseded' };
|
|
5178
|
+
}
|
|
5179
|
+
return {
|
|
5180
|
+
status: 'admitted',
|
|
5181
|
+
runId: launch.runId,
|
|
5182
|
+
successorDueAtMs,
|
|
5183
|
+
};
|
|
5184
|
+
}
|
|
5185
|
+
|
|
5186
|
+
/** Fence a malformed/invalid timer so it cannot repeatedly consume workers. */
|
|
5187
|
+
export async function blockPostgresSchedulerCronSchedule(
|
|
5188
|
+
client: PostgresSchedulerQueryClient,
|
|
5189
|
+
input: Pick<
|
|
5190
|
+
FirePostgresSchedulerCronScheduleInput,
|
|
5191
|
+
'scheduleId' | 'generation' | 'fireGeneration' | 'dueAtMs'
|
|
5192
|
+
> & {
|
|
5193
|
+
reason: string;
|
|
5194
|
+
},
|
|
5195
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
5196
|
+
): Promise<boolean> {
|
|
5197
|
+
const result = await client.query<{ schedule_id: string }>(
|
|
5198
|
+
`
|
|
5199
|
+
UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)}
|
|
5200
|
+
SET state = 'blocked',
|
|
5201
|
+
block_reason = left($4::text, 1_000),
|
|
5202
|
+
updated_at = now()
|
|
5203
|
+
WHERE schedule_id = $1::text
|
|
5204
|
+
AND generation = $2::bigint
|
|
5205
|
+
AND state = 'active'
|
|
5206
|
+
AND current_due_at = to_timestamp($3::double precision / 1000.0)
|
|
5207
|
+
AND current_fire_generation = $5::bigint
|
|
5208
|
+
RETURNING schedule_id
|
|
5209
|
+
`,
|
|
5210
|
+
[
|
|
5211
|
+
input.scheduleId,
|
|
5212
|
+
input.generation,
|
|
5213
|
+
input.dueAtMs,
|
|
5214
|
+
input.reason,
|
|
5215
|
+
input.fireGeneration,
|
|
5216
|
+
],
|
|
5217
|
+
);
|
|
5218
|
+
return result.rows.length > 0;
|
|
5219
|
+
}
|
|
5220
|
+
|
|
5221
|
+
/**
|
|
5222
|
+
* Fence a Runtime-owned schedule before its source binding is disabled,
|
|
5223
|
+
* archived, suspended, or replaced. The advisory lock is shared with fire and
|
|
5224
|
+
* seed, so an already-sleeping task either commits before the lifecycle write
|
|
5225
|
+
* (a run valid at that instant) or observes `cancelled` and becomes a
|
|
5226
|
+
* superseded no-op. It can never admit after this mutation commits.
|
|
5227
|
+
*/
|
|
5228
|
+
export async function cancelPostgresSchedulerCronSchedule(
|
|
5229
|
+
client: PostgresSchedulerQueryClient,
|
|
5230
|
+
input: {
|
|
5231
|
+
scheduleId: string;
|
|
5232
|
+
generation: number;
|
|
5233
|
+
reason: string;
|
|
5234
|
+
},
|
|
5235
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
5236
|
+
): Promise<boolean> {
|
|
5237
|
+
if (!input.scheduleId.trim()) {
|
|
5238
|
+
throw new Error(
|
|
5239
|
+
'Cron schedule cancellation requires a non-empty scheduleId.',
|
|
5240
|
+
);
|
|
5241
|
+
}
|
|
5242
|
+
if (!Number.isSafeInteger(input.generation) || input.generation <= 0) {
|
|
5243
|
+
throw new Error(
|
|
5244
|
+
'Cron schedule cancellation requires a positive generation.',
|
|
5245
|
+
);
|
|
5246
|
+
}
|
|
5247
|
+
const result = await client.query<{ schedule_id: string }>(
|
|
5248
|
+
`
|
|
5249
|
+
WITH schedule_lock AS MATERIALIZED (
|
|
5250
|
+
SELECT pg_advisory_xact_lock(
|
|
5251
|
+
${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
|
|
5252
|
+
${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
|
|
5253
|
+
'cron-schedule:' || $1::text
|
|
5254
|
+
)
|
|
5255
|
+
)
|
|
5256
|
+
)
|
|
5257
|
+
UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5258
|
+
SET state = 'cancelled',
|
|
5259
|
+
cancellation_reason = left($3::text, 1_000),
|
|
5260
|
+
updated_at = now()
|
|
5261
|
+
FROM schedule_lock
|
|
5262
|
+
WHERE schedule.schedule_id = $1::text
|
|
5263
|
+
AND schedule.generation = $2::bigint
|
|
5264
|
+
AND schedule.state = 'active'
|
|
5265
|
+
RETURNING schedule.schedule_id
|
|
5266
|
+
`,
|
|
5267
|
+
[input.scheduleId.trim(), input.generation, input.reason],
|
|
5268
|
+
);
|
|
5269
|
+
return result.rows.length > 0;
|
|
5270
|
+
}
|
|
5271
|
+
|
|
5272
|
+
/**
|
|
5273
|
+
* Replace an observed missing, cancelled, or exhausted timer without changing
|
|
5274
|
+
* its scheduled occurrence. Callers must first inspect the engine task and
|
|
5275
|
+
* pass the observed `fireGeneration`; the CTE then advances that fence and
|
|
5276
|
+
* spawns the replacement in one transaction. A late old timer is superseded
|
|
5277
|
+
* by the generation comparison in `firePostgresSchedulerCronSchedule`.
|
|
5278
|
+
*/
|
|
5279
|
+
export async function repairPostgresSchedulerCronFire(
|
|
5280
|
+
client: PostgresSchedulerQueryClient,
|
|
5281
|
+
input: {
|
|
5282
|
+
scheduleId: string;
|
|
5283
|
+
generation: number;
|
|
5284
|
+
fireGeneration: number;
|
|
5285
|
+
dueAtMs: number;
|
|
5286
|
+
queueName: string;
|
|
5287
|
+
},
|
|
5288
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
5289
|
+
): Promise<{
|
|
5290
|
+
repaired: boolean;
|
|
5291
|
+
fireTaskId: string | null;
|
|
5292
|
+
fireGeneration: number | null;
|
|
5293
|
+
}> {
|
|
5294
|
+
assertPostgresSchedulerCronScheduleInput(input);
|
|
5295
|
+
const schedule = await readPostgresSchedulerCronSchedule(
|
|
5296
|
+
client,
|
|
5297
|
+
input.scheduleId,
|
|
5298
|
+
options,
|
|
5299
|
+
);
|
|
5300
|
+
if (
|
|
5301
|
+
!schedule ||
|
|
5302
|
+
schedule.state !== 'active' ||
|
|
5303
|
+
schedule.generation !== input.generation ||
|
|
5304
|
+
schedule.dueAtMs !== input.dueAtMs ||
|
|
5305
|
+
schedule.currentFireGeneration !== input.fireGeneration
|
|
5306
|
+
) {
|
|
5307
|
+
return { repaired: false, fireTaskId: null, fireGeneration: null };
|
|
5308
|
+
}
|
|
5309
|
+
const replacementGeneration = schedule.currentFireGeneration + 1;
|
|
5310
|
+
const task = buildAbsurdCronFireTask({
|
|
5311
|
+
scheduleId: schedule.scheduleId,
|
|
5312
|
+
generation: schedule.generation,
|
|
5313
|
+
fireGeneration: replacementGeneration,
|
|
5314
|
+
dueAtMs: schedule.dueAtMs,
|
|
5315
|
+
schedulerSchema:
|
|
5316
|
+
options?.schema ?? schedule.sourcePin.schedulerSchema ?? null,
|
|
5317
|
+
orgId: schedule.sourcePin.orgId,
|
|
5318
|
+
});
|
|
5319
|
+
const result = await client.query<{
|
|
5320
|
+
repaired: boolean;
|
|
5321
|
+
fire_task_id: string | null;
|
|
5322
|
+
}>(
|
|
5323
|
+
`
|
|
5324
|
+
WITH schedule_lock AS MATERIALIZED (
|
|
5325
|
+
SELECT pg_advisory_xact_lock(
|
|
5326
|
+
${POSTGRES_SCHEDULER_RELEASE_ADMISSION_LOCK_NAMESPACE},
|
|
5327
|
+
${quoteIdent(schemaName(options))}.runtime_release_admission_lock_id(
|
|
5328
|
+
'cron-schedule:' || $1::text
|
|
5329
|
+
)
|
|
5330
|
+
)
|
|
5331
|
+
), matching_schedule AS MATERIALIZED (
|
|
5332
|
+
SELECT schedule.schedule_id
|
|
5333
|
+
FROM schedule_lock
|
|
5334
|
+
JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5335
|
+
ON schedule.schedule_id = $1::text
|
|
5336
|
+
WHERE schedule.generation = $2::bigint
|
|
5337
|
+
AND schedule.state = 'active'
|
|
5338
|
+
AND schedule.current_due_at = to_timestamp($4::double precision / 1000.0)
|
|
5339
|
+
AND schedule.current_fire_generation = $3::bigint
|
|
5340
|
+
FOR UPDATE
|
|
5341
|
+
), task_spawn AS MATERIALIZED (
|
|
5342
|
+
SELECT spawned.task_id::text AS task_id
|
|
5343
|
+
FROM matching_schedule
|
|
5344
|
+
CROSS JOIN LATERAL absurd.spawn_task(
|
|
5345
|
+
$5::text,
|
|
5346
|
+
$6::text,
|
|
5347
|
+
$7::jsonb,
|
|
5348
|
+
$8::jsonb
|
|
5349
|
+
) AS spawned
|
|
5350
|
+
), repaired_schedule AS (
|
|
5351
|
+
UPDATE ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
|
|
5352
|
+
SET current_fire_task_id = (SELECT task_id FROM task_spawn),
|
|
5353
|
+
current_fire_generation = $9::bigint,
|
|
5354
|
+
current_fire_queue_name = $5::text,
|
|
5355
|
+
updated_at = now()
|
|
5356
|
+
WHERE schedule.schedule_id = $1::text
|
|
5357
|
+
AND schedule.generation = $2::bigint
|
|
5358
|
+
AND schedule.state = 'active'
|
|
5359
|
+
AND schedule.current_due_at = to_timestamp($4::double precision / 1000.0)
|
|
5360
|
+
AND schedule.current_fire_generation = $3::bigint
|
|
5361
|
+
AND EXISTS (SELECT 1 FROM task_spawn)
|
|
5362
|
+
RETURNING schedule.schedule_id
|
|
5363
|
+
)
|
|
5364
|
+
SELECT EXISTS (SELECT 1 FROM repaired_schedule) AS repaired,
|
|
5365
|
+
(SELECT task_id FROM task_spawn) AS fire_task_id
|
|
5366
|
+
`,
|
|
5367
|
+
[
|
|
5368
|
+
schedule.scheduleId,
|
|
5369
|
+
schedule.generation,
|
|
5370
|
+
schedule.currentFireGeneration,
|
|
5371
|
+
schedule.dueAtMs,
|
|
5372
|
+
input.queueName,
|
|
5373
|
+
task.taskName,
|
|
5374
|
+
stringifyPostgresJson(task.params),
|
|
5375
|
+
stringifyPostgresJson(task.options),
|
|
5376
|
+
replacementGeneration,
|
|
5377
|
+
],
|
|
5378
|
+
);
|
|
5379
|
+
const row = result.rows[0];
|
|
5380
|
+
return {
|
|
5381
|
+
repaired: row?.repaired === true,
|
|
5382
|
+
fireTaskId: row?.fire_task_id ?? null,
|
|
5383
|
+
fireGeneration: row?.repaired === true ? replacementGeneration : null,
|
|
5384
|
+
};
|
|
5385
|
+
}
|
|
5386
|
+
|
|
4101
5387
|
export async function setPostgresSchedulerRuntimeReleaseAdmission(
|
|
4102
5388
|
client: PostgresSchedulerQueryClient,
|
|
4103
5389
|
input: {
|
|
@@ -5923,6 +7209,58 @@ export async function readPostgresSchedulerLaunch(
|
|
|
5923
7209
|
}
|
|
5924
7210
|
}
|
|
5925
7211
|
|
|
7212
|
+
/**
|
|
7213
|
+
* One round-trip replay inspection for externally retried admissions. The
|
|
7214
|
+
* caller uses this only to return an already-committed Run without rebuilding
|
|
7215
|
+
* mutable launch dependencies; it is never an admission authority.
|
|
7216
|
+
*/
|
|
7217
|
+
export type PostgresSchedulerAdmissionSnapshot = {
|
|
7218
|
+
runId: string;
|
|
7219
|
+
/** Empty only for historic rows that predate an owning organization. */
|
|
7220
|
+
orgId: string;
|
|
7221
|
+
status: string;
|
|
7222
|
+
absurdTaskId: string | null;
|
|
7223
|
+
launch: PlaySchedulerSubmitInput;
|
|
7224
|
+
};
|
|
7225
|
+
|
|
7226
|
+
export async function readPostgresSchedulerAdmission(
|
|
7227
|
+
client: PostgresSchedulerQueryClient,
|
|
7228
|
+
runId: string,
|
|
7229
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
7230
|
+
): Promise<PostgresSchedulerAdmissionSnapshot | null> {
|
|
7231
|
+
const result = await client.query<{
|
|
7232
|
+
run_id: string;
|
|
7233
|
+
org_id: string | null;
|
|
7234
|
+
status: string;
|
|
7235
|
+
absurd_task_id: string | null;
|
|
7236
|
+
launch_json: unknown;
|
|
7237
|
+
}>(
|
|
7238
|
+
`
|
|
7239
|
+
SELECT runs.run_id,
|
|
7240
|
+
runs.org_id,
|
|
7241
|
+
runs.status,
|
|
7242
|
+
runs.absurd_task_id,
|
|
7243
|
+
runs.launch_json || coalesce(runs.admission_json, '{}'::jsonb)
|
|
7244
|
+
AS launch_json
|
|
7245
|
+
FROM ${tableName('runs', options)} AS runs
|
|
7246
|
+
WHERE runs.run_id = $1
|
|
7247
|
+
`,
|
|
7248
|
+
[runId],
|
|
7249
|
+
);
|
|
7250
|
+
const row = result.rows[0] ?? null;
|
|
7251
|
+
if (!row) return null;
|
|
7252
|
+
return {
|
|
7253
|
+
runId: row.run_id,
|
|
7254
|
+
orgId: row.org_id ?? '',
|
|
7255
|
+
status: row.status,
|
|
7256
|
+
absurdTaskId:
|
|
7257
|
+
typeof row.absurd_task_id === 'string' && row.absurd_task_id.trim()
|
|
7258
|
+
? row.absurd_task_id
|
|
7259
|
+
: null,
|
|
7260
|
+
launch: parseLaunchJson(row.launch_json),
|
|
7261
|
+
};
|
|
7262
|
+
}
|
|
7263
|
+
|
|
5926
7264
|
/**
|
|
5927
7265
|
* Resolve the immutable launch only when the caller proves ownership of the
|
|
5928
7266
|
* currently live scheduler attempt. Used by the Vercel token issuer so a Fly
|
|
@@ -14411,6 +15749,40 @@ function parseLaunchJson(value: unknown): PlaySchedulerSubmitInput {
|
|
|
14411
15749
|
return launch as PlaySchedulerSubmitInput;
|
|
14412
15750
|
}
|
|
14413
15751
|
|
|
15752
|
+
function parsePostgresSchedulerCronSourcePin(
|
|
15753
|
+
value: unknown,
|
|
15754
|
+
): PostgresSchedulerCronSourcePin {
|
|
15755
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
15756
|
+
throw new Error('Cron source pin is not an object.');
|
|
15757
|
+
}
|
|
15758
|
+
const pin = value as Partial<PostgresSchedulerCronSourcePin>;
|
|
15759
|
+
if (
|
|
15760
|
+
typeof pin.orgId !== 'string' ||
|
|
15761
|
+
typeof pin.playName !== 'string' ||
|
|
15762
|
+
typeof pin.artifactStorageKey !== 'string' ||
|
|
15763
|
+
!('input' in pin) ||
|
|
15764
|
+
(pin.source !== 'published' && pin.source !== 'draft')
|
|
15765
|
+
) {
|
|
15766
|
+
throw new Error('Cron source pin is missing required immutable fields.');
|
|
15767
|
+
}
|
|
15768
|
+
return {
|
|
15769
|
+
orgId: pin.orgId,
|
|
15770
|
+
userId: typeof pin.userId === 'string' ? pin.userId : null,
|
|
15771
|
+
playName: pin.playName,
|
|
15772
|
+
revisionId: typeof pin.revisionId === 'string' ? pin.revisionId : null,
|
|
15773
|
+
artifactStorageKey: pin.artifactStorageKey,
|
|
15774
|
+
artifactKind: pin.artifactKind === 'cjs_node20' ? 'cjs_node20' : null,
|
|
15775
|
+
artifactHash:
|
|
15776
|
+
typeof pin.artifactHash === 'string' ? pin.artifactHash : null,
|
|
15777
|
+
graphHash: typeof pin.graphHash === 'string' ? pin.graphHash : null,
|
|
15778
|
+
staticPipeline: pin.staticPipeline ?? null,
|
|
15779
|
+
input: pin.input as PlaySchedulerSubmitInput['input'],
|
|
15780
|
+
source: pin.source,
|
|
15781
|
+
schedulerSchema:
|
|
15782
|
+
typeof pin.schedulerSchema === 'string' ? pin.schedulerSchema : null,
|
|
15783
|
+
};
|
|
15784
|
+
}
|
|
15785
|
+
|
|
14414
15786
|
function errorMessage(error: unknown): string {
|
|
14415
15787
|
if (error && typeof error === 'object') {
|
|
14416
15788
|
const message = (error as { message?: unknown }).message;
|