@dbos-inc/dbos-sdk 5.4.11-preview → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/schemas/system_db_schema.d.ts +7 -0
  2. package/dist/schemas/system_db_schema.d.ts.map +1 -1
  3. package/dist/schemas/system_db_schema.js.map +1 -1
  4. package/dist/src/client.d.ts +26 -2
  5. package/dist/src/client.d.ts.map +1 -1
  6. package/dist/src/client.js +65 -1
  7. package/dist/src/client.js.map +1 -1
  8. package/dist/src/config.d.ts.map +1 -1
  9. package/dist/src/config.js +3 -0
  10. package/dist/src/config.js.map +1 -1
  11. package/dist/src/dbos-executor.d.ts +26 -1
  12. package/dist/src/dbos-executor.d.ts.map +1 -1
  13. package/dist/src/dbos-executor.js +50 -7
  14. package/dist/src/dbos-executor.js.map +1 -1
  15. package/dist/src/dbos.d.ts +74 -5
  16. package/dist/src/dbos.d.ts.map +1 -1
  17. package/dist/src/dbos.js +139 -22
  18. package/dist/src/dbos.js.map +1 -1
  19. package/dist/src/debugpoint.d.ts +1 -0
  20. package/dist/src/debugpoint.d.ts.map +1 -1
  21. package/dist/src/debugpoint.js +4 -2
  22. package/dist/src/debugpoint.js.map +1 -1
  23. package/dist/src/enterprise.d.ts +56 -1
  24. package/dist/src/enterprise.d.ts.map +1 -1
  25. package/dist/src/enterprise.js +26 -4
  26. package/dist/src/enterprise.js.map +1 -1
  27. package/dist/src/error.d.ts +42 -0
  28. package/dist/src/error.d.ts.map +1 -1
  29. package/dist/src/error.js +85 -1
  30. package/dist/src/error.js.map +1 -1
  31. package/dist/src/external_storage.d.ts +27 -0
  32. package/dist/src/external_storage.d.ts.map +1 -0
  33. package/dist/src/external_storage.js +28 -0
  34. package/dist/src/external_storage.js.map +1 -0
  35. package/dist/src/index.d.ts +4 -3
  36. package/dist/src/index.d.ts.map +1 -1
  37. package/dist/src/index.js +2 -1
  38. package/dist/src/index.js.map +1 -1
  39. package/dist/src/scheduler/scheduler.d.ts +78 -3
  40. package/dist/src/scheduler/scheduler.d.ts.map +1 -1
  41. package/dist/src/scheduler/scheduler.js +414 -173
  42. package/dist/src/scheduler/scheduler.js.map +1 -1
  43. package/dist/src/sysdb_migrations/internal/migrations.js +82 -0
  44. package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
  45. package/dist/src/system_database.d.ts +51 -10
  46. package/dist/src/system_database.d.ts.map +1 -1
  47. package/dist/src/system_database.js +418 -207
  48. package/dist/src/system_database.js.map +1 -1
  49. package/dist/src/utils.d.ts +2 -0
  50. package/dist/src/utils.d.ts.map +1 -1
  51. package/dist/src/utils.js +9 -1
  52. package/dist/src/utils.js.map +1 -1
  53. package/dist/src/wfqueue.js +1 -1
  54. package/dist/src/wfqueue.js.map +1 -1
  55. package/dist/src/workflow.d.ts +3 -1
  56. package/dist/src/workflow.d.ts.map +1 -1
  57. package/dist/src/workflow.js +16 -3
  58. package/dist/src/workflow.js.map +1 -1
  59. package/dist/src/workflow_management.d.ts +16 -4
  60. package/dist/src/workflow_management.d.ts.map +1 -1
  61. package/dist/src/workflow_management.js +74 -32
  62. package/dist/src/workflow_management.js.map +1 -1
  63. package/dist/tsconfig.tsbuildinfo +1 -1
  64. package/package.json +2 -2
@@ -32,7 +32,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
32
32
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
33
33
  };
34
34
  Object.defineProperty(exports, "__esModule", { value: true });
35
- exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
35
+ exports.SystemDatabase = exports.createSystemDatabase = exports.withDbRetry = exports.retriablePostgresException = exports.unwrapErrors = exports.isPgDatabaseError = exports.qualify = exports.holdsShardedSchema = exports.verifySystemDatabase = exports.migrateSystemDatabase = exports.ensureSystemDatabase = exports.isCockroachDB = exports.advisoryLockKey = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateIdleTransactionTimeoutMs = exports.compareCodeUnits = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
36
36
  const dbos_executor_1 = require("./dbos-executor");
37
37
  const pg_1 = require("pg");
38
38
  const error_1 = require("./error");
@@ -97,6 +97,11 @@ exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutM
97
97
  exports.DEFAULT_IDLE_TRANSACTION_TIMEOUT_MS = 60_000;
98
98
  // How long a background job's lock may sit idle before Postgres ends its holder's session.
99
99
  exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS = 5000;
100
+ /** Orders strings by code unit, which matches Postgres's "C" collation outside surrogate pairs. */
101
+ function compareCodeUnits(a, b) {
102
+ return a < b ? -1 : a > b ? 1 : 0;
103
+ }
104
+ exports.compareCodeUnits = compareCodeUnits;
100
105
  /** The timeout is rendered into a `SET idle_in_transaction_session_timeout`, which takes only an in-range integer. */
101
106
  function validateIdleTransactionTimeoutMs(value) {
102
107
  if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
@@ -620,6 +625,15 @@ function dbRetry(options = {}) {
620
625
  }
621
626
  /** Build the system database these options describe. */
622
627
  function createSystemDatabase(options) {
628
+ const sysdb = createBaseSystemDatabase(options);
629
+ if (options.suspendMode) {
630
+ const { Suspend } = enterprise.load(enterprise.SUSPEND.description);
631
+ sysdb.suspend = new Suspend(sysdb);
632
+ }
633
+ return sysdb;
634
+ }
635
+ exports.createSystemDatabase = createSystemDatabase;
636
+ function createBaseSystemDatabase(options) {
623
637
  if (options.sharding) {
624
638
  const { ShardedSystemDatabase } = enterprise.load(enterprise.SHARDING.description);
625
639
  return new ShardedSystemDatabase({
@@ -637,7 +651,6 @@ function createSystemDatabase(options) {
637
651
  }
638
652
  return new SystemDatabase(options.systemDatabaseUrl, options.logger, options.serializer, options.sysDbPoolSize, options.systemDatabasePool, options.schemaName, options.useListenNotify, options.pollingConcurrency, options.notificationCoalesceMs, options.appName, options.observabilityQueryTimeoutMs, options.idleTransactionTimeoutMs);
639
653
  }
640
- exports.createSystemDatabase = createSystemDatabase;
641
654
  /**
642
655
  * General notes:
643
656
  * The responsibilities of the `SystemDatabase` are to store data for workflows, and
@@ -681,8 +694,11 @@ class SystemDatabase {
681
694
  dbPollingIntervalStreamMs = 1000;
682
695
  dbPollingIntervalCancelMs = 1000;
683
696
  shouldUseDBNotifications = true;
697
+ // How long a background job's lock may sit idle before Postgres ends its holder's session.
698
+ backgroundJobLockIdleTimeoutMs = exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS;
684
699
  notificationsMap = new NotificationMap();
685
700
  workflowEventsMap = new NotificationMap();
701
+ suspend = undefined;
686
702
  streamsMap = new NotificationMap();
687
703
  customPool = false;
688
704
  // Interval for coalescing LISTEN/NOTIFY notifications pushed off the write path (Postgres + L/N only).
@@ -902,6 +918,23 @@ class SystemDatabase {
902
918
  claimGate(_queueName, _partitionKey, _params) {
903
919
  return 'TRUE';
904
920
  }
921
+ /** Statement whose `acquired` says whether a claim took its budget scope's transaction lock; no row means nothing to claim. */
922
+ claimLockQuery(queueName, partitionKey, params) {
923
+ const name = JSON.stringify(['dbos.queue_claim', this.schemaName, queueName, partitionKey ?? null]);
924
+ params.push(advisoryLockKey(name).toString());
925
+ return `SELECT pg_try_advisory_xact_lock($${params.length}) AS acquired`;
926
+ }
927
+ /** Statement whose `held` lists which of `partitionKeys` a sweep's transaction locked; no row means nothing to claim. */
928
+ partitionLockQuery(queueName, partitionKeys, params) {
929
+ const name = JSON.stringify(['dbos.partition_claim', this.schemaName, queueName]);
930
+ // Two-int keys: the queue, then the partition, hashed so every worker computes the same lock.
931
+ const lockIDs = partitionKeys.map((pk) => (0, crypto_1.createHash)('sha256').update(pk).digest().readInt32BE(0));
932
+ params.push(partitionKeys, lockIDs, Number(BigInt.asIntN(32, advisoryLockKey(name))));
933
+ const n = params.length;
934
+ return `SELECT coalesce(array_agg(p.pk), '{}') AS held
935
+ FROM unnest($${n - 2}::text[], $${n - 1}::int4[]) AS p(pk, lock_id)
936
+ WHERE pg_try_advisory_xact_lock($${n}::int4, p.lock_id)`;
937
+ }
905
938
  /** Predicate confining a statement that names no single workflow to the current shard. */
906
939
  scoped(_alias, _params) {
907
940
  return 'TRUE';
@@ -958,24 +991,29 @@ class SystemDatabase {
958
991
  }
959
992
  return (0, shard_keys_1.bindShardKeys)([[workflowID, key]], callback);
960
993
  }
961
- /** Run `callback` under this application's lock for a background job, passing whether it was taken. */
994
+ /** Run `callback` under this application's lock for a background job, passing undefined if refused, else a call renewing it in place. */
962
995
  async backgroundJobLock(jobName, callback) {
963
996
  // CockroachDB grants every advisory lock, so each process runs every job.
964
997
  if (await this.#isCockroach()) {
965
- return await callback(true);
998
+ return await callback(() => Promise.resolve());
966
999
  }
967
1000
  // Leading 8 bytes of SHA-256, big-endian signed, as for the retention lock.
968
1001
  const key = advisoryLockKey(`dbos.background_job.${this.schemaName}.${this.appName ?? ''}.${jobName}`);
969
- return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock($1) AS acquired FROM set_config('idle_in_transaction_session_timeout', $2, true)`, [key.toString(), String(exports.BACKGROUND_JOB_LOCK_IDLE_TIMEOUT_MS)], callback);
970
- }
971
- /** Take a transaction-scoped background job lock with `lockQuery`, which returns `acquired`, and run `callback`. */
972
- async lockedBackgroundJob(jobName, lockQuery, lockParams, callback) {
1002
+ const timeout = `'${Math.trunc(this.backgroundJobLockIdleTimeoutMs)}'`;
1003
+ return await this.lockedBackgroundJob(jobName, `SELECT pg_try_advisory_xact_lock(${key}) FROM set_config('idle_in_transaction_session_timeout', ${timeout}, true)`,
1004
+ // Snapshot-free, so a concurrent index build never waits on it; fails if the session ended.
1005
+ async (client) => {
1006
+ await client.query(`SET LOCAL idle_in_transaction_session_timeout = ${timeout}`);
1007
+ }, callback);
1008
+ }
1009
+ /** Take a transaction-scoped lock with literal `lockQuery` (its first column says if taken), as a bound statement's portal keeps its snapshot (an xmin) while the session idles. */
1010
+ async lockedBackgroundJob(jobName, lockQuery, renew, callback) {
973
1011
  const client = await this.connect();
974
1012
  try {
975
- await client.query('BEGIN');
1013
+ await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
976
1014
  // The local idle timeout ends a frozen holder's session, releasing its lock.
977
- const { rows } = await client.query(lockQuery, lockParams);
978
- return await callback(rows[0]?.acquired === true);
1015
+ const { rows } = await client.query({ text: lockQuery, rowMode: 'array' });
1016
+ return await callback(rows[0]?.[0] === true ? () => renew(client) : undefined);
979
1017
  }
980
1018
  finally {
981
1019
  try {
@@ -1280,7 +1318,7 @@ class SystemDatabase {
1280
1318
  * Stamp created_at monotonic within each partition key so per-key order holds across batches.
1281
1319
  * Unordered rows (no partition key) get wall-clock time and never touch the cursors.
1282
1320
  */
1283
- async #assignBatchCreatedAt(statuses) {
1321
+ async assignBatchCreatedAt(statuses) {
1284
1322
  const nowMS = Date.now();
1285
1323
  const batchKeys = new Set();
1286
1324
  for (const status of statuses) {
@@ -1325,9 +1363,8 @@ class SystemDatabase {
1325
1363
  * shutdown. Callers retry this themselves.
1326
1364
  */
1327
1365
  async enqueueWorkflows(statuses) {
1328
- const inserted = new Set();
1329
1366
  if (statuses.length === 0)
1330
- return inserted;
1367
+ return new Set();
1331
1368
  for (const status of statuses) {
1332
1369
  if (status.status !== workflow_1.StatusString.ENQUEUED) {
1333
1370
  throw new error_1.DBOSError(`enqueueWorkflows only accepts ${workflow_1.StatusString.ENQUEUED} workflows, but ${status.workflowUUID} is ${status.status}`);
@@ -1336,7 +1373,63 @@ class SystemDatabase {
1336
1373
  throw new error_1.DBOSError(`enqueueWorkflows does not support deduplication IDs, but ${status.workflowUUID} has one`);
1337
1374
  }
1338
1375
  }
1339
- const createdAts = await this.#assignBatchCreatedAt(statuses);
1376
+ // Stamped before the transaction, as seeding a key's cursor reads on another connection.
1377
+ const createdAts = await this.assignBatchCreatedAt(statuses);
1378
+ const client = await this.connect();
1379
+ try {
1380
+ return await this.inTransaction(client, () => this.insertWorkflowBatch(client, statuses, createdAts));
1381
+ }
1382
+ finally {
1383
+ client.release();
1384
+ }
1385
+ }
1386
+ /** Batch-enqueue scheduled runs, skipping existing IDs, stamping last_fired_at from `firedAt` (ID to tick) for those inserted, in one transaction. */
1387
+ async enqueueScheduledWorkflows(statuses, firedAt) {
1388
+ if (statuses.length === 0)
1389
+ return new Set();
1390
+ // Stamped before the transaction, as seeding a key's cursor reads on another connection.
1391
+ const createdAts = await this.assignBatchCreatedAt(statuses);
1392
+ const client = await this.connect();
1393
+ try {
1394
+ return await this.inTransaction(client, async () => {
1395
+ const inserted = await this.insertWorkflowBatch(client, statuses, createdAts);
1396
+ await this.stampLastFiredAt(client, statuses, firedAt, inserted);
1397
+ return inserted;
1398
+ });
1399
+ }
1400
+ finally {
1401
+ client.release();
1402
+ }
1403
+ }
1404
+ /** Set each schedule's last_fired_at to its last inserted tick, `statuses` being in tick order. */
1405
+ async stampLastFiredAt(client, statuses, firedAt, inserted) {
1406
+ const latest = new Map();
1407
+ for (const status of statuses) {
1408
+ const fired = firedAt.get(status.workflowUUID);
1409
+ if (inserted.has(status.workflowUUID) && fired !== undefined && status.scheduleName !== undefined) {
1410
+ latest.set(status.scheduleName, fired);
1411
+ }
1412
+ }
1413
+ if (latest.size === 0)
1414
+ return;
1415
+ // Sorted, like applySchedules, so concurrent writers lock schedule rows in one order.
1416
+ const names = Array.from(latest.keys()).sort(compareCodeUnits);
1417
+ const lockParams = [names];
1418
+ await client.query(`SELECT 1 FROM "${this.schemaName}".workflow_schedules
1419
+ WHERE ${this.global(undefined, lockParams)} AND schedule_name = ANY($1::text[])
1420
+ ORDER BY schedule_name COLLATE "C" FOR UPDATE`, lockParams);
1421
+ const params = [names];
1422
+ const cases = names.map((name) => `WHEN schedule_name = $${params.push(name)} THEN $${params.push(latest.get(name))}`);
1423
+ await client.query(`UPDATE "${this.schemaName}".workflow_schedules SET last_fired_at = CASE ${cases.join(' ')} END
1424
+ WHERE ${this.global(undefined, params)} AND schedule_name = ANY($1::text[])`, params);
1425
+ }
1426
+ /** Insert these ENQUEUED workflows' rows on `client` with `stamped` from assignBatchCreatedAt, skipping existing IDs. Returns the IDs inserted. */
1427
+ async insertWorkflowBatch(client, batch, stamped) {
1428
+ // Inserted in one global order, so batches overlapping across processes wait on each other instead of deadlocking.
1429
+ const order = batch.map((_, i) => i).sort((a, b) => compareCodeUnits(batch[a].workflowUUID, batch[b].workflowUUID));
1430
+ const statuses = order.map((i) => batch[i]);
1431
+ const createdAts = order.map((i) => stamped[i]);
1432
+ const inserted = new Set();
1340
1433
  const columns = [
1341
1434
  'workflow_uuid',
1342
1435
  'status',
@@ -1367,76 +1460,64 @@ class SystemDatabase {
1367
1460
  'schedule_name',
1368
1461
  'application_name',
1369
1462
  ];
1370
- const client = await this.connect();
1371
- try {
1372
- await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1373
- // Chunk to stay well under the bind-parameter limit.
1374
- const chunkSize = 500;
1375
- for (let start = 0; start < statuses.length; start += chunkSize) {
1376
- const chunk = statuses.slice(start, start + chunkSize);
1377
- const keyed = chunk.map((status) => ({
1378
- ...this.wfValues(status.workflowUUID, status.shardKey),
1379
- ...this.parentValues(status.parentWorkflowID),
1380
- }));
1381
- const rows = chunk.map((status, i) => {
1382
- const createdAt = createdAts[start + i];
1383
- return [
1384
- ...Object.values(keyed[i]),
1385
- status.workflowUUID,
1386
- status.status,
1387
- status.workflowName,
1388
- // For cross-language compatibility, these MUST be NULL in the database when not set
1389
- status.workflowClassName === '' ? null : status.workflowClassName,
1390
- status.workflowConfigName === '' ? null : status.workflowConfigName,
1391
- status.queueName ?? null,
1392
- status.authenticatedUser,
1393
- status.assumedRole,
1394
- JSON.stringify(status.authenticatedRoles),
1395
- status.executorId,
1396
- status.applicationVersion ?? null,
1397
- status.applicationID,
1398
- createdAt,
1399
- 0,
1400
- createdAt,
1401
- status.timeoutMS ?? null,
1402
- status.deadlineEpochMS ?? null,
1403
- // Legacy column: the payload lives in workflow_input.
1404
- null,
1405
- null,
1406
- status.priority,
1407
- status.queuePartitionKey ?? null,
1408
- status.parentWorkflowID ?? null,
1409
- status.serialization,
1410
- null,
1411
- status.delayUntilEpochMS ?? null,
1412
- status.attributes ? JSON.stringify(status.attributes) : null,
1413
- status.scheduleName ?? null,
1414
- status.applicationName ?? null,
1415
- ];
1416
- });
1417
- const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
1418
- for (const row of returned) {
1419
- inserted.add(row.workflow_uuid);
1420
- }
1421
- }
1422
- for (let start = 0; start < statuses.length; start += chunkSize) {
1423
- const chunk = statuses.slice(start, start + chunkSize);
1424
- const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
1425
- await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
1463
+ // Chunk to stay well under the bind-parameter limit.
1464
+ const chunkSize = 500;
1465
+ for (let start = 0; start < statuses.length; start += chunkSize) {
1466
+ const chunk = statuses.slice(start, start + chunkSize);
1467
+ const keyed = chunk.map((status) => ({
1468
+ ...this.wfValues(status.workflowUUID, status.shardKey),
1469
+ ...this.parentValues(status.parentWorkflowID),
1470
+ }));
1471
+ const rows = chunk.map((status, i) => {
1472
+ const createdAt = createdAts[start + i];
1473
+ return [
1426
1474
  ...Object.values(keyed[i]),
1427
1475
  status.workflowUUID,
1428
- status.input,
1429
- createdAts[start + i],
1430
- ]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
1476
+ status.status,
1477
+ status.workflowName,
1478
+ // For cross-language compatibility, these MUST be NULL in the database when not set
1479
+ status.workflowClassName === '' ? null : status.workflowClassName,
1480
+ status.workflowConfigName === '' ? null : status.workflowConfigName,
1481
+ status.queueName ?? null,
1482
+ status.authenticatedUser,
1483
+ status.assumedRole,
1484
+ JSON.stringify(status.authenticatedRoles),
1485
+ status.executorId,
1486
+ status.applicationVersion ?? null,
1487
+ status.applicationID,
1488
+ createdAt,
1489
+ 0,
1490
+ createdAt,
1491
+ status.timeoutMS ?? null,
1492
+ status.deadlineEpochMS ?? null,
1493
+ // Legacy column: the payload lives in workflow_input.
1494
+ null,
1495
+ null,
1496
+ status.priority,
1497
+ status.queuePartitionKey ?? null,
1498
+ status.parentWorkflowID ?? null,
1499
+ status.serialization,
1500
+ null,
1501
+ status.delayUntilEpochMS ?? null,
1502
+ status.attributes ? JSON.stringify(status.attributes) : null,
1503
+ status.scheduleName ?? null,
1504
+ status.applicationName ?? null,
1505
+ ];
1506
+ });
1507
+ const returned = await this.insertRows(client, 'workflow_status', [...Object.keys(keyed[0]), ...columns], rows, `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING RETURNING workflow_uuid`);
1508
+ for (const row of returned) {
1509
+ inserted.add(row.workflow_uuid);
1431
1510
  }
1432
- await client.query('COMMIT');
1433
1511
  }
1434
- catch (e) {
1435
- await client.query('ROLLBACK');
1436
- throw e;
1437
- }
1438
- finally {
1439
- client.release();
1512
+ for (let start = 0; start < statuses.length; start += chunkSize) {
1513
+ const chunk = statuses.slice(start, start + chunkSize);
1514
+ const keyed = chunk.map((status) => this.wfValues(status.workflowUUID, status.shardKey));
1515
+ await this.insertRows(client, 'workflow_input', [...Object.keys(keyed[0]), 'workflow_uuid', 'inputs', 'retention_timestamp'], chunk.map((status, i) => [
1516
+ ...Object.values(keyed[i]),
1517
+ status.workflowUUID,
1518
+ status.input,
1519
+ createdAts[start + i],
1520
+ ]), `ON CONFLICT (${this.keyConflict('workflow_uuid')}) DO NOTHING`);
1440
1521
  }
1441
1522
  return inserted;
1442
1523
  }
@@ -1756,13 +1837,20 @@ class SystemDatabase {
1756
1837
  return { isPatched: true, hasEntry: checkpointName === patchName };
1757
1838
  }
1758
1839
  // Nondeprecated - skip matching entry, unpatched if nonmatching entry,
1759
- // If there is no entry, we insert one that indicates it is patched, as an owner-checked checkpoint.
1840
+ // If the history ends before this point, we insert an entry that indicates it is patched, as an owner-checked checkpoint.
1841
+ // If the history has a checkpoint past this point but none here, the execution reached this point on pre-patch code (e.g., recv).
1842
+ // It is unpatched, and nothing is consumed.
1760
1843
  const client = await this.connect();
1761
1844
  try {
1762
1845
  return await this.inTransaction(client, async () => {
1763
- const checkpointName = await readCheckpointName(client);
1764
- if (checkpointName !== undefined) {
1765
- const matches = checkpointName === patchName;
1846
+ const params = [functionID];
1847
+ const { rows } = await client.query(`SELECT function_id, function_name
1848
+ FROM "${this.schemaName}".operation_outputs
1849
+ WHERE ${this.wf(undefined, workflowID, params)} AND function_id>=$1
1850
+ ORDER BY function_id
1851
+ LIMIT 1`, params);
1852
+ if (rows.length !== 0) {
1853
+ const matches = rows[0].function_id === functionID && rows[0].function_name === patchName;
1766
1854
  return { isPatched: matches, hasEntry: matches };
1767
1855
  }
1768
1856
  const dn = Date.now();
@@ -1796,6 +1884,7 @@ class SystemDatabase {
1796
1884
  const params = [workflow_1.StatusString.CANCELLED, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR];
1797
1885
  await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
1798
1886
  SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
1887
+ suspend_key = NULL,
1799
1888
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1800
1889
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
1801
1890
  WHERE ${this.wfIn(undefined, workflowIDs, params)}
@@ -1826,7 +1915,7 @@ class SystemDatabase {
1826
1915
  await client.query(`UPDATE "${this.schemaName}".workflow_status
1827
1916
  SET status = $1, owner_xid = NULL, queue_name = $2, recovery_attempts = 0,
1828
1917
  workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
1829
- started_at_epoch_ms = NULL,
1918
+ started_at_epoch_ms = NULL, suspend_key = NULL,
1830
1919
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
1831
1920
  completed_at = NULL
1832
1921
  WHERE ${this.wfIn(undefined, workflowIDs, updateParams)}
@@ -2043,7 +2132,7 @@ class SystemDatabase {
2043
2132
  throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
2044
2133
  }
2045
2134
  const status = statusRows[0].status;
2046
- if (status === workflow_1.StatusString.PENDING || status === workflow_1.StatusString.ENQUEUED || status === workflow_1.StatusString.DELAYED) {
2135
+ if ((0, workflow_1.isWorkflowActive)(status)) {
2047
2136
  throw new error_1.DBOSError(`Cannot rewind ${workflowID} (${status}): only a workflow in a terminal state can be rewound, so cancel it first`);
2048
2137
  }
2049
2138
  // Roll workflow_events back to the last value published before the cut, using
@@ -2101,7 +2190,7 @@ class SystemDatabase {
2101
2190
  const { rowCount } = await client.query(`UPDATE "${schema}".workflow_status
2102
2191
  SET status = $1, owner_xid = NULL, queue_name = $2, queue_partition_key = $3, recovery_attempts = 0,
2103
2192
  workflow_deadline_epoch_ms = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
2104
- completed_at = NULL, output = NULL, error = NULL,
2193
+ suspend_key = NULL, completed_at = NULL, output = NULL, error = NULL,
2105
2194
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint${setVersion}
2106
2195
  WHERE ${this.wf(undefined, workflowID, params)} AND status = $4`, params);
2107
2196
  if (rowCount !== 1) {
@@ -2608,7 +2697,7 @@ class SystemDatabase {
2608
2697
  const ids = this.wfIn(undefined, workflowIds, params);
2609
2698
  const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2610
2699
  WHERE ${ids}
2611
- AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
2700
+ AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')
2612
2701
  LIMIT 1`, params));
2613
2702
  return rows[0]?.workflow_uuid;
2614
2703
  }
@@ -2634,7 +2723,7 @@ class SystemDatabase {
2634
2723
  const ids = this.wfIn(undefined, workflowIds, params);
2635
2724
  const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
2636
2725
  WHERE ${ids}
2637
- AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, params));
2726
+ AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')`, params));
2638
2727
  return rows.map((row) => row.workflow_uuid);
2639
2728
  }
2640
2729
  // ==================== Sleep ====================
@@ -2708,6 +2797,16 @@ class SystemDatabase {
2708
2797
  return await this.sendDirectInternal(this.pool, destinationID, message, topic, serialization, idempotencyKey, shardKey);
2709
2798
  }
2710
2799
  async sendDirectInternal(db, destinationID, message, topic, serialization, idempotencyKey, shardKey) {
2800
+ if (db === this.pool) {
2801
+ // All sends need a transaction now because they wake up suspended workflows unconditionally.
2802
+ const client = await this.connect();
2803
+ try {
2804
+ return await this.inTransaction(client, () => this.sendDirectInternal(client, destinationID, message, topic, serialization, idempotencyKey, shardKey));
2805
+ }
2806
+ finally {
2807
+ client.release();
2808
+ }
2809
+ }
2711
2810
  topic = topic ?? this.nullTopic;
2712
2811
  // An idempotency key is scoped per destination: suffix it with the recipient's workflow ID.
2713
2812
  const messageUUID = idempotencyKey ? `${idempotencyKey}::${destinationID}` : (0, crypto_1.randomUUID)();
@@ -2729,6 +2828,23 @@ class SystemDatabase {
2729
2828
  }
2730
2829
  throw err;
2731
2830
  }
2831
+ await this.wakeReceivers(db, [destinationID], [topic]);
2832
+ }
2833
+ /**
2834
+ * Make SUSPENDED receivers waiting on one of `topics` due, for the background pass to re-enqueue.
2835
+ * Runs on every send, whether or not this process uses suspend mode, so any sender wakes a workflow
2836
+ * another process parked. A primary-key probe per destination; it matches nothing in the common case.
2837
+ *
2838
+ * The match is destinations x topics, so a batch sending A on topic X and B on topic Y also wakes A
2839
+ * if it waits on Y. The spurious wake-up is harmless: the replayed recv finds no message and parks again.
2840
+ */
2841
+ async wakeReceivers(db, destinations, topics) {
2842
+ const params = [topics];
2843
+ await db.query(
2844
+ // Status is tested in SET, not WHERE, so the planner cannot scan idx_workflow_status_suspended.
2845
+ `UPDATE "${this.schemaName}".workflow_status
2846
+ SET delay_until_epoch_ms = CASE WHEN status = '${workflow_1.StatusString.SUSPENDED}' THEN 0 ELSE delay_until_epoch_ms END
2847
+ WHERE ${this.wfIn(undefined, destinations, params)} AND suspend_key = ANY($1::text[])`, params);
2732
2848
  }
2733
2849
  async recv(workflowID, functionID, timeoutFunctionID, topic, timeoutSeconds = dbos_executor_1.DBOSExecutor.defaultNotificationTimeoutSec, pollingIntervalMs) {
2734
2850
  topic = topic ?? this.nullTopic;
@@ -2782,7 +2898,24 @@ class SystemDatabase {
2782
2898
  this.notificationsMap.deregisterCallback(cbr);
2783
2899
  }
2784
2900
  }
2785
- await this.checkIfCanceled(workflowID);
2901
+ return await this.recvConsume(workflowID, functionID, topic, startTime);
2902
+ }
2903
+ /**
2904
+ * Consume and return the oldest message on `topic`, or null if there is none, recording it as the recv at `functionID`. Idempotent on `functionID`.
2905
+ */
2906
+ async recvConsume(workflowID, functionID, topic, startTime) {
2907
+ topic = topic ?? this.nullTopic;
2908
+ const res = await this.getOperationResultAndThrowIfCancelled(workflowID, functionID);
2909
+ if (res) {
2910
+ if (res.functionName !== exports.DBOS_FUNCNAME_RECV) {
2911
+ throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_RECV, res.functionName);
2912
+ }
2913
+ const ownerXid = (0, context_1.currentOwnerXid)(workflowID);
2914
+ if (ownerXid !== undefined && (await this.getWorkflowOwner(workflowID)) !== ownerXid) {
2915
+ throw new error_1.DBOSWorkflowConflictError(workflowID);
2916
+ }
2917
+ return { serializedValue: res.output, serialization: res.serialization ?? null };
2918
+ }
2786
2919
  // Transactionally consume and return the message if it's in the DB, otherwise return null.
2787
2920
  let message = null;
2788
2921
  let serialization = null;
@@ -3148,15 +3281,16 @@ class SystemDatabase {
3148
3281
  async cancelTimedOutWorkflows(limit) {
3149
3282
  const params = [workflow_1.StatusString.CANCELLED, Date.now(), limit];
3150
3283
  const scope = this.appNameFilter('application_name', this.appName, params);
3151
- // Literal statuses let the planner match idx_workflow_status_deadline under any plan mode.
3284
+ // Literal statuses let the planner match idx_workflow_status_deadline_v2 under any plan mode.
3152
3285
  // SKIP LOCKED leaves a row a dequeue or a peer's sweep holds for the next sweep.
3153
3286
  const { rows } = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
3154
3287
  SET status = $1, owner_xid = NULL, queue_name = NULL, deduplication_id = NULL, started_at_epoch_ms = NULL,
3288
+ suspend_key = NULL,
3155
3289
  updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
3156
3290
  completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
3157
3291
  WHERE workflow_uuid IN (
3158
3292
  SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
3159
- WHERE status IN ('${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.DELAYED}')
3293
+ WHERE status IN ('${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.DELAYED}', '${workflow_1.StatusString.SUSPENDED}')
3160
3294
  AND workflow_deadline_epoch_ms IS NOT NULL
3161
3295
  AND workflow_deadline_epoch_ms <= $2
3162
3296
  AND ${scope}
@@ -3210,17 +3344,49 @@ class SystemDatabase {
3210
3344
  queue.partitionConcurrency !== undefined ||
3211
3345
  queue.rateLimit !== undefined ||
3212
3346
  queue.partitionRateLimit !== undefined;
3213
- // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
3347
+ // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it: lock the whole queue.
3214
3348
  const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
3349
+ // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
3350
+ let maxTasks = Infinity;
3351
+ // The first limit found with no room, which the caller reads as why nothing was claimed.
3352
+ let stopReason = undefined;
3353
+ const limit = (room, reason) => {
3354
+ maxTasks = Math.min(maxTasks, room);
3355
+ if (maxTasks <= 0)
3356
+ stopReason ??= reason;
3357
+ };
3358
+ // This worker's own limits need no database, so a full worker returns before taking the claim lock a peer could use.
3359
+ if (queue.workerConcurrency !== undefined) {
3360
+ // Use the in-memory registry for this worker's running count — avoids a DB round trip.
3361
+ limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
3362
+ }
3363
+ if (queue.partitionWorkerConcurrency !== undefined) {
3364
+ limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
3365
+ }
3366
+ if (maxTasks <= 0) {
3367
+ return { claimedIDs, stopReason };
3368
+ }
3215
3369
  // A sharded database keeps the version row under another key, which the claim's transaction cannot span.
3216
3370
  if (isLatestVersion === undefined && this.sharded) {
3217
3371
  isLatestVersion = await this.isLatestVersion(appVersion);
3218
3372
  }
3373
+ // Row locks only collide when claimers see the same queue head, so serialize claims on the budget's scope; CockroachDB has no advisory locks before 26.3.
3374
+ const useClaimLock = hasSharedBudget && (this.sharded || !(await this.#isCockroach()));
3219
3375
  const client = await this.connect();
3220
3376
  try {
3221
- // Default to READ COMMITTED except with a budget shared across executors
3222
- if (hasSharedBudget) {
3223
- await client.query(`BEGIN ISOLATION LEVEL ${hasWriteSkew ? 'SERIALIZABLE' : 'REPEATABLE READ'}`);
3377
+ if (useClaimLock) {
3378
+ await client.query('BEGIN');
3379
+ const lockParams = [];
3380
+ const { rows } = await client.query(this.claimLockQuery(queue.name, hasWriteSkew ? undefined : queuePartitionKey, lockParams), lockParams);
3381
+ if (rows[0]?.acquired !== true) {
3382
+ await client.query('COMMIT');
3383
+ // Another worker is claiming from this scope, so it still has work; the next poll retries.
3384
+ return rows.length > 0 ? { claimedIDs, stopReason: 'claim_lock' } : { claimedIDs };
3385
+ }
3386
+ }
3387
+ else if (hasSharedBudget) {
3388
+ // CockroachDB, without advisory locks before 26.3: SERIALIZABLE aborts claimers that spend the same budget.
3389
+ await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE');
3224
3390
  }
3225
3391
  else {
3226
3392
  await client.query('BEGIN');
@@ -3254,26 +3420,6 @@ class SystemDatabase {
3254
3420
  WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
3255
3421
  return Number(rows[0]?.count ?? 0);
3256
3422
  };
3257
- // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
3258
- let maxTasks = Infinity;
3259
- // The first limit found with no room, which the caller reads as why nothing was claimed.
3260
- let stopReason = undefined;
3261
- const limit = (room, reason) => {
3262
- maxTasks = Math.min(maxTasks, room);
3263
- if (maxTasks <= 0)
3264
- stopReason ??= reason;
3265
- };
3266
- if (queue.workerConcurrency !== undefined) {
3267
- // Use the in-memory registry for this worker's running count — avoids a DB round trip.
3268
- limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
3269
- }
3270
- if (queue.partitionWorkerConcurrency !== undefined) {
3271
- limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
3272
- }
3273
- if (maxTasks <= 0) {
3274
- await client.query('COMMIT');
3275
- return { claimedIDs, stopReason };
3276
- }
3277
3423
  if (queue.rateLimit !== undefined) {
3278
3424
  // Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
3279
3425
  limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
@@ -3404,7 +3550,9 @@ class SystemDatabase {
3404
3550
  }
3405
3551
  /** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
3406
3552
  partitionedDequeueSweepCap = 8192;
3407
- /** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
3553
+ /** Max partitions a sweep claims per transaction, which bounds the locks it holds at once and for how long. */
3554
+ partitionedDequeueBatchSize = 64;
3555
+ /** Dequeue each partition's head-of-line workflow, at most {@link partitionedDequeueSweepCap} per sweep in transactions of {@link partitionedDequeueBatchSize}; only valid for partition-concurrency-1 queues with no queue-wide limit. */
3408
3556
  async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
3409
3557
  // Written to every claimed row: the dispatches of this claim own their executions.
3410
3558
  ownerXid = null, isLatestVersion) {
@@ -3421,13 +3569,31 @@ class SystemDatabase {
3421
3569
  if (isLatestVersion === undefined && this.sharded) {
3422
3570
  isLatestVersion = await this.isLatestVersion(appVersion);
3423
3571
  }
3572
+ // Sweeps whose heads differ lock disjoint rows, so a claim also takes its partition's lock; CockroachDB, without advisory locks before 26.3, relies on its SERIALIZABLE default.
3573
+ const usePartitionLocks = this.sharded || !(await this.#isCockroach());
3424
3574
  const client = await this.connect();
3425
3575
  try {
3426
- await client.query('BEGIN');
3427
3576
  isLatestVersion ??= await this.isLatestVersion(appVersion, client);
3428
3577
  const versionClause = (n) => isLatestVersion
3429
3578
  ? `(application_version = $${n} OR application_version IS NULL)`
3430
3579
  : `application_version = $${n}`;
3580
+ // The partition's head: $1 queue, $2 ENQUEUED, `version` the app version, `pk` the partition key expression.
3581
+ const headOf = (pk, version, scope) => `SELECT workflow_uuid
3582
+ FROM "${this.schemaName}".workflow_status
3583
+ WHERE queue_name = $1 AND status = $2
3584
+ AND queue_partition_key = ${pk}
3585
+ AND ${versionClause(version)}
3586
+ AND ${scope}
3587
+ -- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
3588
+ ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
3589
+ LIMIT 1`;
3590
+ // No application filter by design: a mutual-exclusion probe must block on any owner's row.
3591
+ const noPending = (pk, pending, scope) => `NOT EXISTS (
3592
+ SELECT 1
3593
+ FROM "${this.schemaName}".workflow_status
3594
+ WHERE ${scope} AND queue_name = $1 AND status = $${pending}
3595
+ AND queue_partition_key IS NOT NULL AND queue_partition_key = ${pk}
3596
+ )`;
3431
3597
  // This worker's own budget bounds the sweep alongside the cap.
3432
3598
  const sweepLimit = Math.min(this.partitionedDequeueSweepCap, maxTasks);
3433
3599
  // When the worker's budget is the binding constraint, probe partitions in random order to prevent starvation.
@@ -3442,6 +3608,7 @@ class SystemDatabase {
3442
3608
  const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
3443
3609
  const probeScope = this.scoped(undefined, candidateParams);
3444
3610
  // Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
3611
+ // The walk takes no lock: each batch below picks its heads again under its partitions' locks.
3445
3612
  const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
3446
3613
  (SELECT MIN(queue_partition_key) AS pk
3447
3614
  FROM "${this.schemaName}".workflow_status
@@ -3457,38 +3624,17 @@ class SystemDatabase {
3457
3624
  SELECT partitions.pk
3458
3625
  FROM partitions
3459
3626
  WHERE partitions.pk IS NOT NULL
3460
- -- No application filter by design: a mutual-exclusion probe must block on any owner's row.
3461
- AND NOT EXISTS (
3462
- SELECT 1
3463
- FROM "${this.schemaName}".workflow_status
3464
- WHERE ${probeScope} AND queue_name = $1 AND status = $4
3465
- AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
3466
- )
3627
+ AND ${noPending('partitions.pk', 4, probeScope)}
3467
3628
  ORDER BY ${sweepOrder}
3468
3629
  LIMIT $5
3469
3630
  )
3470
- SELECT head.workflow_uuid
3631
+ SELECT chosen.pk
3471
3632
  FROM chosen
3472
- -- LATERAL plans as a tight nested loop; a correlated scalar subquery runs as a slower per-row SubPlan.
3473
- JOIN LATERAL (
3474
- SELECT workflow_uuid
3475
- FROM "${this.schemaName}".workflow_status
3476
- WHERE queue_name = $1 AND status = $2
3477
- AND queue_partition_key = chosen.pk
3478
- AND ${versionClause(3)}
3479
- AND ${candidateScope}
3480
- -- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
3481
- ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
3482
- LIMIT 1
3483
- ) head ON TRUE
3484
- -- Which partitions were chosen is settled above; order the result so the claim, and the dispatch it feeds, are deterministic.
3633
+ -- Only partitions with a head this worker may run; LATERAL plans as a tight nested loop.
3634
+ JOIN LATERAL (${headOf('chosen.pk', 3, candidateScope)}) head ON TRUE
3635
+ -- Which partitions were chosen is settled above; order them so the claim, and the dispatch it feeds, are deterministic.
3485
3636
  ORDER BY chosen.pk ASC`, candidateParams);
3486
- const candidateIDs = candidateResult.rows.map((row) => row.workflow_uuid);
3487
- if (candidateIDs.length === 0) {
3488
- await client.query('COMMIT');
3489
- return [];
3490
- }
3491
- await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
3637
+ const partitionKeys = candidateResult.rows.map((row) => row.pk);
3492
3638
  // Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
3493
3639
  const claimGuard = (ids, status, name, version, scope) => `${ids}
3494
3640
  AND status = $${status}
@@ -3496,68 +3642,108 @@ class SystemDatabase {
3496
3642
  AND queue_partition_key IS NOT NULL
3497
3643
  AND ${versionClause(version)}
3498
3644
  AND ${scope}`;
3499
- const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3500
- const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
3501
- const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
3502
- // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
3503
- const lockedResult = await client.query(`SELECT workflow_uuid
3504
- FROM "${this.schemaName}".workflow_status
3505
- WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
3506
- FOR UPDATE SKIP LOCKED`, lockedParams);
3507
- const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
3508
- // Preserve partition order for submission.
3509
- const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
3510
- if (claimIDs.length === 0) {
3511
- await client.query('COMMIT');
3512
- return [];
3645
+ /** Claim the heads of these partitions inside the open transaction, returning the IDs it marked PENDING. */
3646
+ const claimBatch = async (batch) => {
3647
+ let heldPks = batch;
3648
+ if (usePartitionLocks) {
3649
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_BEFORE_LOCKS);
3650
+ const lockParams = [];
3651
+ const { rows } = await client.query(this.partitionLockQuery(queue.name, batch, lockParams), lockParams);
3652
+ // Partitions another sweep holds are left to it, or to the next poll.
3653
+ heldPks = rows[0]?.held ?? [];
3654
+ if (heldPks.length === 0)
3655
+ return [];
3656
+ }
3657
+ // Pick the heads again under the locks, so a previous holder's claims and any newly visible head are seen.
3658
+ const headParams = [queue.name, workflow_1.StatusString.ENQUEUED, appVersion, workflow_1.StatusString.PENDING];
3659
+ const headScope = `${this.scoped(undefined, headParams)} AND ${this.appNameFilter('application_name', this.appName, headParams)}`;
3660
+ const headProbeScope = this.scoped(undefined, headParams);
3661
+ headParams.push(heldPks);
3662
+ const headResult = await client.query(`SELECT head.workflow_uuid
3663
+ FROM unnest($${headParams.length}::text[]) AS chosen(pk)
3664
+ JOIN LATERAL (${headOf('chosen.pk', 3, headScope)}) head ON TRUE
3665
+ WHERE ${noPending('chosen.pk', 4, headProbeScope)}
3666
+ ORDER BY chosen.pk ASC`, headParams);
3667
+ const candidateIDs = headResult.rows.map((row) => row.workflow_uuid);
3668
+ if (candidateIDs.length === 0)
3669
+ return [];
3670
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
3671
+ const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3672
+ const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
3673
+ const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
3674
+ // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
3675
+ const lockedResult = await client.query(`SELECT workflow_uuid
3676
+ FROM "${this.schemaName}".workflow_status
3677
+ WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
3678
+ FOR UPDATE SKIP LOCKED`, lockedParams);
3679
+ const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
3680
+ // Preserve partition order for submission.
3681
+ const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
3682
+ if (claimIDs.length === 0)
3683
+ return [];
3684
+ const flipParams = [
3685
+ workflow_1.StatusString.PENDING,
3686
+ executorID,
3687
+ workflow_1.StatusString.ENQUEUED,
3688
+ queue.name,
3689
+ appVersion,
3690
+ // Claim the row, as the unpartitioned dequeue does.
3691
+ this.appName ?? null,
3692
+ ownerXid,
3693
+ startTimeMs,
3694
+ ];
3695
+ const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
3696
+ const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
3697
+ // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
3698
+ const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3699
+ SET status = $1,
3700
+ owner_xid = $7,
3701
+ executor_id = $2,
3702
+ application_version = $5,
3703
+ started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3704
+ rate_limited = FALSE,
3705
+ application_name = COALESCE(application_name, $6),
3706
+ recovery_attempts = recovery_attempts + 1,
3707
+ updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3708
+ workflow_deadline_epoch_ms = CASE
3709
+ WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
3710
+ CASE
3711
+ WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
3712
+ THEN 9223372036854775807
3713
+ ELSE $8::bigint + workflow_timeout_ms
3714
+ END
3715
+ ELSE workflow_deadline_epoch_ms
3716
+ END
3717
+ WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
3718
+ RETURNING workflow_uuid`, flipParams);
3719
+ const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
3720
+ return claimIDs.filter((id) => flippedIDs.has(id));
3721
+ };
3722
+ // One short transaction per batch, so a sweep holds at most a batch's locks, and only while it claims that batch.
3723
+ const claimedIDs = [];
3724
+ for (let start = 0; start < partitionKeys.length; start += this.partitionedDequeueBatchSize) {
3725
+ const batch = partitionKeys.slice(start, start + this.partitionedDequeueBatchSize);
3726
+ try {
3727
+ await client.query('BEGIN');
3728
+ const batchIDs = await claimBatch(batch);
3729
+ await client.query('COMMIT');
3730
+ claimedIDs.push(...batchIDs);
3731
+ }
3732
+ catch (error) {
3733
+ // Swallowed: a rollback failure must not mask why we are rolling back.
3734
+ await client.query('ROLLBACK').catch(() => undefined);
3735
+ // Earlier batches committed their claims, which must still reach dispatch; the next poll retries the rest.
3736
+ if (claimedIDs.length === 0)
3737
+ throw error;
3738
+ this.logger.warn(`[${queue.name}] stopping a partitioned sweep after ${claimedIDs.length} claim(s): ${String(error)}`);
3739
+ break;
3740
+ }
3513
3741
  }
3514
- const flipParams = [
3515
- workflow_1.StatusString.PENDING,
3516
- executorID,
3517
- workflow_1.StatusString.ENQUEUED,
3518
- queue.name,
3519
- appVersion,
3520
- // Claim the row, as the unpartitioned dequeue does.
3521
- this.appName ?? null,
3522
- ownerXid,
3523
- startTimeMs,
3524
- ];
3525
- const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
3526
- const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
3527
- // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
3528
- const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3529
- SET status = $1,
3530
- owner_xid = $7,
3531
- executor_id = $2,
3532
- application_version = $5,
3533
- started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3534
- rate_limited = FALSE,
3535
- application_name = COALESCE(application_name, $6),
3536
- recovery_attempts = recovery_attempts + 1,
3537
- updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3538
- workflow_deadline_epoch_ms = CASE
3539
- WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
3540
- CASE
3541
- WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
3542
- THEN 9223372036854775807
3543
- ELSE $8::bigint + workflow_timeout_ms
3544
- END
3545
- ELSE workflow_deadline_epoch_ms
3546
- END
3547
- WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
3548
- RETURNING workflow_uuid`, flipParams);
3549
- await client.query('COMMIT');
3550
- const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
3551
- const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
3552
3742
  if (claimedIDs.length > 0) {
3553
3743
  this.logger.debug(`[${queue.name}] dequeueing ${claimedIDs.length} task(s)`);
3554
3744
  }
3555
3745
  return claimedIDs;
3556
3746
  }
3557
- catch (error) {
3558
- await client.query('ROLLBACK');
3559
- throw error;
3560
- }
3561
3747
  finally {
3562
3748
  client.release();
3563
3749
  }
@@ -3655,9 +3841,9 @@ class SystemDatabase {
3655
3841
  // If queuesOnly, filter for queued workflows
3656
3842
  if (input.queuesOnly) {
3657
3843
  whereClauses.push(`queue_name IS NOT NULL`);
3658
- whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1}, $${paramCounter + 2})`);
3659
- params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING, workflow_1.StatusString.DELAYED);
3660
- paramCounter += 3;
3844
+ whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1}, $${paramCounter + 2}, $${paramCounter + 3})`);
3845
+ params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING, workflow_1.StatusString.DELAYED, workflow_1.StatusString.SUSPENDED);
3846
+ paramCounter += 4;
3661
3847
  }
3662
3848
  addFilter('name', input.workflowName);
3663
3849
  addFilter('queue_name', input.queueName);
@@ -3930,7 +4116,9 @@ class SystemDatabase {
3930
4116
  const client = await this.connect();
3931
4117
  try {
3932
4118
  await client.query('BEGIN');
3933
- for (const sched of schedules) {
4119
+ // Sorted, so concurrent writers lock schedule rows in one order.
4120
+ const sorted = [...schedules].sort((a, b) => compareCodeUnits(a.scheduleName, b.scheduleName));
4121
+ for (const sched of sorted) {
3934
4122
  const owner = await this.#resolveRowOwner(client, 'workflow_schedules', 'schedule_name', sched.scheduleName, sched.applicationName, 'Schedule');
3935
4123
  const scheduleKeys = this.scheduleValues(sched);
3936
4124
  const values = {
@@ -4273,7 +4461,12 @@ class SystemDatabase {
4273
4461
  queues = await move('queues');
4274
4462
  schedules = await move('workflow_schedules');
4275
4463
  versions = await move('application_versions');
4276
- inFlight = await move('workflow_status', [workflow_1.StatusString.PENDING, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
4464
+ inFlight = await move('workflow_status', [
4465
+ workflow_1.StatusString.PENDING,
4466
+ workflow_1.StatusString.ENQUEUED,
4467
+ workflow_1.StatusString.DELAYED,
4468
+ workflow_1.StatusString.SUSPENDED,
4469
+ ]);
4277
4470
  await client.query('COMMIT');
4278
4471
  }
4279
4472
  catch (e) {
@@ -4724,6 +4917,12 @@ __decorate([
4724
4917
  __metadata("design:paramtypes", [Array, Number]),
4725
4918
  __metadata("design:returntype", Promise)
4726
4919
  ], SystemDatabase.prototype, "deadLetterWorkflows", null);
4920
+ __decorate([
4921
+ dbRetry(),
4922
+ __metadata("design:type", Function),
4923
+ __metadata("design:paramtypes", [Array, Map]),
4924
+ __metadata("design:returntype", Promise)
4925
+ ], SystemDatabase.prototype, "enqueueScheduledWorkflows", null);
4727
4926
  __decorate([
4728
4927
  dbRetry(),
4729
4928
  __metadata("design:type", Function),
@@ -4868,6 +5067,12 @@ __decorate([
4868
5067
  __metadata("design:paramtypes", [String, Number, Number, String, Number, Number]),
4869
5068
  __metadata("design:returntype", Promise)
4870
5069
  ], SystemDatabase.prototype, "recv", null);
5070
+ __decorate([
5071
+ dbRetry(),
5072
+ __metadata("design:type", Function),
5073
+ __metadata("design:paramtypes", [String, Number, Object, Number]),
5074
+ __metadata("design:returntype", Promise)
5075
+ ], SystemDatabase.prototype, "recvConsume", null);
4871
5076
  __decorate([
4872
5077
  dbRetry(),
4873
5078
  __metadata("design:type", Function),
@@ -4934,4 +5139,10 @@ __decorate([
4934
5139
  __metadata("design:paramtypes", [String]),
4935
5140
  __metadata("design:returntype", Promise)
4936
5141
  ], SystemDatabase.prototype, "getWorkflowOwner", null);
5142
+ __decorate([
5143
+ dbRetry(),
5144
+ __metadata("design:type", Function),
5145
+ __metadata("design:paramtypes", [String, Number, Number, Boolean]),
5146
+ __metadata("design:returntype", Promise)
5147
+ ], SystemDatabase.prototype, "durableSleepInternal", null);
4937
5148
  //# sourceMappingURL=system_database.js.map